#include "kmp.h"
#include "kmp_error.h"
#include "kmp_i18n.h"
#include "kmp_itt.h"
#include "kmp_stats.h"
#include "kmp_str.h"
#include "kmp_lock.h"
#include "kmp_dispatch.h"

Macros
#define	GUIDED_ANALYTICAL_WORKAROUND (x)

#define	OMPT_LOOP_END

#define	OMPT_LOOP_DISPATCH(lb, ub, st, status)

#define	KMP_STATS_LOOP_END /* Nothing */

Functions
void	__kmp_dispatch_deo_error (int gtid_ref, int cid_ref, ident_t *loc_ref)

void	__kmp_dispatch_dxo_error (int gtid_ref, int cid_ref, ident_t *loc_ref)

static int	__kmp_get_monotonicity (ident_t *loc, enum sched_type schedule, bool use_hier=false)

template<typename T >
void	__kmp_initialize_self_buffer (kmp_team_t team, T id, dispatch_private_info_template< T > pr, typename traits_t< T >::unsigned_t nchunks, T nproc, typename traits_t< T >::unsigned_t &init, T &small_chunk, T &extras, T &p_extra)

template<typename T >
void	__kmp_dispatch_init_algorithm (ident_t loc, int gtid, dispatch_private_info_template< T > pr, enum sched_type schedule, T lb, T ub, typename traits_t< T >::signed_t st, typename traits_t< T >::signed_t chunk, T nproc, T tid)

template<typename T >
static void	__kmp_dispatch_init (ident_t *loc, int gtid, enum sched_type schedule, T lb, T ub, typename traits_t< T >::signed_t st, typename traits_t< T >::signed_t chunk, int push_ws)

template<typename UT >
static void	__kmp_dispatch_finish (int gtid, ident_t *loc)

template<typename T >
int	__kmp_dispatch_next_algorithm (int gtid, dispatch_private_info_template< T > pr, dispatch_shared_info_template< T > volatile sh, kmp_int32 p_last, T p_lb, T p_ub, typename traits_t< T >::signed_t p_st, T nproc, T tid)

template<typename T >
static int	__kmp_dispatch_next (ident_t loc, int gtid, kmp_int32 p_last, T p_lb, T p_ub, typename traits_t< T >::signed_t *p_st)

kmp_int32	__kmpc_sections_init (ident_t *loc, kmp_int32 gtid)

kmp_int32	__kmpc_next_section (ident_t *loc, kmp_int32 gtid, kmp_int32 numberOfSections)

void	__kmpc_end_sections (ident_t *loc, kmp_int32 gtid)

template<typename T >
static void	__kmp_dist_get_bounds (ident_t loc, kmp_int32 gtid, kmp_int32 plastiter, T plower, T pupper, typename traits_t< T >::signed_t incr)

kmp_uint32	__kmp_eq_4 (kmp_uint32 value, kmp_uint32 checker)

kmp_uint32	__kmp_neq_4 (kmp_uint32 value, kmp_uint32 checker)

kmp_uint32	__kmp_lt_4 (kmp_uint32 value, kmp_uint32 checker)

kmp_uint32	__kmp_ge_4 (kmp_uint32 value, kmp_uint32 checker)

kmp_uint32	__kmp_le_4 (kmp_uint32 value, kmp_uint32 checker)

kmp_uint32	__kmp_wait_4 (volatile kmp_uint32 spinner, kmp_uint32 checker, kmp_uint32(pred)(kmp_uint32, kmp_uint32), void *obj)

void	__kmp_wait_4_ptr (void spinner, kmp_uint32 checker, kmp_uint32(pred)(void , kmp_uint32), void obj)


void	__kmpc_dispatch_init_4 (ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_int32 lb, kmp_int32 ub, kmp_int32 st, kmp_int32 chunk)

void	__kmpc_dispatch_init_4u (ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_uint32 lb, kmp_uint32 ub, kmp_int32 st, kmp_int32 chunk)
	See __kmpc_dispatch_init_4.

void	__kmpc_dispatch_init_8 (ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_int64 lb, kmp_int64 ub, kmp_int64 st, kmp_int64 chunk)
	See __kmpc_dispatch_init_4.

void	__kmpc_dispatch_init_8u (ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_uint64 lb, kmp_uint64 ub, kmp_int64 st, kmp_int64 chunk)
	See __kmpc_dispatch_init_4.

void	__kmpc_dist_dispatch_init_4 (ident_t loc, kmp_int32 gtid, enum sched_type schedule, kmp_int32 p_last, kmp_int32 lb, kmp_int32 ub, kmp_int32 st, kmp_int32 chunk)
	See __kmpc_dispatch_init_4.

void	__kmpc_dist_dispatch_init_4u (ident_t loc, kmp_int32 gtid, enum sched_type schedule, kmp_int32 p_last, kmp_uint32 lb, kmp_uint32 ub, kmp_int32 st, kmp_int32 chunk)

void	__kmpc_dist_dispatch_init_8 (ident_t loc, kmp_int32 gtid, enum sched_type schedule, kmp_int32 p_last, kmp_int64 lb, kmp_int64 ub, kmp_int64 st, kmp_int64 chunk)

void	__kmpc_dist_dispatch_init_8u (ident_t loc, kmp_int32 gtid, enum sched_type schedule, kmp_int32 p_last, kmp_uint64 lb, kmp_uint64 ub, kmp_int64 st, kmp_int64 chunk)

int	__kmpc_dispatch_next_4 (ident_t loc, kmp_int32 gtid, kmp_int32 p_last, kmp_int32 p_lb, kmp_int32 p_ub, kmp_int32 *p_st)

int	__kmpc_dispatch_next_4u (ident_t loc, kmp_int32 gtid, kmp_int32 p_last, kmp_uint32 p_lb, kmp_uint32 p_ub, kmp_int32 *p_st)
	See __kmpc_dispatch_next_4.

int	__kmpc_dispatch_next_8 (ident_t loc, kmp_int32 gtid, kmp_int32 p_last, kmp_int64 p_lb, kmp_int64 p_ub, kmp_int64 *p_st)
	See __kmpc_dispatch_next_4.

int	__kmpc_dispatch_next_8u (ident_t loc, kmp_int32 gtid, kmp_int32 p_last, kmp_uint64 p_lb, kmp_uint64 p_ub, kmp_int64 *p_st)
	See __kmpc_dispatch_next_4.

void	__kmpc_dispatch_fini_4 (ident_t *loc, kmp_int32 gtid)

void	__kmpc_dispatch_fini_8 (ident_t *loc, kmp_int32 gtid)
	See __kmpc_dispatch_fini_4.

void	__kmpc_dispatch_fini_4u (ident_t *loc, kmp_int32 gtid)
	See __kmpc_dispatch_fini_4.

void	__kmpc_dispatch_fini_8u (ident_t *loc, kmp_int32 gtid)
	See __kmpc_dispatch_fini_4.

Macro Definition Documentation

◆ GUIDED_ANALYTICAL_WORKAROUND

#define GUIDED_ANALYTICAL_WORKAROUND (x)

◆ KMP_STATS_LOOP_END

#define KMP_STATS_LOOP_END /* Nothing */

Definition at line 2175 of file kmp_dispatch.cpp.

◆ OMPT_LOOP_DISPATCH

#define OMPT_LOOP_DISPATCH	(	lb,
		ub,
		st,
		status
	)

Definition at line 2143 of file kmp_dispatch.cpp.

◆ OMPT_LOOP_END

#define OMPT_LOOP_END

Definition at line 2142 of file kmp_dispatch.cpp.

Function Documentation

◆ __kmp_dispatch_deo_error()

void __kmp_dispatch_deo_error	(	int *	gtid_ref,
		int *	cid_ref,
		ident_t *	loc_ref
	)

Definition at line 42 of file kmp_dispatch.cpp.

References __kmp_env_consistency_check, __kmp_push_sync(), __kmp_threads, ct_none, ct_ordered_in_pdo, and KMP_DEBUG_ASSERT.

Referenced by __kmp_dispatch_init(), and __kmpc_sections_init().

◆ __kmp_dispatch_dxo_error()

void __kmp_dispatch_dxo_error	(	int *	gtid_ref,
		int *	cid_ref,
		ident_t *	loc_ref
	)

Definition at line 60 of file kmp_dispatch.cpp.

References __kmp_env_consistency_check, __kmp_pop_sync(), __kmp_threads, ct_none, and ct_ordered_in_pdo.

Referenced by __kmp_dispatch_init(), and __kmpc_sections_init().

◆ __kmp_dispatch_finish()

template<typename UT >

static void __kmp_dispatch_finish	(	int	gtid,
		ident_t *	loc
	)

static

Definition at line 1180 of file kmp_dispatch.cpp.

References __kmp_assert_valid_gtid(), __kmp_str_format(), __kmp_str_free(), __kmp_threads, KD_TRACE, KMP_DEBUG_ASSERT, KMP_MB, dispatch_private_info_template< T >::ordered_bumped, dispatch_private_info_template< T >::private_info_tmpl::p, ST, dispatch_private_info_template< T >::u, and USE_ITT_BUILD_ARG.

◆ __kmp_dispatch_init()

template<typename T >

static void __kmp_dispatch_init	(	ident_t *	loc,
		int	gtid,
		enum sched_type	schedule,
		T	lb,
		T	ub,
		typename traits_t< T >::signed_t	st,
		typename traits_t< T >::signed_t	chunk,
		int	push_ws
	)

static

Definition at line 962 of file kmp_dispatch.cpp.

References __kmp_assert_valid_gtid(), __kmp_dispatch_deo_error(), __kmp_dispatch_dxo_error(), __kmp_dispatch_init_algorithm(), __kmp_dispatch_num_buffers, __kmp_hier_scheds, __kmp_init_parallel, __kmp_parallel_initialize(), __kmp_resume_if_soft_paused(), __kmp_static, __kmp_str_format(), __kmp_str_free(), __kmp_threads, __ompt_get_task_info_object(), __ompt_get_teaminfo(), CCAST, FALSE, dispatch_private_info_template< T >::flags, KD_TRACE, KMP_BUILD_ASSERT, KMP_COUNT_BLOCK, KMP_DEBUG_ASSERT, KMP_MASTER_GTID, kmp_nm_lower, kmp_nm_upper, kmp_ord_lower, KMP_PUSH_PARTITIONED_TIMER, kmp_sch_dynamic_chunked, kmp_sch_guided_analytical_chunked, kmp_sch_guided_iterative_chunked, kmp_sch_guided_simd, kmp_sch_lower, kmp_sch_runtime, kmp_sch_static_balanced, kmp_sch_static_chunked, kmp_sch_static_greedy, loc, ompt_callbacks, ompt_enabled, kmp_sched_flags::ordered, dispatch_private_info_template< T >::private_info_tmpl::p, ompt_team_info_t::parallel_data, dispatch_private_info_template< T >::schedule, SCHEDULE_WITHOUT_MODIFIERS, kmp_hier_sched_env_t::size, ST, kmp_team::t, ompt_task_info_t::task_data, TCR_4, dispatch_private_info_template< T >::u, kmp_sched_flags::use_hier, and USE_ITT_BUILD_ARG.

◆ __kmp_dispatch_init_algorithm()

template<typename T >

void __kmp_dispatch_init_algorithm	(	ident_t *	loc,
		int	gtid,
		dispatch_private_info_template< T > *	pr,
		enum sched_type	schedule,
		T	lb,
		T	ub,
		typename traits_t< T >::signed_t	st,
		typename traits_t< T >::signed_t	chunk,
		T	nproc,
		T	tid
	)

Definition at line 184 of file kmp_dispatch.cpp.

References __kmp_allocate, __kmp_auto, __kmp_dispatch_guided_remaining(), __kmp_env_consistency_check, __kmp_error_construct(), __kmp_fatal(), __kmp_get_monotonicity(), __kmp_guided, __kmp_init_lock(), __kmp_is_hybrid_cpu(), __kmp_msg_null, __kmp_static, __kmp_str_format(), __kmp_str_free(), __kmp_threads, a, b, ct_pdo, ct_pdo_ordered, d, double, FALSE, dispatch_private_info_template< T >::flags, GUIDED_ANALYTICAL_WORKAROUND, guided_flt_param, guided_int_param, i, if(), int, KD_TRACE, KMP_ASSERT, KMP_ASSERT2, KMP_ATOMIC_ST_REL, KMP_COUNT_BLOCK, KMP_COUNT_VALUE, KMP_DEBUG_ASSERT, KMP_DEFAULT_CHUNK, KMP_HNT, KMP_HW_CORE_TYPE_UNKNOWN, KMP_MASTER_GTID, KMP_MSG, kmp_nm_lower, kmp_nm_upper, kmp_ord_lower, kmp_sch_auto, kmp_sch_dynamic_chunked, kmp_sch_guided_analytical_chunked, kmp_sch_guided_chunked, kmp_sch_guided_iterative_chunked, kmp_sch_guided_simd, kmp_sch_lower, kmp_sch_runtime, kmp_sch_runtime_simd, kmp_sch_static, kmp_sch_static_balanced, kmp_sch_static_balanced_chunked, kmp_sch_static_chunked, kmp_sch_static_greedy, kmp_sch_static_steal, kmp_sch_trapezoidal, kmp_sch_upper, KMP_WARNING, loc, kmp_sched_flags::nomerge, kmp_sched_flags::ordered, dispatch_private_info_template< T >::ordered_bumped, dispatch_private_info_template< T >::private_info_tmpl::p, p, SCHEDULE_MONOTONIC, SCHEDULE_NONMONOTONIC, SCHEDULE_WITHOUT_MODIFIERS, ST, dispatch_private_info_template< T >::steal_flag, kmp_team::t, target(), TRUE, type, dispatch_private_info_template< T >::type_size, dispatch_private_info_template< T >::u, kmp_sched_flags::use_hier, and kmp_sched_flags::use_hybrid.

Referenced by __kmp_dispatch_init(), and kmp_hier_t< T >::next().

◆ __kmp_dispatch_next()

template<typename T >

static int __kmp_dispatch_next	(	ident_t *	loc,
		int	gtid,
		kmp_int32 *	p_last,
		T *	p_lb,
		T *	p_ub,
		typename traits_t< T >::signed_t *	p_st
	)

static

Definition at line 2179 of file kmp_dispatch.cpp.

References __kmp_assert_valid_gtid(), __kmp_destroy_lock(), __kmp_dispatch_num_buffers, __kmp_env_consistency_check, __kmp_free, __kmp_pop_workshare(), __kmp_str_format(), __kmp_str_free(), __kmp_threads, buf, dispatch_shared_info_template< T >::buffer_index, ct_none, FALSE, dispatch_private_info_template< T >::flags, i, dispatch_shared_infoXX_template< T >::iteration, KD_TRACE, KMP_ASSERT, KMP_ATOMIC_ST_RLX, KMP_DEBUG_ASSERT, KMP_MB, kmp_sch_static_steal, KMP_STATS_LOOP_END, KMP_TIME_PARTITIONED_BLOCK, lck, loc, kmp_sched_flags::nomerge, dispatch_shared_infoXX_template< T >::num_done, OMPT_LOOP_DISPATCH, OMPT_LOOP_END, kmp_sched_flags::ordered, dispatch_shared_infoXX_template< T >::ordered_iteration, dispatch_private_info_template< T >::private_info_tmpl::p, dispatch_private_info_template< T >::pushed_ws, dispatch_shared_info_template< T >::shared_info_tmpl::s, dispatch_private_info_template< T >::schedule, ST, status, kmp_team::t, TRUE, dispatch_private_info_template< T >::u, dispatch_shared_info_template< T >::u, and kmp_sched_flags::use_hier.

◆ __kmp_dispatch_next_algorithm()

template<typename T >

int __kmp_dispatch_next_algorithm	(	int	gtid,
		dispatch_private_info_template< T > *	pr,
		dispatch_shared_info_template< T > volatile *	sh,
		kmp_int32 *	p_last,
		T *	p_lb,
		T *	p_ub,
		typename traits_t< T >::signed_t *	p_st,
		T	nproc,
		T	tid
	)

Definition at line 1322 of file kmp_dispatch.cpp.

References __kmp_acquire_lock(), __kmp_dispatch_num_buffers, __kmp_fatal(), __kmp_msg_null, __kmp_release_lock(), __kmp_str_format(), __kmp_str_free(), __kmp_threads, __kmp_type_convert(), b, chunk_size, count, dispatch_private_info_template< T >::flags, dispatch_shared_infoXX_template< T >::iteration, KD_TRACE, KMP_ASSERT, KMP_ATOMIC_LD_ACQ, KMP_ATOMIC_LD_RLX, KMP_ATOMIC_ST_REL, KMP_COMPARE_AND_STORE_REL64, KMP_COUNT_DEVELOPER_VALUE, KMP_CPU_PAUSE, KMP_DEBUG_ASSERT, KMP_HNT, KMP_MSG, kmp_sch_dynamic_chunked, kmp_sch_guided_analytical_chunked, kmp_sch_guided_iterative_chunked, kmp_sch_guided_simd, kmp_sch_static_balanced, kmp_sch_static_chunked, kmp_sch_static_greedy, kmp_sch_static_steal, kmp_sch_trapezoidal, KMP_XCHG_FIXED64, lck, kmp_sched_flags::ordered, dispatch_private_info_template< T >::private_info_tmpl::p, p, RCAST, dispatch_shared_info_template< T >::shared_info_tmpl::s, dispatch_private_info_template< T >::schedule, ST, status, dispatch_private_info_template< T >::steal_flag, kmp_team::t, dispatch_private_info_template< T >::u, dispatch_shared_info_template< T >::u, and VOLATILE_CAST.

◆ __kmp_dist_get_bounds()

template<typename T >

static void __kmp_dist_get_bounds	(	ident_t *	loc,
		kmp_int32	gtid,
		kmp_int32 *	plastiter,
		T *	plower,
		T *	pupper,
		typename traits_t< T >::signed_t	incr
	)

static

Definition at line 2664 of file kmp_dispatch.cpp.

References __kmp_assert_valid_gtid(), __kmp_env_consistency_check, __kmp_error_construct(), __kmp_static, __kmp_str_format(), __kmp_str_free(), __kmp_threads, ct_pdo, KD_TRACE, KE_TRACE, KMP_DEBUG_ASSERT, kmp_sch_static_balanced, kmp_sch_static_greedy, loc, ST, and kmp_team::t.

◆ __kmp_eq_4()

kmp_uint32 __kmp_eq_4	(	kmp_uint32	value,
		kmp_uint32	checker
	)

Definition at line 3017 of file kmp_dispatch.cpp.

References checker, and value.

Referenced by __kmpc_doacross_init().

◆ __kmp_ge_4()

kmp_uint32 __kmp_ge_4	(	kmp_uint32	value,
		kmp_uint32	checker
	)

Definition at line 3029 of file kmp_dispatch.cpp.

References checker, and value.

◆ __kmp_get_monotonicity()

static int __kmp_get_monotonicity	(	ident_t *	loc,
		enum sched_type	schedule,
		bool	use_hier = `false`
	)

inlinestatic

Definition at line 72 of file kmp_dispatch.cpp.

References __kmp_force_monotonic, loc, SCHEDULE_HAS_MONOTONIC, SCHEDULE_HAS_NONMONOTONIC, SCHEDULE_MONOTONIC, and SCHEDULE_NONMONOTONIC.

Referenced by __kmp_dispatch_init_algorithm().

◆ __kmp_initialize_self_buffer()

template<typename T >

void __kmp_initialize_self_buffer	(	kmp_team_t *	team,
		T	id,
		dispatch_private_info_template< T > *	pr,
		typename traits_t< T >::unsigned_t	nchunks,
		T	nproc,
		typename traits_t< T >::unsigned_t &	init,
		T &	small_chunk,
		T &	extras,
		T &	p_extra
	)

inline

Definition at line 105 of file kmp_dispatch.cpp.

References __kmp_gtid_from_tid(), __kmp_threads, dispatch_private_info_template< T >::flags, dispatch_private_info_template< T >::private_info_tmpl::p, type, dispatch_private_info_template< T >::u, and kmp_sched_flags::use_hybrid.

◆ __kmp_le_4()

kmp_uint32 __kmp_le_4	(	kmp_uint32	value,
		kmp_uint32	checker
	)

Definition at line 3033 of file kmp_dispatch.cpp.

References checker, and value.

◆ __kmp_lt_4()

kmp_uint32 __kmp_lt_4	(	kmp_uint32	value,
		kmp_uint32	checker
	)

Definition at line 3025 of file kmp_dispatch.cpp.

References checker, and value.

◆ __kmp_neq_4()

kmp_uint32 __kmp_neq_4	(	kmp_uint32	value,
		kmp_uint32	checker
	)

Definition at line 3021 of file kmp_dispatch.cpp.

References checker, and value.

◆ __kmp_wait_4()

kmp_uint32 __kmp_wait_4	(	volatile kmp_uint32 *	spinner,
		kmp_uint32	checker,
		kmp_uint32(*)(kmp_uint32, kmp_uint32)	pred,
		void *	obj
	)

Definition at line 3038 of file kmp_dispatch.cpp.

References CCAST, checker, f(), KMP_FSYNC_SPIN_ACQUIRED, KMP_FSYNC_SPIN_INIT, KMP_FSYNC_SPIN_PREPARE, KMP_INIT_BACKOFF, KMP_INIT_YIELD, kmp_uint32, KMP_YIELD_OVERSUB_ELSE_SPIN, r, and TCR_4.

Referenced by __kmpc_doacross_init().

◆ __kmp_wait_4_ptr()

void __kmp_wait_4_ptr	(	void *	spinner,
		kmp_uint32	checker,
		kmp_uint32()(void , kmp_uint32)	pred,
		void *	obj
	)

Definition at line 3066 of file kmp_dispatch.cpp.

References checker, f(), KMP_FSYNC_SPIN_ACQUIRED, KMP_FSYNC_SPIN_INIT, KMP_FSYNC_SPIN_PREPARE, KMP_INIT_BACKOFF, KMP_INIT_YIELD, kmp_uint32, and KMP_YIELD_OVERSUB_ELSE_SPIN.

Macros

Functions

Macro Definition Documentation

◆ GUIDED_ANALYTICAL_WORKAROUND

◆ KMP_STATS_LOOP_END

◆ OMPT_LOOP_DISPATCH

◆ OMPT_LOOP_END

Function Documentation

◆ __kmp_dispatch_deo_error()

◆ __kmp_dispatch_dxo_error()

◆ __kmp_dispatch_finish()

◆ __kmp_dispatch_init()

◆ __kmp_dispatch_init_algorithm()

◆ __kmp_dispatch_next()

◆ __kmp_dispatch_next_algorithm()

◆ __kmp_dist_get_bounds()

◆ __kmp_eq_4()

◆ __kmp_ge_4()

◆ __kmp_get_monotonicity()

◆ __kmp_initialize_self_buffer()

◆ __kmp_le_4()

◆ __kmp_lt_4()

◆ __kmp_neq_4()

◆ __kmp_wait_4()

◆ __kmp_wait_4_ptr()