kmp_dispatch.cpp File Reference

#include "kmp.h"
#include "kmp_error.h"
#include "kmp_i18n.h"
#include "kmp_itt.h"
#include "kmp_stats.h"
#include "kmp_str.h"
#include "kmp_lock.h"
#include "kmp_dispatch.h"

Go to the source code of this file.

Macros
#define	GUIDED_ANALYTICAL_WORKAROUND   (x)
#define	OMPT_LOOP_END
#define	OMPT_LOOP_DISPATCH(lb, ub, st, status)
#define	KMP_STATS_LOOP_END   /* Nothing */

Functions
void	__kmp_dispatch_deo_error (int *gtid_ref, int *cid_ref, ident_t *loc_ref)
void	__kmp_dispatch_dxo_error (int *gtid_ref, int *cid_ref, ident_t *loc_ref)
static int	__kmp_get_monotonicity (ident_t *loc, enum sched_type schedule, bool use_hier=false)
template<typename T>
void	__kmp_initialize_self_buffer (kmp_team_t *team, T id, dispatch_private_info_template< T > *pr, typename traits_t< T >::unsigned_t nchunks, T nproc, typename traits_t< T >::unsigned_t &init, T &small_chunk, T &extras, T &p_extra)
template<typename T>
void	__kmp_dispatch_init_algorithm (ident_t *loc, int gtid, dispatch_private_info_template< T > *pr, enum sched_type schedule, T lb, T ub, typename traits_t< T >::signed_t st, typename traits_t< T >::signed_t chunk, T nproc, T tid)
template<typename T>
static void	__kmp_dispatch_init (ident_t *loc, int gtid, enum sched_type schedule, T lb, T ub, typename traits_t< T >::signed_t st, typename traits_t< T >::signed_t chunk, int push_ws)
template<typename UT>
static void	__kmp_dispatch_finish (int gtid, ident_t *loc)
template<typename T>
int	__kmp_dispatch_next_algorithm (int gtid, dispatch_private_info_template< T > *pr, dispatch_shared_info_template< T > volatile *sh, kmp_int32 *p_last, T *p_lb, T *p_ub, typename traits_t< T >::signed_t *p_st, T nproc, T tid)
template<typename T>
static int	__kmp_dispatch_next (ident_t *loc, int gtid, kmp_int32 *p_last, T *p_lb, T *p_ub, typename traits_t< T >::signed_t *p_st)
kmp_int32	__kmpc_sections_init (ident_t *loc, kmp_int32 gtid)
kmp_int32	__kmpc_next_section (ident_t *loc, kmp_int32 gtid, kmp_int32 numberOfSections)
void	__kmpc_end_sections (ident_t *loc, kmp_int32 gtid)
template<typename T>
static void	__kmp_dist_get_bounds (ident_t *loc, kmp_int32 gtid, kmp_int32 *plastiter, T *plower, T *pupper, typename traits_t< T >::signed_t incr)
kmp_uint32	__kmp_eq_4 (kmp_uint32 value, kmp_uint32 checker)
kmp_uint32	__kmp_neq_4 (kmp_uint32 value, kmp_uint32 checker)
kmp_uint32	__kmp_lt_4 (kmp_uint32 value, kmp_uint32 checker)
kmp_uint32	__kmp_ge_4 (kmp_uint32 value, kmp_uint32 checker)
kmp_uint32	__kmp_le_4 (kmp_uint32 value, kmp_uint32 checker)
kmp_uint32	__kmp_wait_4 (volatile kmp_uint32 *spinner, kmp_uint32 checker, kmp_uint32(*pred)(kmp_uint32, kmp_uint32), void *obj)
void	__kmp_wait_4_ptr (void *spinner, kmp_uint32 checker, kmp_uint32(*pred)(void *, kmp_uint32), void *obj)
void	__kmpc_dispatch_init_4 (ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_int32 lb, kmp_int32 ub, kmp_int32 st, kmp_int32 chunk)
void	__kmpc_dispatch_init_4u (ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_uint32 lb, kmp_uint32 ub, kmp_int32 st, kmp_int32 chunk)
	See __kmpc_dispatch_init_4.
void	__kmpc_dispatch_init_8 (ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_int64 lb, kmp_int64 ub, kmp_int64 st, kmp_int64 chunk)
	See __kmpc_dispatch_init_4.
void	__kmpc_dispatch_init_8u (ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_uint64 lb, kmp_uint64 ub, kmp_int64 st, kmp_int64 chunk)
	See __kmpc_dispatch_init_4.
void	__kmpc_dist_dispatch_init_4 (ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_int32 *p_last, kmp_int32 lb, kmp_int32 ub, kmp_int32 st, kmp_int32 chunk)
	See __kmpc_dispatch_init_4.
void	__kmpc_dist_dispatch_init_4u (ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_int32 *p_last, kmp_uint32 lb, kmp_uint32 ub, kmp_int32 st, kmp_int32 chunk)
void	__kmpc_dist_dispatch_init_8 (ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_int32 *p_last, kmp_int64 lb, kmp_int64 ub, kmp_int64 st, kmp_int64 chunk)
void	__kmpc_dist_dispatch_init_8u (ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_int32 *p_last, kmp_uint64 lb, kmp_uint64 ub, kmp_int64 st, kmp_int64 chunk)
int	__kmpc_dispatch_next_4 (ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, kmp_int32 *p_lb, kmp_int32 *p_ub, kmp_int32 *p_st)
int	__kmpc_dispatch_next_4u (ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, kmp_uint32 *p_lb, kmp_uint32 *p_ub, kmp_int32 *p_st)
	See __kmpc_dispatch_next_4.
int	__kmpc_dispatch_next_8 (ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, kmp_int64 *p_lb, kmp_int64 *p_ub, kmp_int64 *p_st)
	See __kmpc_dispatch_next_4.
int	__kmpc_dispatch_next_8u (ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, kmp_uint64 *p_lb, kmp_uint64 *p_ub, kmp_int64 *p_st)
	See __kmpc_dispatch_next_4.
void	__kmpc_dispatch_fini_4 (ident_t *loc, kmp_int32 gtid)
void	__kmpc_dispatch_fini_8 (ident_t *loc, kmp_int32 gtid)
	See __kmpc_dispatch_fini_4.
void	__kmpc_dispatch_fini_4u (ident_t *loc, kmp_int32 gtid)
	See __kmpc_dispatch_fini_4.
void	__kmpc_dispatch_fini_8u (ident_t *loc, kmp_int32 gtid)
	See __kmpc_dispatch_fini_4.
void	__kmpc_dispatch_deinit (ident_t *loc, kmp_int32 gtid)
	See __kmpc_dispatch_deinit.

Macro Definition Documentation

GUIDED_ANALYTICAL_WORKAROUND

#define GUIDED_ANALYTICAL_WORKAROUND   (x)

Referenced by __kmp_dispatch_init_algorithm().

KMP_STATS_LOOP_END

#define KMP_STATS_LOOP_END   /* Nothing */

Definition at line 2176 of file kmp_dispatch.cpp.

Referenced by __kmp_dispatch_next().

OMPT_LOOP_DISPATCH

#define OMPT_LOOP_DISPATCH	(		lb,
			ub,
			st,
			status )

Definition at line 2144 of file kmp_dispatch.cpp.

Referenced by __kmp_dispatch_next().

OMPT_LOOP_END

#define OMPT_LOOP_END

Definition at line 2143 of file kmp_dispatch.cpp.

Referenced by __kmp_dispatch_next().

Function Documentation

__kmp_dispatch_deo_error()

void __kmp_dispatch_deo_error	(	int *	gtid_ref,
		int *	cid_ref,
		ident_t *	loc_ref )

Definition at line 42 of file kmp_dispatch.cpp.

References __kmp_env_consistency_check, __kmp_push_sync(), __kmp_threads, ct_none, ct_ordered_in_pdo, and KMP_DEBUG_ASSERT.

Referenced by __kmp_dispatch_init(), and __kmpc_sections_init().

__kmp_dispatch_dxo_error()

void __kmp_dispatch_dxo_error	(	int *	gtid_ref,
		int *	cid_ref,
		ident_t *	loc_ref )

Definition at line 60 of file kmp_dispatch.cpp.

References __kmp_env_consistency_check, __kmp_pop_sync(), __kmp_threads, ct_none, and ct_ordered_in_pdo.

Referenced by __kmp_dispatch_init(), and __kmpc_sections_init().

__kmp_dispatch_finish()

template<typename UT>

void __kmp_dispatch_finish ( int gtid, ident_t * loc )

static

Definition at line 1181 of file kmp_dispatch.cpp.

References __kmp_assert_valid_gtid(), __kmp_ge(), __kmp_str_format(), __kmp_str_free(), __kmp_threads, __kmp_wait(), KD_TRACE, KMP_DEBUG_ASSERT, KMP_MB, loc, dispatch_private_info_template< T >::ordered_bumped, dispatch_private_info_template< T >::private_info_tmpl::p, ST, test_then_inc(), dispatch_private_info_template< T >::u, and USE_ITT_BUILD_ARG.

Referenced by __kmpc_dispatch_fini_4(), __kmpc_dispatch_fini_4u(), __kmpc_dispatch_fini_8(), and __kmpc_dispatch_fini_8u().

__kmp_dispatch_init()

template<typename T>

void __kmp_dispatch_init ( ident_t * loc, int gtid, enum sched_type schedule, T lb, T ub, typename traits_t< T >::signed_t st, typename traits_t< T >::signed_t chunk, int push_ws )

static

Definition at line 962 of file kmp_dispatch.cpp.

References __kmp_assert_valid_gtid(), __kmp_dispatch_deo(), __kmp_dispatch_deo_error(), __kmp_dispatch_dxo(), __kmp_dispatch_dxo_error(), __kmp_dispatch_init_algorithm(), __kmp_dispatch_num_buffers, __kmp_eq(), __kmp_hier_scheds, __kmp_init_parallel, __kmp_parallel_initialize(), __kmp_resume_if_soft_paused(), __kmp_static, __kmp_str_format(), __kmp_str_free(), __kmp_threads, __kmp_wait(), __ompt_get_task_info_object(), __ompt_get_teaminfo(), CCAST, FALSE, dispatch_private_info_template< T >::flags, KD_TRACE, KMP_BUILD_ASSERT, KMP_COUNT_BLOCK, KMP_DEBUG_ASSERT, KMP_MASTER_GTID, kmp_nm_lower, kmp_nm_upper, kmp_ord_lower, KMP_PUSH_PARTITIONED_TIMER, kmp_sch_dynamic_chunked, kmp_sch_guided_analytical_chunked, kmp_sch_guided_iterative_chunked, kmp_sch_guided_simd, kmp_sch_lower, kmp_sch_runtime, kmp_sch_static_balanced, kmp_sch_static_chunked, kmp_sch_static_greedy, kmp_uint32, loc, ompt_callbacks, ompt_enabled, kmp_sched_flags::ordered, dispatch_private_info_template< T >::private_info_tmpl::p, ompt_team_info_t::parallel_data, dispatch_private_info_template< T >::schedule, SCHEDULE_WITHOUT_MODIFIERS, ST, kmp_team::t, ompt_task_info_t::task_data, TCR_4, dispatch_private_info_template< T >::u, kmp_sched_flags::use_hier, and USE_ITT_BUILD_ARG.

Referenced by __kmpc_dispatch_init_4(), __kmpc_dispatch_init_4u(), __kmpc_dispatch_init_8(), __kmpc_dispatch_init_8u(), __kmpc_dist_dispatch_init_4(), __kmpc_dist_dispatch_init_4u(), __kmpc_dist_dispatch_init_8(), and __kmpc_dist_dispatch_init_8u().

__kmp_dispatch_init_algorithm()

template<typename T>

void __kmp_dispatch_init_algorithm	(	ident_t *	loc,
		int	gtid,
		dispatch_private_info_template< T > *	pr,
		enum sched_type	schedule,
		T	lb,
		T	ub,
		typename traits_t< T >::signed_t	st,
		typename traits_t< T >::signed_t	chunk,
		T	nproc,
		T	tid )

Definition at line 184 of file kmp_dispatch.cpp.

References __kmp_allocate, __kmp_auto, __kmp_dispatch_guided_remaining(), __kmp_env_consistency_check, __kmp_error_construct(), __kmp_fatal(), __kmp_get_monotonicity(), __kmp_guided, __kmp_init_lock(), __kmp_is_hybrid_cpu(), __kmp_msg_null, __kmp_pow(), __kmp_static, __kmp_str_format(), __kmp_str_free(), __kmp_threads, a, b, ct_pdo, ct_pdo_ordered, d, double, FALSE, dispatch_private_info_template< T >::flags, GUIDED_ANALYTICAL_WORKAROUND, guided_flt_param, guided_int_param, i, init(), int, KD_TRACE, KMP_ASSERT, KMP_ASSERT2, KMP_ATOMIC_ST_REL, KMP_COUNT_BLOCK, KMP_COUNT_VALUE, KMP_DEBUG_ASSERT, KMP_DEFAULT_CHUNK, KMP_HNT, KMP_HW_CORE_TYPE_UNKNOWN, KMP_MASTER_GTID, KMP_MSG, kmp_nm_lower, kmp_nm_upper, kmp_ord_lower, kmp_sch_auto, kmp_sch_dynamic_chunked, kmp_sch_guided_analytical_chunked, kmp_sch_guided_chunked, kmp_sch_guided_iterative_chunked, kmp_sch_guided_simd, kmp_sch_lower, kmp_sch_runtime, kmp_sch_runtime_simd, kmp_sch_static, kmp_sch_static_balanced, kmp_sch_static_balanced_chunked, kmp_sch_static_chunked, kmp_sch_static_greedy, kmp_sch_static_steal, kmp_sch_trapezoidal, kmp_sch_upper, kmp_uint32, KMP_WARNING, loc, kmp_sched_flags::nomerge, kmp_sched_flags::ordered, dispatch_private_info_template< T >::ordered_bumped, dispatch_private_info_template< T >::private_info_tmpl::p, p, dispatch_private_info_template< T >::schedule, SCHEDULE_MONOTONIC, SCHEDULE_NONMONOTONIC, SCHEDULE_WITHOUT_MODIFIERS, ST, dispatch_private_info_template< T >::steal_flag, kmp_team::t, target(), TRUE, type, dispatch_private_info_template< T >::type_size, dispatch_private_info_template< T >::u, kmp_sched_flags::use_hier, and kmp_sched_flags::use_hybrid.

Referenced by __kmp_dispatch_init(), __kmp_dispatch_init_hierarchy(), and kmp_hier_t< T >::next().

__kmp_dispatch_next()

template<typename T>

int __kmp_dispatch_next ( ident_t * loc, int gtid, kmp_int32 * p_last, T * p_lb, T * p_ub, typename traits_t< T >::signed_t * p_st )

static

Definition at line 2180 of file kmp_dispatch.cpp.

References __kmp_assert_valid_gtid(), __kmp_destroy_lock(), __kmp_dispatch_next_algorithm(), __kmp_dispatch_num_buffers, __kmp_env_consistency_check, __kmp_free, __kmp_pop_workshare(), __kmp_str_format(), __kmp_str_free(), __kmp_threads, buf, dispatch_shared_info_template< T >::buffer_index, ct_none, FALSE, dispatch_private_info_template< T >::flags, i, init(), dispatch_shared_infoXX_template< T >::iteration, KD_TRACE, KMP_ASSERT, KMP_ATOMIC_ST_RLX, KMP_DEBUG_ASSERT, KMP_MB, kmp_sch_static_steal, KMP_STATS_LOOP_END, KMP_TIME_PARTITIONED_BLOCK, lck, loc, kmp_sched_flags::nomerge, dispatch_shared_infoXX_template< T >::num_done, OMPT_LOOP_DISPATCH, OMPT_LOOP_END, kmp_sched_flags::ordered, dispatch_shared_infoXX_template< T >::ordered_iteration, dispatch_private_info_template< T >::private_info_tmpl::p, dispatch_private_info_template< T >::pushed_ws, dispatch_shared_info_template< T >::shared_info_tmpl::s, dispatch_private_info_template< T >::schedule, ST, status, kmp_team::t, test_then_inc(), TRUE, dispatch_private_info_template< T >::u, dispatch_shared_info_template< T >::u, and kmp_sched_flags::use_hier.

Referenced by __kmpc_dispatch_next_4(), __kmpc_dispatch_next_4u(), __kmpc_dispatch_next_8(), and __kmpc_dispatch_next_8u().

__kmp_dispatch_next_algorithm()

template<typename T>

int __kmp_dispatch_next_algorithm	(	int	gtid,
		dispatch_private_info_template< T > *	pr,
		dispatch_shared_info_template< T > volatile *	sh,
		kmp_int32 *	p_last,
		T *	p_lb,
		T *	p_ub,
		typename traits_t< T >::signed_t *	p_st,
		T	nproc,
		T	tid )

Definition at line 1323 of file kmp_dispatch.cpp.

References __kmp_acquire_lock(), __kmp_dispatch_guided_remaining(), __kmp_dispatch_num_buffers, __kmp_fatal(), __kmp_initialize_self_buffer(), __kmp_msg_null, __kmp_release_lock(), __kmp_str_format(), __kmp_str_free(), __kmp_threads, __kmp_type_convert(), b, chunk_size, compare_and_swap(), count, dispatch_private_info_template< T >::flags, init(), dispatch_shared_infoXX_template< T >::iteration, KD_TRACE, KMP_ASSERT, KMP_ATOMIC_LD_ACQ, KMP_ATOMIC_LD_RLX, KMP_ATOMIC_ST_REL, KMP_COMPARE_AND_STORE_REL64, KMP_COUNT_DEVELOPER_VALUE, KMP_CPU_PAUSE, KMP_DEBUG_ASSERT, KMP_HNT, KMP_MSG, kmp_sch_dynamic_chunked, kmp_sch_guided_analytical_chunked, kmp_sch_guided_iterative_chunked, kmp_sch_guided_simd, kmp_sch_static_balanced, kmp_sch_static_chunked, kmp_sch_static_greedy, kmp_sch_static_steal, kmp_sch_trapezoidal, kmp_uint32, KMP_XCHG_FIXED64, lck, kmp_sched_flags::ordered, dispatch_private_info_template< T >::private_info_tmpl::p, p, RCAST, dispatch_shared_info_template< T >::shared_info_tmpl::s, dispatch_private_info_template< T >::schedule, ST, status, dispatch_private_info_template< T >::steal_flag, kmp_team::t, test_then_add(), test_then_inc(), test_then_inc_acq(), dispatch_private_info_template< T >::u, dispatch_shared_info_template< T >::u, and VOLATILE_CAST.

Referenced by __kmp_dispatch_next(), and kmp_hier_t< T >::next().

__kmp_dist_get_bounds()

template<typename T>

void __kmp_dist_get_bounds ( ident_t * loc, kmp_int32 gtid, kmp_int32 * plastiter, T * plower, T * pupper, typename traits_t< T >::signed_t incr )

static

Definition at line 2665 of file kmp_dispatch.cpp.

References __kmp_assert_valid_gtid(), __kmp_env_consistency_check, __kmp_error_construct(), __kmp_static, __kmp_str_format(), __kmp_str_free(), __kmp_threads, ct_pdo, KD_TRACE, KE_TRACE, KMP_DEBUG_ASSERT, kmp_sch_static_balanced, kmp_sch_static_greedy, kmp_uint32, loc, ST, and kmp_team::t.

Referenced by __kmpc_dist_dispatch_init_4(), __kmpc_dist_dispatch_init_4u(), __kmpc_dist_dispatch_init_8(), and __kmpc_dist_dispatch_init_8u().

__kmp_eq_4()

kmp_uint32 __kmp_eq_4	(	kmp_uint32	value,
		kmp_uint32	checker )

Definition at line 3023 of file kmp_dispatch.cpp.

References checker, kmp_uint32, and value.

Referenced by __kmpc_doacross_init().

__kmp_ge_4()

kmp_uint32 __kmp_ge_4	(	kmp_uint32	value,
		kmp_uint32	checker )

Definition at line 3035 of file kmp_dispatch.cpp.

References checker, kmp_uint32, and value.

__kmp_get_monotonicity()

int __kmp_get_monotonicity ( ident_t * loc, enum sched_type schedule, bool use_hier = false )

inlinestatic

Definition at line 72 of file kmp_dispatch.cpp.

References __kmp_force_monotonic, loc, SCHEDULE_HAS_MONOTONIC, SCHEDULE_HAS_NONMONOTONIC, SCHEDULE_MONOTONIC, and SCHEDULE_NONMONOTONIC.

Referenced by __kmp_dispatch_init_algorithm().

__kmp_initialize_self_buffer()

template<typename T>

void __kmp_initialize_self_buffer ( kmp_team_t * team, T id, dispatch_private_info_template< T > * pr, typename traits_t< T >::unsigned_t nchunks, T nproc, typename traits_t< T >::unsigned_t & init, T & small_chunk, T & extras, T & p_extra )

inline

Definition at line 105 of file kmp_dispatch.cpp.

References __kmp_gtid_from_tid(), __kmp_threads, dispatch_private_info_template< T >::flags, init(), dispatch_private_info_template< T >::private_info_tmpl::p, type, dispatch_private_info_template< T >::u, and kmp_sched_flags::use_hybrid.

Referenced by __kmp_dispatch_next_algorithm().

__kmp_le_4()

kmp_uint32 __kmp_le_4	(	kmp_uint32	value,
		kmp_uint32	checker )

Definition at line 3039 of file kmp_dispatch.cpp.

References checker, kmp_uint32, and value.

__kmp_lt_4()

kmp_uint32 __kmp_lt_4	(	kmp_uint32	value,
		kmp_uint32	checker )

Definition at line 3031 of file kmp_dispatch.cpp.

References checker, kmp_uint32, and value.

__kmp_neq_4()

kmp_uint32 __kmp_neq_4	(	kmp_uint32	value,
		kmp_uint32	checker )

Definition at line 3027 of file kmp_dispatch.cpp.

References checker, kmp_uint32, and value.

__kmp_wait_4()

kmp_uint32 __kmp_wait_4	(	volatile kmp_uint32 *	spinner,
		kmp_uint32	checker,
		kmp_uint32(*	pred )(kmp_uint32, kmp_uint32),
		void *	obj )

Definition at line 3044 of file kmp_dispatch.cpp.

References CCAST, checker, f(), KMP_FSYNC_SPIN_ACQUIRED, KMP_FSYNC_SPIN_INIT, KMP_FSYNC_SPIN_PREPARE, KMP_INIT_BACKOFF, KMP_INIT_YIELD, kmp_uint32, KMP_YIELD_OVERSUB_ELSE_SPIN, r, and TCR_4.

Referenced by __kmpc_doacross_init().

__kmp_wait_4_ptr()

void __kmp_wait_4_ptr	(	void *	spinner,
		kmp_uint32	checker,
		kmp_uint32(*	pred )(void *, kmp_uint32),
		void *	obj )

Definition at line 3072 of file kmp_dispatch.cpp.

References checker, f(), KMP_FSYNC_SPIN_ACQUIRED, KMP_FSYNC_SPIN_INIT, KMP_FSYNC_SPIN_PREPARE, KMP_INIT_BACKOFF, KMP_INIT_YIELD, kmp_uint32, and KMP_YIELD_OVERSUB_ELSE_SPIN.