565#ifndef KMP_GOMP_COMPAT
605#define KMP_ATOMIC_VOLATILE volatile
607#if (KMP_ARCH_X86) && KMP_HAVE_QUAD
609static inline Quad_a4_t operator+(Quad_a4_t &lhs, Quad_a4_t &rhs) {
610 return lhs.q + rhs.q;
612static inline Quad_a4_t
operator-(Quad_a4_t &lhs, Quad_a4_t &rhs) {
613 return lhs.q - rhs.q;
615static inline Quad_a4_t operator*(Quad_a4_t &lhs, Quad_a4_t &rhs) {
616 return lhs.q * rhs.q;
618static inline Quad_a4_t operator/(Quad_a4_t &lhs, Quad_a4_t &rhs) {
619 return lhs.q / rhs.q;
621static inline bool operator<(Quad_a4_t &lhs, Quad_a4_t &rhs) {
622 return lhs.q < rhs.q;
624static inline bool operator>(Quad_a4_t &lhs, Quad_a4_t &rhs) {
625 return lhs.q > rhs.q;
628static inline Quad_a16_t operator+(Quad_a16_t &lhs, Quad_a16_t &rhs) {
629 return lhs.q + rhs.q;
631static inline Quad_a16_t
operator-(Quad_a16_t &lhs, Quad_a16_t &rhs) {
632 return lhs.q - rhs.q;
634static inline Quad_a16_t operator*(Quad_a16_t &lhs, Quad_a16_t &rhs) {
635 return lhs.q * rhs.q;
637static inline Quad_a16_t operator/(Quad_a16_t &lhs, Quad_a16_t &rhs) {
638 return lhs.q / rhs.q;
640static inline bool operator<(Quad_a16_t &lhs, Quad_a16_t &rhs) {
641 return lhs.q < rhs.q;
643static inline bool operator>(Quad_a16_t &lhs, Quad_a16_t &rhs) {
644 return lhs.q > rhs.q;
647static inline kmp_cmplx128_a4_t operator+(kmp_cmplx128_a4_t &lhs,
648 kmp_cmplx128_a4_t &rhs) {
649 return lhs.q + rhs.q;
651static inline kmp_cmplx128_a4_t
operator-(kmp_cmplx128_a4_t &lhs,
652 kmp_cmplx128_a4_t &rhs) {
653 return lhs.q - rhs.q;
655static inline kmp_cmplx128_a4_t operator*(kmp_cmplx128_a4_t &lhs,
656 kmp_cmplx128_a4_t &rhs) {
657 return lhs.q * rhs.q;
659static inline kmp_cmplx128_a4_t operator/(kmp_cmplx128_a4_t &lhs,
660 kmp_cmplx128_a4_t &rhs) {
661 return lhs.q / rhs.q;
664static inline kmp_cmplx128_a16_t operator+(kmp_cmplx128_a16_t &lhs,
665 kmp_cmplx128_a16_t &rhs) {
666 return lhs.q + rhs.q;
668static inline kmp_cmplx128_a16_t
operator-(kmp_cmplx128_a16_t &lhs,
669 kmp_cmplx128_a16_t &rhs) {
670 return lhs.q - rhs.q;
672static inline kmp_cmplx128_a16_t operator*(kmp_cmplx128_a16_t &lhs,
673 kmp_cmplx128_a16_t &rhs) {
674 return lhs.q * rhs.q;
676static inline kmp_cmplx128_a16_t operator/(kmp_cmplx128_a16_t &lhs,
677 kmp_cmplx128_a16_t &rhs) {
678 return lhs.q / rhs.q;
688#define KMP_CHECK_GTID \
689 if (gtid == KMP_GTID_UNKNOWN) { \
690 gtid = __kmp_entry_gtid(); \
698#define ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
699 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
700 TYPE *lhs, TYPE rhs) { \
701 KMP_DEBUG_ASSERT(__kmp_init_serial); \
702 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
706#define ATOMIC_LOCK0 __kmp_atomic_lock
707#define ATOMIC_LOCK1i __kmp_atomic_lock_1i
708#define ATOMIC_LOCK2i __kmp_atomic_lock_2i
709#define ATOMIC_LOCK4i __kmp_atomic_lock_4i
710#define ATOMIC_LOCK4r __kmp_atomic_lock_4r
711#define ATOMIC_LOCK8i __kmp_atomic_lock_8i
712#define ATOMIC_LOCK8r __kmp_atomic_lock_8r
713#define ATOMIC_LOCK8c __kmp_atomic_lock_8c
714#define ATOMIC_LOCK10r __kmp_atomic_lock_10r
715#define ATOMIC_LOCK16r __kmp_atomic_lock_16r
716#define ATOMIC_LOCK16c __kmp_atomic_lock_16c
717#define ATOMIC_LOCK20c __kmp_atomic_lock_20c
718#define ATOMIC_LOCK32c __kmp_atomic_lock_32c
726#define OP_CRITICAL(OP, LCK_ID) \
727 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
731 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
733#define OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \
734 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
735 (*lhs) = (TYPE)((*lhs)OP rhs); \
736 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
760#ifdef KMP_GOMP_COMPAT
761#define OP_GOMP_CRITICAL(OP, FLAG) \
762 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
764 OP_CRITICAL(OP, 0); \
768#define OP_UPDATE_GOMP_CRITICAL(TYPE, OP, FLAG) \
769 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
771 OP_UPDATE_CRITICAL(TYPE, OP, 0); \
775#define OP_GOMP_CRITICAL(OP, FLAG)
776#define OP_UPDATE_GOMP_CRITICAL(TYPE, OP, FLAG)
780#define KMP_DO_PAUSE _mm_delay_32(1)
790#define OP_CMPXCHG(TYPE, BITS, OP) \
792 TYPE old_value, new_value; \
793 old_value = *(TYPE volatile *)lhs; \
794 new_value = (TYPE)(old_value OP rhs); \
795 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
796 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
797 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
800 old_value = *(TYPE volatile *)lhs; \
801 new_value = (TYPE)(old_value OP rhs); \
812#define OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
816 kmp_int##BITS *vvv; \
818 struct _sss old_value, new_value; \
819 old_value.vvv = (kmp_int##BITS *)&old_value.cmp; \
820 new_value.vvv = (kmp_int##BITS *)&new_value.cmp; \
821 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
822 new_value.cmp = (TYPE)(old_value.cmp OP rhs); \
823 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
824 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \
825 *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv)) { \
828 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
829 new_value.cmp = (TYPE)(old_value.cmp OP rhs); \
835#if KMP_OS_WINDOWS && (KMP_ARCH_AARCH64 || KMP_ARCH_ARM)
839#define OP_CMPXCHG(TYPE, BITS, OP) \
843 kmp_int##BITS *vvv; \
845 struct _sss old_value, new_value; \
846 old_value.vvv = (kmp_int##BITS *)&old_value.cmp; \
847 new_value.vvv = (kmp_int##BITS *)&new_value.cmp; \
848 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
849 new_value.cmp = old_value.cmp OP rhs; \
850 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
851 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \
852 *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv)) { \
855 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
856 new_value.cmp = old_value.cmp OP rhs; \
860#undef OP_UPDATE_CRITICAL
861#define OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \
862 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
863 (*lhs) = (*lhs)OP rhs; \
864 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
868#if KMP_ARCH_X86 || KMP_ARCH_X86_64
872#define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
874 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
875 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
877 KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
880#define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
882 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
883 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
884 OP_CMPXCHG(TYPE, BITS, OP) \
889#define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \
891 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
892 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
893 OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
901#define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
903 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
904 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
905 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
907 KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
910 OP_UPDATE_CRITICAL(TYPE, OP, \
915#define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
917 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
918 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
919 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
920 OP_CMPXCHG(TYPE, BITS, OP) \
923 OP_UPDATE_CRITICAL(TYPE, OP, \
930#define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \
932 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
933 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
934 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
935 OP_CMPXCHG(TYPE, BITS, OP) \
938 OP_UPDATE_CRITICAL(TYPE, OP, \
1079#define ATOMIC_CRIT_L(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1080 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1081 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
1082 OP_CRITICAL(= *lhs OP, LCK_ID) \
1085#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1089#define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \
1090 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1091 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
1092 OP_CMPXCHG(TYPE, BITS, OP) \
1098#define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \
1099 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1100 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
1101 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1102 OP_CMPXCHG(TYPE, BITS, OP) \
1105 OP_CRITICAL(= *lhs OP, LCK_ID) \
1136#define MIN_MAX_CRITSECT(OP, LCK_ID) \
1137 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1139 if (*lhs OP rhs) { \
1142 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1145#ifdef KMP_GOMP_COMPAT
1146#define GOMP_MIN_MAX_CRITSECT(OP, FLAG) \
1147 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1149 MIN_MAX_CRITSECT(OP, 0); \
1153#define GOMP_MIN_MAX_CRITSECT(OP, FLAG)
1157#define MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
1159 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1162 old_value = temp_val; \
1163 while (old_value OP rhs && \
1164 !KMP_COMPARE_AND_STORE_ACQ##BITS( \
1165 (kmp_int##BITS *)lhs, \
1166 *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
1167 *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \
1169 old_value = temp_val; \
1175#define MIN_MAX_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1176 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1177 if (*lhs OP rhs) { \
1178 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
1179 MIN_MAX_CRITSECT(OP, LCK_ID) \
1183#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1187#define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1189 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1190 if (*lhs OP rhs) { \
1191 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
1192 MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
1199#define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1201 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1202 if (*lhs OP rhs) { \
1203 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
1204 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1205 MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
1208 MIN_MAX_CRITSECT(OP, LCK_ID) \
1238#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1259#define ATOMIC_CRIT_EQV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1260 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1261 OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) \
1262 OP_CRITICAL(^= (TYPE) ~, LCK_ID) \
1266#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1269#define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1271 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1272 OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) \
1273 OP_CMPXCHG(TYPE, BITS, OP) \
1279#define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1281 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1282 OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) \
1283 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1284 OP_CMPXCHG(TYPE, BITS, OP) \
1287 OP_CRITICAL(^= (TYPE) ~, LCK_ID) \
1315#define ATOMIC_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1316 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1317 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1318 OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \
1322#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1378#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1411#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1419#define OP_CRITICAL_REV(TYPE, OP, LCK_ID) \
1420 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1422 (*lhs) = (TYPE)((rhs)OP(*lhs)); \
1424 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1426#ifdef KMP_GOMP_COMPAT
1427#define OP_GOMP_CRITICAL_REV(TYPE, OP, FLAG) \
1428 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1430 OP_CRITICAL_REV(TYPE, OP, 0); \
1435#define OP_GOMP_CRITICAL_REV(TYPE, OP, FLAG)
1443#define ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
1444 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_rev(ident_t *id_ref, int gtid, \
1445 TYPE *lhs, TYPE rhs) { \
1446 KMP_DEBUG_ASSERT(__kmp_init_serial); \
1447 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_rev: T#%d\n", gtid));
1456#define OP_CMPXCHG_REV(TYPE, BITS, OP) \
1458 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1459 TYPE old_value, new_value; \
1461 old_value = temp_val; \
1462 new_value = (TYPE)(rhs OP old_value); \
1463 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
1464 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
1465 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
1469 old_value = temp_val; \
1470 new_value = (TYPE)(rhs OP old_value); \
1475#define ATOMIC_CMPXCHG_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, GOMP_FLAG) \
1476 ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \
1477 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
1478 OP_CMPXCHG_REV(TYPE, BITS, OP) \
1510ATOMIC_CMPXCHG_REV(fixed2u,
div, kmp_uint16, 16, /, 2
i,
1516ATOMIC_CMPXCHG_REV(fixed2u,
shr, kmp_uint16, 16, >>, 2
i,
1564#define ATOMIC_CRITICAL_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1565 ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \
1566 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
1567 OP_CRITICAL_REV(TYPE, OP, LCK_ID) \
1572ATOMIC_CRITICAL_REV(float10,
sub,
long double, -, 10
r,
1574ATOMIC_CRITICAL_REV(float10,
div,
long double, /, 10
r,
1583ATOMIC_CRITICAL_REV(float16, sub_a16, Quad_a16_t, -, 16
r,
1585ATOMIC_CRITICAL_REV(float16, div_a16, Quad_a16_t, /, 16
r,
1609ATOMIC_CRITICAL_REV(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1611ATOMIC_CRITICAL_REV(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1630#define ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1631 void __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \
1632 ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs) { \
1633 KMP_DEBUG_ASSERT(__kmp_init_serial); \
1635 ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \
1639#define ATOMIC_CRITICAL_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, LCK_ID, \
1641 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1642 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1643 OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \
1647#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1650#define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1651 LCK_ID, MASK, GOMP_FLAG) \
1652 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1653 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1654 OP_CMPXCHG(TYPE, BITS, OP) \
1660#define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1661 LCK_ID, MASK, GOMP_FLAG) \
1662 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1663 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1664 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1665 OP_CMPXCHG(TYPE, BITS, OP) \
1668 OP_UPDATE_CRITICAL(TYPE, OP, \
1675#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1677#define ATOMIC_CMPXCHG_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
1678 RTYPE, LCK_ID, MASK, GOMP_FLAG) \
1679 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1680 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
1681 OP_CMPXCHG_REV(TYPE, BITS, OP) \
1683#define ATOMIC_CRITICAL_REV_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
1684 LCK_ID, GOMP_FLAG) \
1685 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1686 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
1687 OP_CRITICAL_REV(TYPE, OP, LCK_ID) \
1806#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1817ATOMIC_CMPXCHG_REV_MIX(fixed1,
char, sub_rev, 8, -, fp, _Quad, 1
i, 0,
1819ATOMIC_CMPXCHG_REV_MIX(fixed1u,
uchar, sub_rev, 8, -, fp, _Quad, 1
i, 0,
1821ATOMIC_CMPXCHG_REV_MIX(fixed1,
char, div_rev, 8, /, fp, _Quad, 1
i, 0,
1823ATOMIC_CMPXCHG_REV_MIX(fixed1u,
uchar, div_rev, 8, /, fp, _Quad, 1
i, 0,
1826ATOMIC_CMPXCHG_REV_MIX(fixed2,
short, sub_rev, 16, -, fp, _Quad, 2
i, 1,
1828ATOMIC_CMPXCHG_REV_MIX(fixed2u,
ushort, sub_rev, 16, -, fp, _Quad, 2
i, 1,
1830ATOMIC_CMPXCHG_REV_MIX(fixed2,
short, div_rev, 16, /, fp, _Quad, 2
i, 1,
1832ATOMIC_CMPXCHG_REV_MIX(fixed2u,
ushort, div_rev, 16, /, fp, _Quad, 2
i, 1,
1835ATOMIC_CMPXCHG_REV_MIX(fixed4,
kmp_int32, sub_rev, 32, -, fp, _Quad, 4
i, 3,
1837ATOMIC_CMPXCHG_REV_MIX(fixed4u,
kmp_uint32, sub_rev, 32, -, fp, _Quad, 4
i, 3,
1839ATOMIC_CMPXCHG_REV_MIX(fixed4,
kmp_int32, div_rev, 32, /, fp, _Quad, 4
i, 3,
1841ATOMIC_CMPXCHG_REV_MIX(fixed4u,
kmp_uint32, div_rev, 32, /, fp, _Quad, 4
i, 3,
1844ATOMIC_CMPXCHG_REV_MIX(fixed8,
kmp_int64, sub_rev, 64, -, fp, _Quad, 8
i, 7,
1846ATOMIC_CMPXCHG_REV_MIX(fixed8u,
kmp_uint64, sub_rev, 64, -, fp, _Quad, 8
i, 7,
1848ATOMIC_CMPXCHG_REV_MIX(fixed8,
kmp_int64, div_rev, 64, /, fp, _Quad, 8
i, 7,
1850ATOMIC_CMPXCHG_REV_MIX(fixed8u,
kmp_uint64, div_rev, 64, /, fp, _Quad, 8
i, 7,
1853ATOMIC_CMPXCHG_REV_MIX(float4,
kmp_real32, sub_rev, 32, -, fp, _Quad, 4
r, 3,
1855ATOMIC_CMPXCHG_REV_MIX(float4,
kmp_real32, div_rev, 32, /, fp, _Quad, 4
r, 3,
1863ATOMIC_CRITICAL_REV_FP(float10,
long double, sub_rev, -, fp, _Quad, 10
r,
1865ATOMIC_CRITICAL_REV_FP(float10,
long double, div_rev, /, fp, _Quad, 10
r,
1871#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1876#define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1877 LCK_ID, MASK, GOMP_FLAG) \
1878 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1879 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1880 OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
1884#define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1885 LCK_ID, MASK, GOMP_FLAG) \
1886 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1887 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1888 OP_CMPXCHG(TYPE, BITS, OP) \
1894#define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1895 LCK_ID, MASK, GOMP_FLAG) \
1896 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1897 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1898 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1899 OP_CMPXCHG(TYPE, BITS, OP) \
1902 OP_UPDATE_CRITICAL(TYPE, OP, \
1928#define ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
1929 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
1931 KMP_DEBUG_ASSERT(__kmp_init_serial); \
1932 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
1943#define OP_CMPXCHG_READ(TYPE, BITS, OP) \
1945 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1948 kmp_int##BITS i_val; \
1950 union f_i_union old_value; \
1952 old_value.f_val = temp_val; \
1953 old_value.i_val = KMP_COMPARE_AND_STORE_RET##BITS( \
1954 (kmp_int##BITS *)loc, \
1955 *VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val, \
1956 *VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val); \
1957 new_value = old_value.f_val; \
1967#define OP_CRITICAL_READ(OP, LCK_ID) \
1968 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1970 new_value = (*loc); \
1972 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1975#ifdef KMP_GOMP_COMPAT
1976#define OP_GOMP_CRITICAL_READ(OP, FLAG) \
1977 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1979 OP_CRITICAL_READ(OP, 0); \
1983#define OP_GOMP_CRITICAL_READ(OP, FLAG)
1987#define ATOMIC_FIXED_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
1988 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \
1990 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \
1991 new_value = KMP_TEST_THEN_ADD##BITS(loc, OP 0); \
1995#define ATOMIC_CMPXCHG_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
1996 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \
1998 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \
1999 OP_CMPXCHG_READ(TYPE, BITS, OP) \
2007#define ATOMIC_CRITICAL_READ(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2008 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \
2010 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \
2011 OP_CRITICAL_READ(OP, LCK_ID) \
2021#define OP_CRITICAL_READ_WRK(OP, LCK_ID) \
2022 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2026 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
2028#ifdef KMP_GOMP_COMPAT
2029#define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG) \
2030 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2032 OP_CRITICAL_READ_WRK(OP, 0); \
2035#define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG)
2038#define ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \
2039 void __kmpc_atomic_##TYPE_ID##_##OP_ID(TYPE *out, ident_t *id_ref, int gtid, \
2041 KMP_DEBUG_ASSERT(__kmp_init_serial); \
2042 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
2045#define ATOMIC_CRITICAL_READ_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2046 ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \
2047 OP_GOMP_CRITICAL_READ_WRK(OP## =, GOMP_FLAG) \
2048 OP_CRITICAL_READ_WRK(OP, LCK_ID) \
2102#define ATOMIC_XCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2103 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2104 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2105 KMP_XCHG_FIXED##BITS(lhs, rhs); \
2108#define ATOMIC_XCHG_FLOAT_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2109 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2110 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2111 KMP_XCHG_REAL##BITS(lhs, rhs); \
2121#define OP_CMPXCHG_WR(TYPE, BITS, OP) \
2123 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2124 TYPE old_value, new_value; \
2126 old_value = temp_val; \
2128 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
2129 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2130 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
2132 old_value = temp_val; \
2138#define ATOMIC_CMPXCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2139 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2140 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2141 OP_CMPXCHG_WR(TYPE, BITS, OP) \
2150#define ATOMIC_CRITICAL_WR(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2151 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2152 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2153 OP_CRITICAL(OP, LCK_ID) \
2211#define ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
2212 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
2213 TYPE *lhs, TYPE rhs, int flag) { \
2214 KMP_DEBUG_ASSERT(__kmp_init_serial); \
2215 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
2223#define OP_CRITICAL_CPT(OP, LCK_ID) \
2224 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2228 new_value = (*lhs); \
2230 new_value = (*lhs); \
2234 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2237#define OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) \
2238 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2241 (*lhs) = (TYPE)((*lhs)OP rhs); \
2242 new_value = (*lhs); \
2244 new_value = (*lhs); \
2245 (*lhs) = (TYPE)((*lhs)OP rhs); \
2248 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2252#ifdef KMP_GOMP_COMPAT
2253#define OP_GOMP_CRITICAL_CPT(TYPE, OP, FLAG) \
2254 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2256 OP_UPDATE_CRITICAL_CPT(TYPE, OP, 0); \
2259#define OP_GOMP_CRITICAL_CPT(TYPE, OP, FLAG)
2269#define OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2271 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2272 TYPE old_value, new_value; \
2274 old_value = temp_val; \
2275 new_value = (TYPE)(old_value OP rhs); \
2276 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
2277 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2278 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
2280 old_value = temp_val; \
2281 new_value = (TYPE)(old_value OP rhs); \
2290#define ATOMIC_CMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2291 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2294 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
2295 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2299#define ATOMIC_FIXED_ADD_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2300 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2301 TYPE old_value, new_value; \
2303 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
2305 old_value = KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
2307 return old_value OP rhs; \
2441#define ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
2442 TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \
2443 ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs, int flag) { \
2444 KMP_DEBUG_ASSERT(__kmp_init_serial); \
2446 ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \
2450#define ATOMIC_CMPXCHG_CPT_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
2451 RTYPE, LCK_ID, MASK, GOMP_FLAG) \
2452 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
2455 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
2456 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2460#define ATOMIC_CRITICAL_CPT_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
2461 LCK_ID, GOMP_FLAG) \
2462 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
2465 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
2466 OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) \
2469ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char,
add_cpt, 8, +, fp, _Quad, 1
i, 0,
2471ATOMIC_CMPXCHG_CPT_MIX(fixed1u,
uchar,
add_cpt, 8, +, fp, _Quad, 1
i, 0,
2473ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char,
sub_cpt, 8, -, fp, _Quad, 1
i, 0,
2475ATOMIC_CMPXCHG_CPT_MIX(fixed1u,
uchar,
sub_cpt, 8, -, fp, _Quad, 1
i, 0,
2477ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char,
mul_cpt, 8, *, fp, _Quad, 1
i, 0,
2479ATOMIC_CMPXCHG_CPT_MIX(fixed1u,
uchar,
mul_cpt, 8, *, fp, _Quad, 1
i, 0,
2481ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char,
div_cpt, 8, /, fp, _Quad, 1
i, 0,
2483ATOMIC_CMPXCHG_CPT_MIX(fixed1u,
uchar,
div_cpt, 8, /, fp, _Quad, 1
i, 0,
2486ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short,
add_cpt, 16, +, fp, _Quad, 2
i, 1,
2488ATOMIC_CMPXCHG_CPT_MIX(fixed2u,
ushort,
add_cpt, 16, +, fp, _Quad, 2
i, 1,
2490ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short,
sub_cpt, 16, -, fp, _Quad, 2
i, 1,
2492ATOMIC_CMPXCHG_CPT_MIX(fixed2u,
ushort,
sub_cpt, 16, -, fp, _Quad, 2
i, 1,
2494ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short,
mul_cpt, 16, *, fp, _Quad, 2
i, 1,
2496ATOMIC_CMPXCHG_CPT_MIX(fixed2u,
ushort,
mul_cpt, 16, *, fp, _Quad, 2
i, 1,
2498ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short,
div_cpt, 16, /, fp, _Quad, 2
i, 1,
2500ATOMIC_CMPXCHG_CPT_MIX(fixed2u,
ushort,
div_cpt, 16, /, fp, _Quad, 2
i, 1,
2555ATOMIC_CRITICAL_CPT_MIX(float10,
long double,
add_cpt, +, fp, _Quad, 10
r,
2557ATOMIC_CRITICAL_CPT_MIX(float10,
long double,
sub_cpt, -, fp, _Quad, 10
r,
2559ATOMIC_CRITICAL_CPT_MIX(float10,
long double,
mul_cpt, *, fp, _Quad, 10
r,
2561ATOMIC_CRITICAL_CPT_MIX(float10,
long double,
div_cpt, /, fp, _Quad, 10
r,
2575#define OP_CRITICAL_L_CPT(OP, LCK_ID) \
2576 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2580 (*lhs) = new_value; \
2582 new_value = (*lhs); \
2586 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
2589#ifdef KMP_GOMP_COMPAT
2590#define OP_GOMP_CRITICAL_L_CPT(OP, FLAG) \
2591 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2593 OP_CRITICAL_L_CPT(OP, 0); \
2597#define OP_GOMP_CRITICAL_L_CPT(OP, FLAG)
2602#define ATOMIC_CMPX_L_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2603 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2606 OP_GOMP_CRITICAL_L_CPT(= *lhs OP, GOMP_FLAG) \
2607 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2636#define MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \
2637 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2639 if (*lhs OP rhs) { \
2645 new_value = old_value; \
2649 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2653#ifdef KMP_GOMP_COMPAT
2654#define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG) \
2655 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2657 MIN_MAX_CRITSECT_CPT(OP, 0); \
2660#define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG)
2664#define MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \
2666 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2669 old_value = temp_val; \
2670 while (old_value OP rhs && \
2671 !KMP_COMPARE_AND_STORE_ACQ##BITS( \
2672 (kmp_int##BITS *)lhs, \
2673 *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2674 *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \
2676 old_value = temp_val; \
2686#define MIN_MAX_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2687 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2688 TYPE new_value, old_value; \
2689 if (*lhs OP rhs) { \
2690 GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \
2691 MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \
2696#define MIN_MAX_COMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2697 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2698 TYPE new_value, old_value; \
2700 if (*lhs OP rhs) { \
2701 GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \
2702 MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \
2749#ifdef KMP_GOMP_COMPAT
2750#define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG) \
2751 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2753 OP_CRITICAL_CPT(OP, 0); \
2756#define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG)
2759#define ATOMIC_CMPX_EQV_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2760 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2763 OP_GOMP_CRITICAL_EQV_CPT(^= (TYPE) ~, GOMP_FLAG) \
2764 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2792#define ATOMIC_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2793 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2795 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
2796 OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) \
2802#define OP_CRITICAL_CPT_WRK(OP, LCK_ID) \
2803 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2813 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2817#ifdef KMP_GOMP_COMPAT
2818#define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG) \
2819 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2821 OP_CRITICAL_CPT_WRK(OP## =, 0); \
2824#define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG)
2828#define ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \
2829 void __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, TYPE *lhs, \
2830 TYPE rhs, TYPE *out, int flag) { \
2831 KMP_DEBUG_ASSERT(__kmp_init_serial); \
2832 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
2835#define ATOMIC_CRITICAL_CPT_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2836 ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \
2837 OP_GOMP_CRITICAL_CPT_WRK(OP, GOMP_FLAG) \
2838 OP_CRITICAL_CPT_WRK(OP## =, LCK_ID) \
2927#if KMP_ARCH_X86 || KMP_ARCH_X86_64
2934#define OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \
2935 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2939 (*lhs) = (TYPE)((rhs)OP(*lhs)); \
2940 new_value = (*lhs); \
2942 new_value = (*lhs); \
2943 (*lhs) = (TYPE)((rhs)OP(*lhs)); \
2945 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2949#ifdef KMP_GOMP_COMPAT
2950#define OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, FLAG) \
2951 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2953 OP_CRITICAL_CPT_REV(TYPE, OP, 0); \
2956#define OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, FLAG)
2966#define OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
2968 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2969 TYPE old_value, new_value; \
2971 old_value = temp_val; \
2972 new_value = (TYPE)(rhs OP old_value); \
2973 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
2974 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2975 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
2977 old_value = temp_val; \
2978 new_value = (TYPE)(rhs OP old_value); \
2987#define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2988 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2991 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \
2992 OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
2995ATOMIC_CMPXCHG_CPT_REV(fixed1, div_cpt_rev,
kmp_int8, 8, /,
2997ATOMIC_CMPXCHG_CPT_REV(fixed1u, div_cpt_rev,
kmp_uint8, 8, /,
2999ATOMIC_CMPXCHG_CPT_REV(fixed1, shl_cpt_rev,
kmp_int8, 8, <<,
3001ATOMIC_CMPXCHG_CPT_REV(fixed1, shr_cpt_rev,
kmp_int8, 8, >>,
3003ATOMIC_CMPXCHG_CPT_REV(fixed1u, shr_cpt_rev,
kmp_uint8, 8, >>,
3005ATOMIC_CMPXCHG_CPT_REV(fixed1, sub_cpt_rev,
kmp_int8, 8, -,
3007ATOMIC_CMPXCHG_CPT_REV(fixed2, div_cpt_rev,
kmp_int16, 16, /,
3009ATOMIC_CMPXCHG_CPT_REV(fixed2u, div_cpt_rev, kmp_uint16, 16, /,
3011ATOMIC_CMPXCHG_CPT_REV(fixed2, shl_cpt_rev,
kmp_int16, 16, <<,
3013ATOMIC_CMPXCHG_CPT_REV(fixed2, shr_cpt_rev,
kmp_int16, 16, >>,
3015ATOMIC_CMPXCHG_CPT_REV(fixed2u, shr_cpt_rev, kmp_uint16, 16, >>,
3017ATOMIC_CMPXCHG_CPT_REV(fixed2, sub_cpt_rev,
kmp_int16, 16, -,
3019ATOMIC_CMPXCHG_CPT_REV(fixed4, div_cpt_rev,
kmp_int32, 32, /,
3021ATOMIC_CMPXCHG_CPT_REV(fixed4u, div_cpt_rev,
kmp_uint32, 32, /,
3023ATOMIC_CMPXCHG_CPT_REV(fixed4, shl_cpt_rev,
kmp_int32, 32, <<,
3025ATOMIC_CMPXCHG_CPT_REV(fixed4, shr_cpt_rev,
kmp_int32, 32, >>,
3027ATOMIC_CMPXCHG_CPT_REV(fixed4u, shr_cpt_rev,
kmp_uint32, 32, >>,
3029ATOMIC_CMPXCHG_CPT_REV(fixed4, sub_cpt_rev,
kmp_int32, 32, -,
3031ATOMIC_CMPXCHG_CPT_REV(fixed8, div_cpt_rev,
kmp_int64, 64, /,
3033ATOMIC_CMPXCHG_CPT_REV(fixed8u, div_cpt_rev,
kmp_uint64, 64, /,
3035ATOMIC_CMPXCHG_CPT_REV(fixed8, shl_cpt_rev,
kmp_int64, 64, <<,
3037ATOMIC_CMPXCHG_CPT_REV(fixed8, shr_cpt_rev,
kmp_int64, 64, >>,
3039ATOMIC_CMPXCHG_CPT_REV(fixed8u, shr_cpt_rev,
kmp_uint64, 64, >>,
3041ATOMIC_CMPXCHG_CPT_REV(fixed8, sub_cpt_rev,
kmp_int64, 64, -,
3043ATOMIC_CMPXCHG_CPT_REV(float4, div_cpt_rev,
kmp_real32, 32, /,
3045ATOMIC_CMPXCHG_CPT_REV(float4, sub_cpt_rev,
kmp_real32, 32, -,
3059#define ATOMIC_CRITICAL_CPT_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
3060 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
3063 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \
3064 OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \
3069ATOMIC_CRITICAL_CPT_REV(float10, sub_cpt_rev,
long double, -, 10
r,
3071ATOMIC_CRITICAL_CPT_REV(float10, div_cpt_rev,
long double, /, 10
r,
3075ATOMIC_CRITICAL_CPT_REV(float16, sub_cpt_rev,
QUAD_LEGACY, -, 16
r,
3077ATOMIC_CRITICAL_CPT_REV(float16, div_cpt_rev,
QUAD_LEGACY, /, 16
r,
3080ATOMIC_CRITICAL_CPT_REV(float16, sub_a16_cpt_rev, Quad_a16_t, -, 16
r,
3082ATOMIC_CRITICAL_CPT_REV(float16, div_a16_cpt_rev, Quad_a16_t, /, 16
r,
3092#define OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \
3093 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3096 (*lhs) = (rhs)OP(*lhs); \
3100 (*lhs) = (rhs)OP(*lhs); \
3103 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3107#ifdef KMP_GOMP_COMPAT
3108#define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG) \
3109 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
3111 OP_CRITICAL_CPT_REV_WRK(OP, 0); \
3114#define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG)
3118#define ATOMIC_CRITICAL_CPT_REV_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, \
3120 ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \
3121 OP_GOMP_CRITICAL_CPT_REV_WRK(OP, GOMP_FLAG) \
3122 OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \
3128ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, sub_cpt_rev,
kmp_cmplx32, -, 8c,
3130ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, div_cpt_rev,
kmp_cmplx32, /, 8c,
3137ATOMIC_CRITICAL_CPT_REV(cmplx10, sub_cpt_rev,
kmp_cmplx80, -, 20c,
3139ATOMIC_CRITICAL_CPT_REV(cmplx10, div_cpt_rev,
kmp_cmplx80, /, 20c,
3142ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_cpt_rev,
CPLX128_LEG, -, 32c,
3144ATOMIC_CRITICAL_CPT_REV(cmplx16, div_cpt_rev,
CPLX128_LEG, /, 32c,
3147ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_a16_cpt_rev, kmp_cmplx128_a16_t, -, 32c,
3149ATOMIC_CRITICAL_CPT_REV(cmplx16, div_a16_cpt_rev, kmp_cmplx128_a16_t, /, 32c,
3163#define ATOMIC_CMPXCHG_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
3164 RTYPE, LCK_ID, MASK, GOMP_FLAG) \
3165 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
3168 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \
3169 OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
3173#define ATOMIC_CRITICAL_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
3174 LCK_ID, GOMP_FLAG) \
3175 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
3178 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \
3179 OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \
3182ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1,
char, sub_cpt_rev, 8, -, fp, _Quad, 1
i, 0,
3184ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u,
uchar, sub_cpt_rev, 8, -, fp, _Quad, 1
i, 0,
3186ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1,
char, div_cpt_rev, 8, /, fp, _Quad, 1
i, 0,
3188ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u,
uchar, div_cpt_rev, 8, /, fp, _Quad, 1
i, 0,
3191ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2,
short, sub_cpt_rev, 16, -, fp, _Quad, 2
i, 1,
3193ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u,
ushort, sub_cpt_rev, 16, -, fp, _Quad, 2
i,
3196ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2,
short, div_cpt_rev, 16, /, fp, _Quad, 2
i, 1,
3198ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u,
ushort, div_cpt_rev, 16, /, fp, _Quad, 2
i,
3202ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4,
kmp_int32, sub_cpt_rev, 32, -, fp, _Quad, 4
i,
3204ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u,
kmp_uint32, sub_cpt_rev, 32, -, fp, _Quad,
3206ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4,
kmp_int32, div_cpt_rev, 32, /, fp, _Quad, 4
i,
3208ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u,
kmp_uint32, div_cpt_rev, 32, /, fp, _Quad,
3211ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8,
kmp_int64, sub_cpt_rev, 64, -, fp, _Quad, 8
i,
3214ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u,
kmp_uint64, sub_cpt_rev, 64, -, fp, _Quad,
3217ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8,
kmp_int64, div_cpt_rev, 64, /, fp, _Quad, 8
i,
3220ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u,
kmp_uint64, div_cpt_rev, 64, /, fp, _Quad,
3224ATOMIC_CMPXCHG_CPT_REV_MIX(float4,
kmp_real32, sub_cpt_rev, 32, -, fp, _Quad,
3227ATOMIC_CMPXCHG_CPT_REV_MIX(float4,
kmp_real32, div_cpt_rev, 32, /, fp, _Quad,
3231ATOMIC_CMPXCHG_CPT_REV_MIX(
float8,
kmp_real64, sub_cpt_rev, 64, -, fp, _Quad,
3234ATOMIC_CMPXCHG_CPT_REV_MIX(
float8,
kmp_real64, div_cpt_rev, 64, /, fp, _Quad,
3238ATOMIC_CRITICAL_CPT_REV_MIX(float10,
long double, sub_cpt_rev, -, fp, _Quad,
3240ATOMIC_CRITICAL_CPT_REV_MIX(float10,
long double, div_cpt_rev, /, fp, _Quad,
3247#define ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3248 TYPE __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \
3250 KMP_DEBUG_ASSERT(__kmp_init_serial); \
3251 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid));
3253#define CRITICAL_SWP(LCK_ID) \
3254 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3256 old_value = (*lhs); \
3259 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3263#ifdef KMP_GOMP_COMPAT
3264#define GOMP_CRITICAL_SWP(FLAG) \
3265 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
3270#define GOMP_CRITICAL_SWP(FLAG)
3273#define ATOMIC_XCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \
3274 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3276 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3277 old_value = KMP_XCHG_FIXED##BITS(lhs, rhs); \
3281#define ATOMIC_XCHG_FLOAT_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \
3282 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3284 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3285 old_value = KMP_XCHG_REAL##BITS(lhs, rhs); \
3290#define CMPXCHG_SWP(TYPE, BITS) \
3292 TYPE KMP_ATOMIC_VOLATILE temp_val; \
3293 TYPE old_value, new_value; \
3295 old_value = temp_val; \
3297 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
3298 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
3299 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
3301 old_value = temp_val; \
3308#define ATOMIC_CMPXCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \
3309 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3312 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3313 CMPXCHG_SWP(TYPE, BITS) \
3324ATOMIC_CMPXCHG_SWP(fixed8,
kmp_int64, 64,
3337#define ATOMIC_CRITICAL_SWP(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \
3338 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3340 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3341 CRITICAL_SWP(LCK_ID) \
3349#define ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \
3350 void __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \
3351 TYPE rhs, TYPE *out) { \
3352 KMP_DEBUG_ASSERT(__kmp_init_serial); \
3353 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid));
3355#define CRITICAL_SWP_WRK(LCK_ID) \
3356 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3361 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3365#ifdef KMP_GOMP_COMPAT
3366#define GOMP_CRITICAL_SWP_WRK(FLAG) \
3367 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
3369 CRITICAL_SWP_WRK(0); \
3372#define GOMP_CRITICAL_SWP_WRK(FLAG)
3376#define ATOMIC_CRITICAL_SWP_WRK(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \
3377 ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \
3379 GOMP_CRITICAL_SWP_WRK(GOMP_FLAG) \
3380 CRITICAL_SWP_WRK(LCK_ID) \
3384ATOMIC_CRITICAL_SWP(float10,
long double, 10
r, 1)
3389ATOMIC_CRITICAL_SWP_WRK(cmplx4,
kmp_cmplx32, 8c, 1)
3399ATOMIC_CRITICAL_SWP(float16_a16, Quad_a16_t, 16
r,
3401ATOMIC_CRITICAL_SWP(cmplx16_a16, kmp_cmplx128_a16_t, 32c,
3416 void (*
f)(
void *,
void *,
void *)) {
3429 (*f)(&new_value, &old_value, rhs);
3437 (*f)(&new_value, &old_value, rhs);
3444#ifdef KMP_GOMP_COMPAT
3451 (*f)(lhs, lhs, rhs);
3453#ifdef KMP_GOMP_COMPAT
3463 void (*
f)(
void *,
void *,
void *)) {
3476 (*f)(&new_value, &old_value, rhs);
3484 (*f)(&new_value, &old_value, rhs);
3491#ifdef KMP_GOMP_COMPAT
3498 (*f)(lhs, lhs, rhs);
3500#ifdef KMP_GOMP_COMPAT
3510 void (*
f)(
void *,
void *,
void *)) {
3525 (*f)(&new_value, &old_value, rhs);
3533 (*f)(&new_value, &old_value, rhs);
3541#ifdef KMP_GOMP_COMPAT
3548 (*f)(lhs, lhs, rhs);
3550#ifdef KMP_GOMP_COMPAT
3560 void (*
f)(
void *,
void *,
void *)) {
3575 (*f)(&new_value, &old_value, rhs);
3582 (*f)(&new_value, &old_value, rhs);
3590#ifdef KMP_GOMP_COMPAT
3597 (*f)(lhs, lhs, rhs);
3599#ifdef KMP_GOMP_COMPAT
3607#if KMP_ARCH_X86 || KMP_ARCH_X86_64
3609 void (*
f)(
void *,
void *,
void *)) {
3612#ifdef KMP_GOMP_COMPAT
3619 (*f)(lhs, lhs, rhs);
3621#ifdef KMP_GOMP_COMPAT
3631 void (*
f)(
void *,
void *,
void *)) {
3634#ifdef KMP_GOMP_COMPAT
3641 (*f)(lhs, lhs, rhs);
3643#ifdef KMP_GOMP_COMPAT
3650#if KMP_ARCH_X86 || KMP_ARCH_X86_64
3652 void (*
f)(
void *,
void *,
void *)) {
3655#ifdef KMP_GOMP_COMPAT
3662 (*f)(lhs, lhs, rhs);
3664#ifdef KMP_GOMP_COMPAT
3673 void (*
f)(
void *,
void *,
void *)) {
3676#ifdef KMP_GOMP_COMPAT
3683 (*f)(lhs, lhs, rhs);
3685#ifdef KMP_GOMP_COMPAT
3698 KA_TRACE(20, (
"__kmpc_atomic_start: T#%d\n", gtid));
3704 KA_TRACE(20, (
"__kmpc_atomic_end: T#%d\n", gtid));
3708#if KMP_ARCH_X86 || KMP_ARCH_X86_64
3798 short d,
short *pv) {
3845 *pv = old == e ?
d : old;
3849 short d,
short *pv) {
3852 *pv = old == e ?
d : old;
3859 *pv = old == e ?
d : old;
3866 *pv = old == e ?
d : old;
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d
#define __kmp_entry_gtid()
volatile int __kmp_init_serial
kmp_atomic_lock_t __kmp_atomic_lock_8c
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86<<, 2i, 1, KMP_ARCH_X86) ATOMIC_CMPXCHG(fixed2, shr, kmp_int16, 16, > KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 xor
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 div
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 orb
kmp_atomic_lock_t __kmp_atomic_lock_8r
kmp_atomic_lock_t __kmp_atomic_lock_4i
#define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, LCK_ID, MASK, GOMP_FLAG)
#define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG)
#define ATOMIC_CRITICAL_WR(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG)
#define ATOMIC_CMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG)
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 kmp_int16
kmp_atomic_lock_t __kmp_atomic_lock_20c
#define ATOMIC_FIXED_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG)
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86<<, KMP_ARCH_X86) ATOMIC_CMPXCHG_CPT(fixed2, shr_cpt, kmp_int16, 16, > KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 xor_cpt
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 mul_cpt
KMP_ARCH_X86 KMP_ARCH_X86 eqv_cpt
kmp_atomic_lock_t __kmp_atomic_lock_16c
#define ATOMIC_CMPX_L_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG)
#define ATOMIC_XCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG)
kmp_atomic_lock_t __kmp_atomic_lock_2i
#define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG)
#define MIN_MAX_COMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG)
KMP_ARCH_X86 KMP_ARCH_X86 eqv
#define ATOMIC_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG)
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 add_cpt
kmp_atomic_lock_t __kmp_atomic_lock_32c
kmp_atomic_lock_t __kmp_atomic_lock_8i
void __kmpc_atomic_2(ident_t *id_ref, int gtid, void *lhs, void *rhs, void(*f)(void *, void *, void *))
#define ATOMIC_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG)
long long void __kmpc_atomic_1(ident_t *id_ref, int gtid, void *lhs, void *rhs, void(*f)(void *, void *, void *))
#define ATOMIC_CMPXCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG)
void __kmpc_atomic_4(ident_t *id_ref, int gtid, void *lhs, void *rhs, void(*f)(void *, void *, void *))
#define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, LCK_ID, MASK, GOMP_FLAG)
KMP_ARCH_X86 KMP_ARCH_X86 long wr
kmp_atomic_lock_t __kmp_atomic_lock
#define ATOMIC_CRITICAL_READ(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG)
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 div_cpt
void __kmpc_atomic_end(void)
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 shl
#define ATOMIC_CRITICAL_CPT_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG)
#define ATOMIC_CMPX_EQV_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG)
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 andb
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 mul
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 shl_cpt
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 andb_cpt
#define MIN_MAX_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG)
#define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG)
kmp_atomic_lock_t __kmp_atomic_lock_10r
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 kmp_int8
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 shr_cpt
void __kmpc_atomic_start(void)
#define ATOMIC_XCHG_FLOAT_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG)
void __kmpc_atomic_16(ident_t *id_ref, int gtid, void *lhs, void *rhs, void(*f)(void *, void *, void *))
#define ATOMIC_CRITICAL_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, LCK_ID, GOMP_FLAG)
#define ATOMIC_FIXED_ADD_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG)
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 orb_cpt
kmp_atomic_lock_t __kmp_atomic_lock_1i
#define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG)
void __kmpc_atomic_32(ident_t *id_ref, int gtid, void *lhs, void *rhs, void(*f)(void *, void *, void *))
#define MIN_MAX_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG)
kmp_atomic_lock_t __kmp_atomic_lock_16r
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 shr
void __kmpc_atomic_8(ident_t *id_ref, int gtid, void *lhs, void *rhs, void(*f)(void *, void *, void *))
#define ATOMIC_CMPXCHG_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG)
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 add
#define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG)
kmp_atomic_lock_t __kmp_atomic_lock_4r
double _Complex kmp_cmplx64
long double _Complex kmp_cmplx80
static void __kmp_acquire_atomic_lock(kmp_atomic_lock_t *lck, kmp_int32 gtid)
float _Complex kmp_cmplx32
static void __kmp_release_atomic_lock(kmp_atomic_lock_t *lck, kmp_int32 gtid)
void __kmpc_atomic_20(ident_t *id_ref, int gtid, void *lhs, void *rhs, void(*f)(void *, void *, void *))
void __kmpc_atomic_10(ident_t *id_ref, int gtid, void *lhs, void *rhs, void(*f)(void *, void *, void *))
bool __kmpc_atomic_bool_4_cas(ident_t *loc, int gtid, int *x, int e, int d)
long long __kmpc_atomic_val_8_cas(ident_t *loc, int gtid, long long *x, long long e, long long d)
int __kmpc_atomic_val_4_cas(ident_t *loc, int gtid, int *x, int e, int d)
bool __kmpc_atomic_bool_8_cas(ident_t *loc, int gtid, long long *x, long long e, long long d)
short __kmpc_atomic_val_2_cas(ident_t *loc, int gtid, short *x, short e, short d)
bool __kmpc_atomic_bool_2_cas(ident_t *loc, int gtid, short *x, short e, short d)
char __kmpc_atomic_val_1_cas(ident_t *loc, int gtid, char *x, char e, char d)
bool __kmpc_atomic_bool_1_cas(ident_t *loc, int gtid, char *x, char e, char d)
bool __kmpc_atomic_bool_1_cas_cpt(ident_t *loc, int gtid, char *x, char e, char d, char *pv)
char __kmpc_atomic_val_1_cas_cpt(ident_t *loc, int gtid, char *x, char e, char d, char *pv)
int __kmpc_atomic_val_4_cas_cpt(ident_t *loc, int gtid, int *x, int e, int d, int *pv)
bool __kmpc_atomic_bool_8_cas_cpt(ident_t *loc, int gtid, long long *x, long long e, long long d, long long *pv)
bool __kmpc_atomic_bool_2_cas_cpt(ident_t *loc, int gtid, short *x, short e, short d, short *pv)
bool __kmpc_atomic_bool_4_cas_cpt(ident_t *loc, int gtid, int *x, int e, int d, int *pv)
long long __kmpc_atomic_val_8_cas_cpt(ident_t *loc, int gtid, long long *x, long long e, long long d, long long *pv)
short __kmpc_atomic_val_2_cas_cpt(ident_t *loc, int gtid, short *x, short e, short d, short *pv)
#define KMP_DEBUG_ASSERT(cond)
unsigned long long kmp_uint64
#define KMP_COMPARE_AND_STORE_RET8(p, cv, sv)
#define KMP_COMPARE_AND_STORE_RET64(p, cv, sv)
#define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv)
#define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv)
#define KMP_COMPARE_AND_STORE_RET32(p, cv, sv)
#define KMP_COMPARE_AND_STORE_RET16(p, cv, sv)
#define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv)
#define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv)
unsigned long kmp_uintptr_t
tsc_tick_count::tsc_interval_t operator-(const tsc_tick_count &t1, const tsc_tick_count &t0)