jcs.org / qemu
qemu with hax to log dma reads & writes jcs.org/2018/11/12/vfio
fork atom

tcg: Add atomic128 helpers

Force the use of cmpxchg16b on x86_64.

Wikipedia suggests that only very old AMD64 (circa 2004) did not have
this instruction. Further, it's required by Windows 8 so no new cpus
will ever omit it.

If we truely care about these, then we could check this at startup time
and then avoid executing paths that use it.

Reviewed-by: Emilio G. Cota <cota@braap.org>
Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Richard Henderson <rth@twiddle.net>

Richard Henderson 7ebee43e c482cb11

+119 -3
+39 -1
atomic_template.h
··· 18 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 19 */ 20 20 21 - #if DATA_SIZE == 8 21 + #if DATA_SIZE == 16 22 + # define SUFFIX o 23 + # define DATA_TYPE Int128 24 + # define BSWAP bswap128 25 + #elif DATA_SIZE == 8 22 26 # define SUFFIX q 23 27 # define DATA_TYPE uint64_t 24 28 # define BSWAP bswap64 ··· 61 65 return atomic_cmpxchg__nocheck(haddr, cmpv, newv); 62 66 } 63 67 68 + #if DATA_SIZE >= 16 69 + ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) 70 + { 71 + DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; 72 + __atomic_load(haddr, &val, __ATOMIC_RELAXED); 73 + return val; 74 + } 75 + 76 + void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, 77 + ABI_TYPE val EXTRA_ARGS) 78 + { 79 + DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 80 + __atomic_store(haddr, &val, __ATOMIC_RELAXED); 81 + } 82 + #else 64 83 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, 65 84 ABI_TYPE val EXTRA_ARGS) 66 85 { ··· 86 105 GEN_ATOMIC_HELPER(xor_fetch) 87 106 88 107 #undef GEN_ATOMIC_HELPER 108 + #endif /* DATA SIZE >= 16 */ 109 + 89 110 #undef END 90 111 91 112 #if DATA_SIZE > 1 ··· 105 126 return BSWAP(atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv))); 106 127 } 107 128 129 + #if DATA_SIZE >= 16 130 + ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) 131 + { 132 + DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; 133 + __atomic_load(haddr, &val, __ATOMIC_RELAXED); 134 + return BSWAP(val); 135 + } 136 + 137 + void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, 138 + ABI_TYPE val EXTRA_ARGS) 139 + { 140 + DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 141 + val = BSWAP(val); 142 + __atomic_store(haddr, &val, __ATOMIC_RELAXED); 143 + } 144 + #else 108 145 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, 109 146 ABI_TYPE val EXTRA_ARGS) 110 147 { ··· 166 203 ldo = ldn; 167 204 } 168 205 } 206 + #endif /* DATA_SIZE >= 16 */ 169 207 170 208 #undef END 171 209 #endif /* DATA_SIZE > 1 */
+28 -1
configure
··· 1216 1216 cc_i386='$(CC) -m32' 1217 1217 ;; 1218 1218 x86_64) 1219 - CPU_CFLAGS="-m64" 1219 + # ??? Only extremely old AMD cpus do not have cmpxchg16b. 1220 + # If we truly care, we should simply detect this case at 1221 + # runtime and generate the fallback to serial emulation. 1222 + CPU_CFLAGS="-m64 -mcx16" 1220 1223 LDFLAGS="-m64 $LDFLAGS" 1221 1224 cc_i386='$(CC) -m32' 1222 1225 ;; ··· 4521 4524 int128=yes 4522 4525 fi 4523 4526 4527 + ######################################### 4528 + # See if 128-bit atomic operations are supported. 4529 + 4530 + atomic128=no 4531 + if test "$int128" = "yes"; then 4532 + cat > $TMPC << EOF 4533 + int main(void) 4534 + { 4535 + unsigned __int128 x = 0, y = 0; 4536 + y = __atomic_load_16(&x, 0); 4537 + __atomic_store_16(&x, y, 0); 4538 + __atomic_compare_exchange_16(&x, &y, x, 0, 0, 0); 4539 + return 0; 4540 + } 4541 + EOF 4542 + if compile_prog "" "" ; then 4543 + atomic128=yes 4544 + fi 4545 + fi 4546 + 4524 4547 ######################################## 4525 4548 # check if getauxval is available. 4526 4549 ··· 5481 5504 5482 5505 if test "$int128" = "yes" ; then 5483 5506 echo "CONFIG_INT128=y" >> $config_host_mak 5507 + fi 5508 + 5509 + if test "$atomic128" = "yes" ; then 5510 + echo "CONFIG_ATOMIC128=y" >> $config_host_mak 5484 5511 fi 5485 5512 5486 5513 if test "$getauxval" = "yes" ; then
+5
cputlb.c
··· 690 690 #define DATA_SIZE 8 691 691 #include "atomic_template.h" 692 692 693 + #ifdef CONFIG_ATOMIC128 694 + #define DATA_SIZE 16 695 + #include "atomic_template.h" 696 + #endif 697 + 693 698 /* Second set of helpers are directly callable from TCG as helpers. */ 694 699 695 700 #undef EXTRA_ARGS
+6
include/qemu/int128.h
··· 2 2 #define INT128_H 3 3 4 4 #ifdef CONFIG_INT128 5 + #include "qemu/bswap.h" 5 6 6 7 typedef __int128_t Int128; 7 8 ··· 135 136 static inline void int128_subfrom(Int128 *a, Int128 b) 136 137 { 137 138 *a -= b; 139 + } 140 + 141 + static inline Int128 bswap128(Int128 a) 142 + { 143 + return int128_make128(bswap64(int128_gethi(a)), bswap64(int128_getlo(a))); 138 144 } 139 145 140 146 #else /* !CONFIG_INT128 */
+18
tcg-runtime.c
··· 133 133 #define DATA_SIZE 8 134 134 #include "atomic_template.h" 135 135 136 + /* The following is only callable from other helpers, and matches up 137 + with the softmmu version. */ 138 + 139 + #ifdef CONFIG_ATOMIC128 140 + 141 + #undef EXTRA_ARGS 142 + #undef ATOMIC_NAME 143 + #undef ATOMIC_MMU_LOOKUP 144 + 145 + #define EXTRA_ARGS , TCGMemOpIdx oi, uintptr_t retaddr 146 + #define ATOMIC_NAME(X) \ 147 + HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu)) 148 + #define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, DATA_SIZE, retaddr) 149 + 150 + #define DATA_SIZE 16 151 + #include "atomic_template.h" 152 + #endif /* CONFIG_ATOMIC128 */ 153 + 136 154 #endif /* !CONFIG_SOFTMMU */
+23 -1
tcg/tcg.h
··· 1229 1229 1230 1230 #undef GEN_ATOMIC_HELPER_ALL 1231 1231 #undef GEN_ATOMIC_HELPER 1232 + #endif /* CONFIG_SOFTMMU */ 1232 1233 1233 - #endif /* CONFIG_SOFTMMU */ 1234 + #ifdef CONFIG_ATOMIC128 1235 + #include "qemu/int128.h" 1236 + 1237 + /* These aren't really a "proper" helpers because TCG cannot manage Int128. 1238 + However, use the same format as the others, for use by the backends. */ 1239 + Int128 helper_atomic_cmpxchgo_le_mmu(CPUArchState *env, target_ulong addr, 1240 + Int128 cmpv, Int128 newv, 1241 + TCGMemOpIdx oi, uintptr_t retaddr); 1242 + Int128 helper_atomic_cmpxchgo_be_mmu(CPUArchState *env, target_ulong addr, 1243 + Int128 cmpv, Int128 newv, 1244 + TCGMemOpIdx oi, uintptr_t retaddr); 1245 + 1246 + Int128 helper_atomic_ldo_le_mmu(CPUArchState *env, target_ulong addr, 1247 + TCGMemOpIdx oi, uintptr_t retaddr); 1248 + Int128 helper_atomic_ldo_be_mmu(CPUArchState *env, target_ulong addr, 1249 + TCGMemOpIdx oi, uintptr_t retaddr); 1250 + void helper_atomic_sto_le_mmu(CPUArchState *env, target_ulong addr, Int128 val, 1251 + TCGMemOpIdx oi, uintptr_t retaddr); 1252 + void helper_atomic_sto_be_mmu(CPUArchState *env, target_ulong addr, Int128 val, 1253 + TCGMemOpIdx oi, uintptr_t retaddr); 1254 + 1255 + #endif /* CONFIG_ATOMIC128 */ 1234 1256 1235 1257 #endif /* TCG_H */