17 #ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
18 #include <Kokkos_Macros.hpp>
20 "Including non-public Kokkos header files is not allowed.");
22 #ifndef KOKKOS_DESUL_ATOMICS_WRAPPER_HPP_
23 #define KOKKOS_DESUL_ATOMICS_WRAPPER_HPP_
24 #include <Kokkos_Macros.hpp>
26 #include <Kokkos_Atomics_Desul_Config.hpp>
27 #include <desul/atomics.hpp>
29 #include <impl/Kokkos_Volatile_Load.hpp>
36 inline const char* atomic_query_version() {
return "KOKKOS_DESUL_ATOMICS"; }
38 #if defined(KOKKOS_COMPILER_GNU) && !defined(__PGIC__) && \
39 !defined(__CUDA_ARCH__)
41 #define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr, 0, 0)
42 #define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr, 1, 0)
46 #define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0)
47 #define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0)
52 #ifdef KOKKOS_ENABLE_ATOMICS_BYPASS
53 #define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeCaller()
55 #define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeDevice()
58 template<
class T> KOKKOS_INLINE_FUNCTION
59 T atomic_load(T*
const dest) {
return desul::atomic_load(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
61 template<
class T> KOKKOS_INLINE_FUNCTION
62 void atomic_store(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_store(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
64 template<
class T> KOKKOS_INLINE_FUNCTION
65 void atomic_assign(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { atomic_store(dest,val); }
67 KOKKOS_INLINE_FUNCTION
69 desul::atomic_thread_fence(desul::MemoryOrderSeqCst(), KOKKOS_DESUL_MEM_SCOPE);
72 KOKKOS_INLINE_FUNCTION
73 void load_fence() {
return desul::atomic_thread_fence(desul::MemoryOrderAcquire(), KOKKOS_DESUL_MEM_SCOPE); }
75 KOKKOS_INLINE_FUNCTION
76 void store_fence() {
return desul::atomic_thread_fence(desul::MemoryOrderRelease(), KOKKOS_DESUL_MEM_SCOPE); }
79 template<
class T> KOKKOS_INLINE_FUNCTION
80 T atomic_fetch_add (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
82 template<
class T> KOKKOS_INLINE_FUNCTION
83 T atomic_fetch_sub (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
85 template<
class T> KOKKOS_INLINE_FUNCTION
86 T atomic_fetch_max (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_max (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
88 template<
class T> KOKKOS_INLINE_FUNCTION
89 T atomic_fetch_min (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_min (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
91 template<
class T> KOKKOS_INLINE_FUNCTION
92 T atomic_fetch_mul (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_mul (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
94 template<
class T> KOKKOS_INLINE_FUNCTION
95 T atomic_fetch_div (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_div (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
97 template<
class T> KOKKOS_INLINE_FUNCTION
98 T atomic_fetch_mod (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_mod (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
100 template<
class T> KOKKOS_INLINE_FUNCTION
101 T atomic_fetch_and (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_and (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
103 template<
class T> KOKKOS_INLINE_FUNCTION
104 T atomic_fetch_or (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_or (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
106 template<
class T> KOKKOS_INLINE_FUNCTION
107 T atomic_fetch_xor (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_xor (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
109 template<
class T> KOKKOS_INLINE_FUNCTION
110 T atomic_fetch_nand(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_nand(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
112 template<
class T> KOKKOS_INLINE_FUNCTION
113 T atomic_fetch_lshift(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_lshift(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
115 template<
class T> KOKKOS_INLINE_FUNCTION
116 T atomic_fetch_rshift(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_rshift(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
118 template<
class T> KOKKOS_INLINE_FUNCTION
119 T atomic_fetch_inc(T*
const dest) {
return desul::atomic_fetch_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
121 template<
class T> KOKKOS_INLINE_FUNCTION
122 T atomic_fetch_dec(T*
const dest) {
return desul::atomic_fetch_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
126 template<
class T> KOKKOS_INLINE_FUNCTION
127 T atomic_add_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_add_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
129 template<
class T> KOKKOS_INLINE_FUNCTION
130 T atomic_sub_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_sub_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
132 template<
class T> KOKKOS_INLINE_FUNCTION
133 T atomic_max_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_max_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
135 template<
class T> KOKKOS_INLINE_FUNCTION
136 T atomic_min_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_min_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
138 template<
class T> KOKKOS_INLINE_FUNCTION
139 T atomic_mul_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_mul_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
141 template<
class T> KOKKOS_INLINE_FUNCTION
142 T atomic_div_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_div_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
144 template<
class T> KOKKOS_INLINE_FUNCTION
145 T atomic_mod_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_mod_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
147 template<
class T> KOKKOS_INLINE_FUNCTION
148 T atomic_and_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_and_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
150 template<
class T> KOKKOS_INLINE_FUNCTION
151 T atomic_or_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_or_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
153 template<
class T> KOKKOS_INLINE_FUNCTION
154 T atomic_xor_fetch (T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_xor_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
156 template<
class T> KOKKOS_INLINE_FUNCTION
157 T atomic_nand_fetch(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_nand_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
159 template<
class T> KOKKOS_INLINE_FUNCTION
160 T atomic_lshift_fetch(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_lshift_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
162 template<
class T> KOKKOS_INLINE_FUNCTION
163 T atomic_rshift_fetch(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_rshift_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
165 template<
class T> KOKKOS_INLINE_FUNCTION
166 T atomic_inc_fetch(T*
const dest) {
return desul::atomic_inc_fetch(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
168 template<
class T> KOKKOS_INLINE_FUNCTION
169 T atomic_dec_fetch(T*
const dest) {
return desul::atomic_dec_fetch(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
173 template<
class T> KOKKOS_INLINE_FUNCTION
174 void atomic_add(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
176 template<
class T> KOKKOS_INLINE_FUNCTION
177 void atomic_sub(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
179 template<
class T> KOKKOS_INLINE_FUNCTION
180 void atomic_mul(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_mul (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
182 template<
class T> KOKKOS_INLINE_FUNCTION
183 void atomic_div(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_div (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
185 template<
class T> KOKKOS_INLINE_FUNCTION
186 void atomic_min(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_min (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
188 template<
class T> KOKKOS_INLINE_FUNCTION
189 void atomic_max(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_max (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
192 template<
class T> KOKKOS_INLINE_FUNCTION
193 void atomic_and(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { (void) desul::atomic_fetch_and (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
196 template<
class T> KOKKOS_INLINE_FUNCTION
197 void atomic_or(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { (void) desul::atomic_fetch_or (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
199 template<
class T> KOKKOS_INLINE_FUNCTION
200 void atomic_inc(T*
const dest) {
return desul::atomic_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
202 template<
class T> KOKKOS_INLINE_FUNCTION
203 void atomic_dec(T*
const dest) {
return desul::atomic_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
205 template<
class T> KOKKOS_INLINE_FUNCTION
206 void atomic_increment(T*
const dest) {
return desul::atomic_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
208 template<
class T> KOKKOS_INLINE_FUNCTION
209 void atomic_decrement(T*
const dest) {
return desul::atomic_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
213 template<
class T> KOKKOS_INLINE_FUNCTION
214 T atomic_exchange(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_exchange(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
216 template<
class T> KOKKOS_INLINE_FUNCTION
217 bool atomic_compare_exchange_strong(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> expected, desul::Impl::dont_deduce_this_parameter_t<const T> desired) {
218 T expected_ref = expected;
219 return desul::atomic_compare_exchange_strong(dest, expected_ref, desired,
220 desul::MemoryOrderRelaxed(), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE);
223 template<
class T> KOKKOS_INLINE_FUNCTION
224 T atomic_compare_exchange(T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> compare, desul::Impl::dont_deduce_this_parameter_t<const T> desired) {
225 return desul::atomic_compare_exchange(dest, compare, desired,
226 desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE);
230 template<
class T,
class MemOrderSuccess,
class MemOrderFailure> KOKKOS_INLINE_FUNCTION
231 bool atomic_compare_exchange_strong(T*
const dest, T& expected,
const T desired, MemOrderSuccess succ, MemOrderFailure fail) {
232 return desul::atomic_compare_exchange_strong(dest, expected, desired, succ, fail, KOKKOS_DESUL_MEM_SCOPE);
234 template<
class T,
class MemoryOrder>
235 KOKKOS_INLINE_FUNCTION
236 T atomic_load(
const T*
const src, MemoryOrder order) {
237 return desul::atomic_load(src, order, KOKKOS_DESUL_MEM_SCOPE);
239 template<
class T,
class MemoryOrder>
240 KOKKOS_INLINE_FUNCTION
241 void atomic_store(T*
const src,
const T val, MemoryOrder order) {
242 return desul::atomic_store(src, val, order, KOKKOS_DESUL_MEM_SCOPE);
248 #undef KOKKOS_DESUL_MEM_SCOPE