17 #ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
18 #include <Kokkos_Macros.hpp>
20 "Including non-public Kokkos header files is not allowed.");
22 #ifndef KOKKOS_DESUL_ATOMICS_VOLATILE_WRAPPER_HPP_
23 #define KOKKOS_DESUL_ATOMICS_VOLATILE_WRAPPER_HPP_
24 #include <Kokkos_Macros.hpp>
25 #include <Kokkos_Atomics_Desul_Config.hpp>
26 #include <desul/atomics.hpp>
28 #ifdef KOKKOS_ENABLE_ATOMICS_BYPASS
29 #define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeCaller()
31 #define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeDevice()
37 template<
class T> KOKKOS_INLINE_FUNCTION
38 T atomic_load(
volatile T*
const dest) {
return desul::atomic_load(const_cast<T*>(dest), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
40 template<
class T> KOKKOS_INLINE_FUNCTION
41 void atomic_store(
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_store(const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
44 template<
class T> KOKKOS_INLINE_FUNCTION
45 T atomic_fetch_add (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_add (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
47 template<
class T> KOKKOS_INLINE_FUNCTION
48 T atomic_fetch_sub (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_sub (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
50 template<
class T> KOKKOS_INLINE_FUNCTION
51 T atomic_fetch_max (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_max (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
53 template<
class T> KOKKOS_INLINE_FUNCTION
54 T atomic_fetch_min (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_min (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
56 template<
class T> KOKKOS_INLINE_FUNCTION
57 T atomic_fetch_mul (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_mul (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
59 template<
class T> KOKKOS_INLINE_FUNCTION
60 T atomic_fetch_div (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_div (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
62 template<
class T> KOKKOS_INLINE_FUNCTION
63 T atomic_fetch_mod (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_mod (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
65 template<
class T> KOKKOS_INLINE_FUNCTION
66 T atomic_fetch_and (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_and (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
68 template<
class T> KOKKOS_INLINE_FUNCTION
69 T atomic_fetch_or (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_or (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
71 template<
class T> KOKKOS_INLINE_FUNCTION
72 T atomic_fetch_xor (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_xor (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
74 template<
class T> KOKKOS_INLINE_FUNCTION
75 T atomic_fetch_nand(
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_nand(const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
77 template<
class T> KOKKOS_INLINE_FUNCTION
78 T atomic_fetch_lshift(
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_lshift(const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
80 template<
class T> KOKKOS_INLINE_FUNCTION
81 T atomic_fetch_rshift(
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_fetch_rshift(const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
83 template<
class T> KOKKOS_INLINE_FUNCTION
84 T atomic_fetch_inc(
volatile T*
const dest) {
return desul::atomic_fetch_inc(const_cast<T*>(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
86 template<
class T> KOKKOS_INLINE_FUNCTION
87 T atomic_fetch_dec(
volatile T*
const dest) {
return desul::atomic_fetch_dec(const_cast<T*>(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
91 template<
class T> KOKKOS_INLINE_FUNCTION
92 T atomic_add_fetch (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_add_fetch (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
94 template<
class T> KOKKOS_INLINE_FUNCTION
95 T atomic_sub_fetch (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_sub_fetch (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
97 template<
class T> KOKKOS_INLINE_FUNCTION
98 T atomic_max_fetch (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_max_fetch (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
100 template<
class T> KOKKOS_INLINE_FUNCTION
101 T atomic_min_fetch (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_min_fetch (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
103 template<
class T> KOKKOS_INLINE_FUNCTION
104 T atomic_mul_fetch (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_mul_fetch (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
106 template<
class T> KOKKOS_INLINE_FUNCTION
107 T atomic_div_fetch (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_div_fetch (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
109 template<
class T> KOKKOS_INLINE_FUNCTION
110 T atomic_mod_fetch (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_mod_fetch (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
112 template<
class T> KOKKOS_INLINE_FUNCTION
113 T atomic_and_fetch (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_and_fetch (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
115 template<
class T> KOKKOS_INLINE_FUNCTION
116 T atomic_or_fetch (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_or_fetch (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
118 template<
class T> KOKKOS_INLINE_FUNCTION
119 T atomic_xor_fetch (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_xor_fetch (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
121 template<
class T> KOKKOS_INLINE_FUNCTION
122 T atomic_nand_fetch(
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_nand_fetch(const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
124 template<
class T> KOKKOS_INLINE_FUNCTION
125 T atomic_lshift_fetch(
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_lshift_fetch(const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
127 template<
class T> KOKKOS_INLINE_FUNCTION
128 T atomic_rshift_fetch(
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_rshift_fetch(const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
130 template<
class T> KOKKOS_INLINE_FUNCTION
131 T atomic_inc_fetch(
volatile T*
const dest) {
return desul::atomic_inc_fetch(const_cast<T*>(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
133 template<
class T> KOKKOS_INLINE_FUNCTION
134 T atomic_dec_fetch(
volatile T*
const dest) {
return desul::atomic_dec_fetch(const_cast<T*>(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
138 template<
class T> KOKKOS_INLINE_FUNCTION
139 void atomic_add(
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_add (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
141 template<
class T> KOKKOS_INLINE_FUNCTION
142 void atomic_sub(
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_sub (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
144 template<
class T> KOKKOS_INLINE_FUNCTION
145 void atomic_mul(
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_mul (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
147 template<
class T> KOKKOS_INLINE_FUNCTION
148 void atomic_div(
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_div (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
150 template<
class T> KOKKOS_INLINE_FUNCTION
151 void atomic_min(
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_min (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
153 template<
class T> KOKKOS_INLINE_FUNCTION
154 void atomic_max(
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_max (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
157 template<
class T> KOKKOS_INLINE_FUNCTION
158 void atomic_and(
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { (void) desul::atomic_fetch_and (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
161 template<
class T> KOKKOS_INLINE_FUNCTION
162 void atomic_or (
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { (void) desul::atomic_fetch_or (const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
164 template<
class T> KOKKOS_INLINE_FUNCTION
165 void atomic_inc(
volatile T*
const dest) {
return desul::atomic_inc(const_cast<T*>(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
167 template<
class T> KOKKOS_INLINE_FUNCTION
168 void atomic_dec(
volatile T*
const dest) {
return desul::atomic_dec(const_cast<T*>(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
170 template<
class T> KOKKOS_INLINE_FUNCTION
171 void atomic_increment(
volatile T*
const dest) {
return desul::atomic_inc(const_cast<T*>(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
173 template<
class T> KOKKOS_INLINE_FUNCTION
174 void atomic_decrement(
volatile T*
const dest) {
return desul::atomic_dec(const_cast<T*>(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
178 template<
class T> KOKKOS_INLINE_FUNCTION
179 T atomic_exchange(
volatile T*
const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) {
return desul::atomic_exchange(const_cast<T*>(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
181 template<
class T> KOKKOS_INLINE_FUNCTION
182 bool atomic_compare_exchange_strong(
volatile T*
const dest, T& expected,
const T desired) {
183 return desul::atomic_compare_exchange_strong(const_cast<T*>(dest),expected, desired,
184 desul::MemoryOrderRelaxed(), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE);
187 template<
class T> KOKKOS_INLINE_FUNCTION
188 T atomic_compare_exchange(
volatile T*
const dest,
const T compare,
const T desired) {
189 return desul::atomic_compare_exchange(const_cast<T*>(dest),compare, desired,
190 desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE);
194 #undef KOKKOS_DESUL_MEM_SCOPE