Kokkos Core Kernels Package Version of the Day
Loading...
Searching...
No Matches
Kokkos_Atomics_Desul_Wrapper.hpp
1//@HEADER
2// ************************************************************************
3//
4// Kokkos v. 4.0
5// Copyright (2022) National Technology & Engineering
6// Solutions of Sandia, LLC (NTESS).
7//
8// Under the terms of Contract DE-NA0003525 with NTESS,
9// the U.S. Government retains certain rights in this software.
10//
11// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12// See https://kokkos.org/LICENSE for license information.
13// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14//
15//@HEADER
16
17#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
18#include <Kokkos_Macros.hpp>
19static_assert(false,
20 "Including non-public Kokkos header files is not allowed.");
21#endif
22#ifndef KOKKOS_DESUL_ATOMICS_WRAPPER_HPP_
23#define KOKKOS_DESUL_ATOMICS_WRAPPER_HPP_
24#include <Kokkos_Macros.hpp>
25
26#include <Kokkos_Atomics_Desul_Config.hpp>
27#include <desul/atomics.hpp>
28
29#include <impl/Kokkos_Volatile_Load.hpp>
30
31// clang-format off
32namespace Kokkos {
33
34// FIXME: These functions don't have any use/test in unit tests ...
35// ==========================================================
36inline const char* atomic_query_version() { return "KOKKOS_DESUL_ATOMICS"; }
37
38#if defined(KOKKOS_COMPILER_GNU) && !defined(__PGIC__) && \
39 !defined(__CUDA_ARCH__)
40
41#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr, 0, 0)
42#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr, 1, 0)
43
44#else
45
46#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0)
47#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0)
48
49#endif
50// ============================================================
51
52#ifdef KOKKOS_INTERNAL_NOT_PARALLEL
53#define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeCaller()
54#else
55#define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeDevice()
56#endif
57
58template<class T> KOKKOS_INLINE_FUNCTION
59T atomic_load(T* const dest) { return desul::atomic_load(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
60
61template<class T> KOKKOS_INLINE_FUNCTION
62void atomic_store(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_store(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
63
64template<class T> KOKKOS_INLINE_FUNCTION
65void atomic_assign(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { atomic_store(dest,val); }
66
67KOKKOS_INLINE_FUNCTION
68void memory_fence() {
69 desul::atomic_thread_fence(desul::MemoryOrderSeqCst(), KOKKOS_DESUL_MEM_SCOPE);
70}
71
72KOKKOS_INLINE_FUNCTION
73void load_fence() { return desul::atomic_thread_fence(desul::MemoryOrderAcquire(), KOKKOS_DESUL_MEM_SCOPE); }
74
75KOKKOS_INLINE_FUNCTION
76void store_fence() { return desul::atomic_thread_fence(desul::MemoryOrderRelease(), KOKKOS_DESUL_MEM_SCOPE); }
77
78// atomic_fetch_op
79template<class T> KOKKOS_INLINE_FUNCTION
80T atomic_fetch_add (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
81
82template<class T> KOKKOS_INLINE_FUNCTION
83T atomic_fetch_sub (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
84
85template<class T> KOKKOS_INLINE_FUNCTION
86T atomic_fetch_max (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_max (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
87
88template<class T> KOKKOS_INLINE_FUNCTION
89T atomic_fetch_min (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_min (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
90
91template<class T> KOKKOS_INLINE_FUNCTION
92T atomic_fetch_mul (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_mul (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
93
94template<class T> KOKKOS_INLINE_FUNCTION
95T atomic_fetch_div (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_div (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
96
97template<class T> KOKKOS_INLINE_FUNCTION
98T atomic_fetch_mod (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_mod (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
99
100template<class T> KOKKOS_INLINE_FUNCTION
101T atomic_fetch_and (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_and (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
102
103template<class T> KOKKOS_INLINE_FUNCTION
104T atomic_fetch_or (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_or (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
105
106template<class T> KOKKOS_INLINE_FUNCTION
107T atomic_fetch_xor (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_xor (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
108
109template<class T> KOKKOS_INLINE_FUNCTION
110T atomic_fetch_nand(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_nand(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
111
112template<class T> KOKKOS_INLINE_FUNCTION
113T atomic_fetch_lshift(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_lshift(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
114
115template<class T> KOKKOS_INLINE_FUNCTION
116T atomic_fetch_rshift(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_rshift(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
117
118template<class T> KOKKOS_INLINE_FUNCTION
119T atomic_fetch_inc(T* const dest) { return desul::atomic_fetch_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
120
121template<class T> KOKKOS_INLINE_FUNCTION
122T atomic_fetch_dec(T* const dest) { return desul::atomic_fetch_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
123
124
125// atomic_op_fetch
126template<class T> KOKKOS_INLINE_FUNCTION
127T atomic_add_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_add_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
128
129template<class T> KOKKOS_INLINE_FUNCTION
130T atomic_sub_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_sub_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
131
132template<class T> KOKKOS_INLINE_FUNCTION
133T atomic_max_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_max_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
134
135template<class T> KOKKOS_INLINE_FUNCTION
136T atomic_min_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_min_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
137
138template<class T> KOKKOS_INLINE_FUNCTION
139T atomic_mul_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_mul_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
140
141template<class T> KOKKOS_INLINE_FUNCTION
142T atomic_div_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_div_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
143
144template<class T> KOKKOS_INLINE_FUNCTION
145T atomic_mod_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_mod_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
146
147template<class T> KOKKOS_INLINE_FUNCTION
148T atomic_and_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_and_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
149
150template<class T> KOKKOS_INLINE_FUNCTION
151T atomic_or_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_or_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
152
153template<class T> KOKKOS_INLINE_FUNCTION
154T atomic_xor_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_xor_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
155
156template<class T> KOKKOS_INLINE_FUNCTION
157T atomic_nand_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_nand_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
158
159template<class T> KOKKOS_INLINE_FUNCTION
160T atomic_lshift_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_lshift_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
161
162template<class T> KOKKOS_INLINE_FUNCTION
163T atomic_rshift_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_rshift_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
164
165template<class T> KOKKOS_INLINE_FUNCTION
166T atomic_inc_fetch(T* const dest) { return desul::atomic_inc_fetch(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
167
168template<class T> KOKKOS_INLINE_FUNCTION
169T atomic_dec_fetch(T* const dest) { return desul::atomic_dec_fetch(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
170
171
172// atomic_op
173template<class T> KOKKOS_INLINE_FUNCTION
174void atomic_add(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
175
176template<class T> KOKKOS_INLINE_FUNCTION
177void atomic_sub(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
178
179template<class T> KOKKOS_INLINE_FUNCTION
180void atomic_mul(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_mul (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
181
182template<class T> KOKKOS_INLINE_FUNCTION
183void atomic_div(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_div (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
184
185template<class T> KOKKOS_INLINE_FUNCTION
186void atomic_min(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_min (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
187
188template<class T> KOKKOS_INLINE_FUNCTION
189void atomic_max(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_max (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
190
191// FIXME: Desul doesn't have atomic_and yet so call fetch_and
192template<class T> KOKKOS_INLINE_FUNCTION
193void atomic_and(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { (void) desul::atomic_fetch_and (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
194
195// FIXME: Desul doesn't have atomic_or yet so call fetch_or
196template<class T> KOKKOS_INLINE_FUNCTION
197void atomic_or(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { (void) desul::atomic_fetch_or (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
198
199template<class T> KOKKOS_INLINE_FUNCTION
200void atomic_inc(T* const dest) { return desul::atomic_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
201
202template<class T> KOKKOS_INLINE_FUNCTION
203void atomic_dec(T* const dest) { return desul::atomic_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
204
205template<class T> KOKKOS_INLINE_FUNCTION
206void atomic_increment(T* const dest) { return desul::atomic_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
207
208template<class T> KOKKOS_INLINE_FUNCTION
209void atomic_decrement(T* const dest) { return desul::atomic_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
210
211// Exchange
212
213template<class T> KOKKOS_INLINE_FUNCTION
214T atomic_exchange(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_exchange(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
215
216template<class T> KOKKOS_INLINE_FUNCTION
217bool atomic_compare_exchange_strong(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> expected, desul::Impl::dont_deduce_this_parameter_t<const T> desired) {
218 T expected_ref = expected;
219 return desul::atomic_compare_exchange_strong(dest, expected_ref, desired,
220 desul::MemoryOrderRelaxed(), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE);
221}
222
223template<class T> KOKKOS_INLINE_FUNCTION
224T atomic_compare_exchange(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> compare, desul::Impl::dont_deduce_this_parameter_t<const T> desired) {
225 return desul::atomic_compare_exchange(dest, compare, desired,
226 desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE);
227}
228
229namespace Impl {
230 template<class T, class MemOrderSuccess, class MemOrderFailure> KOKKOS_INLINE_FUNCTION
231 bool atomic_compare_exchange_strong(T* const dest, T& expected, const T desired, MemOrderSuccess succ, MemOrderFailure fail) {
232 return desul::atomic_compare_exchange_strong(dest, expected, desired, succ, fail, KOKKOS_DESUL_MEM_SCOPE);
233 }
234 template<class T, class MemoryOrder>
235 KOKKOS_INLINE_FUNCTION
236 T atomic_load(const T* const src, MemoryOrder order) {
237 return desul::atomic_load(src, order, KOKKOS_DESUL_MEM_SCOPE);
238 }
239 template<class T, class MemoryOrder>
240 KOKKOS_INLINE_FUNCTION
241 void atomic_store(T* const src, const T val, MemoryOrder order) {
242 return desul::atomic_store(src, val, order, KOKKOS_DESUL_MEM_SCOPE);
243 }
244} // namespace Impl
245
246} // namespace Kokkos
247
248#undef KOKKOS_DESUL_MEM_SCOPE
249
250// clang-format on
251#endif
Kokkos::Impl
ScopeGuard Some user scope issues have been identified with some Kokkos::finalize calls; ScopeGuard a...
Definition Kokkos_AnonymousSpace.hpp:61
Generated by doxygen 1.17.0