1// Copyright 2014 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// RISC-V's atomic operations have two bits, aq ("acquire") and rl ("release"),
6// which may be toggled on and off. Their precise semantics are defined in
7// section 6.3 of the specification, but the basic idea is as follows:
8//
9// - If neither aq nor rl is set, the CPU may reorder the atomic arbitrarily.
10// It guarantees only that it will execute atomically.
11//
12// - If aq is set, the CPU may move the instruction backward, but not forward.
13//
14// - If rl is set, the CPU may move the instruction forward, but not backward.
15//
16// - If both are set, the CPU may not reorder the instruction at all.
17//
18// These four modes correspond to other well-known memory models on other CPUs.
19// On ARM, aq corresponds to a dmb ishst, aq+rl corresponds to a dmb ish. On
20// Intel, aq corresponds to an lfence, rl to an sfence, and aq+rl to an mfence
21// (or a lock prefix).
22//
23// Go's memory model requires that
24// - if a read happens after a write, the read must observe the write, and
25// that
26// - if a read happens concurrently with a write, the read may observe the
27// write.
28// aq is sufficient to guarantee this, so that's what we use here. (This jibes
29// with ARM, which uses dmb ishst.)
30
31#include "textflag.h"
32
33// func Cas(ptr *uint64, old, new uint64) bool
34// Atomically:
35// if *ptr == old {
36// *ptr = new
37// return true
38// } else {
39// return false
40// }
41TEXT ·Cas(SB), NOSPLIT, $0-17
42 MOV ptr+0(FP), A0
43 MOVW old+8(FP), A1
44 MOVW new+12(FP), A2
45cas_again:
46 LRW (A0), A3
47 BNE A3, A1, cas_fail
48 SCW A2, (A0), A4
49 BNE A4, ZERO, cas_again
50 MOV $1, A0
51 MOVB A0, ret+16(FP)
52 RET
53cas_fail:
54 MOV $0, A0
55 MOV A0, ret+16(FP)
56 RET
57
58// func Cas64(ptr *uint64, old, new uint64) bool
59TEXT ·Cas64(SB), NOSPLIT, $0-25
60 MOV ptr+0(FP), A0
61 MOV old+8(FP), A1
62 MOV new+16(FP), A2
63cas_again:
64 LRD (A0), A3
65 BNE A3, A1, cas_fail
66 SCD A2, (A0), A4
67 BNE A4, ZERO, cas_again
68 MOV $1, A0
69 MOVB A0, ret+24(FP)
70 RET
71cas_fail:
72 MOVB ZERO, ret+24(FP)
73 RET
74
75// func Load(ptr *uint32) uint32
76TEXT ·Load(SB),NOSPLIT|NOFRAME,$0-12
77 MOV ptr+0(FP), A0
78 LRW (A0), A0
79 MOVW A0, ret+8(FP)
80 RET
81
82// func Load8(ptr *uint8) uint8
83TEXT ·Load8(SB),NOSPLIT|NOFRAME,$0-9
84 MOV ptr+0(FP), A0
85 FENCE
86 MOVBU (A0), A1
87 FENCE
88 MOVB A1, ret+8(FP)
89 RET
90
91// func Load64(ptr *uint64) uint64
92TEXT ·Load64(SB),NOSPLIT|NOFRAME,$0-16
93 MOV ptr+0(FP), A0
94 LRD (A0), A0
95 MOV A0, ret+8(FP)
96 RET
97
98// func Store(ptr *uint32, val uint32)
99TEXT ·Store(SB), NOSPLIT, $0-12
100 MOV ptr+0(FP), A0
101 MOVW val+8(FP), A1
102 AMOSWAPW A1, (A0), ZERO
103 RET
104
105// func Store8(ptr *uint8, val uint8)
106TEXT ·Store8(SB), NOSPLIT, $0-9
107 MOV ptr+0(FP), A0
108 MOVBU val+8(FP), A1
109 FENCE
110 MOVB A1, (A0)
111 FENCE
112 RET
113
114// func Store64(ptr *uint64, val uint64)
115TEXT ·Store64(SB), NOSPLIT, $0-16
116 MOV ptr+0(FP), A0
117 MOV val+8(FP), A1
118 AMOSWAPD A1, (A0), ZERO
119 RET
120
121TEXT ·Casp1(SB), NOSPLIT, $0-25
122 JMP ·Cas64(SB)
123
124TEXT ·Casint32(SB),NOSPLIT,$0-17
125 JMP ·Cas(SB)
126
127TEXT ·Casint64(SB),NOSPLIT,$0-25
128 JMP ·Cas64(SB)
129
130TEXT ·Casuintptr(SB),NOSPLIT,$0-25
131 JMP ·Cas64(SB)
132
133TEXT ·CasRel(SB), NOSPLIT, $0-17
134 JMP ·Cas(SB)
135
136TEXT ·Loaduintptr(SB),NOSPLIT,$0-16
137 JMP ·Load64(SB)
138
139TEXT ·Storeint32(SB),NOSPLIT,$0-12
140 JMP ·Store(SB)
141
142TEXT ·Storeint64(SB),NOSPLIT,$0-16
143 JMP ·Store64(SB)
144
145TEXT ·Storeuintptr(SB),NOSPLIT,$0-16
146 JMP ·Store64(SB)
147
148TEXT ·Loaduint(SB),NOSPLIT,$0-16
149 JMP ·Loaduintptr(SB)
150
151TEXT ·Loadint32(SB),NOSPLIT,$0-12
152 JMP ·Load(SB)
153
154TEXT ·Loadint64(SB),NOSPLIT,$0-16
155 JMP ·Load64(SB)
156
157TEXT ·Xaddint32(SB),NOSPLIT,$0-20
158 JMP ·Xadd(SB)
159
160TEXT ·Xaddint64(SB),NOSPLIT,$0-24
161 MOV ptr+0(FP), A0
162 MOV delta+8(FP), A1
163 AMOADDD A1, (A0), A0
164 ADD A0, A1, A0
165 MOVW A0, ret+16(FP)
166 RET
167
168TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-12
169 JMP ·Load(SB)
170
171TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$0-16
172 JMP ·Load64(SB)
173
174TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-16
175 JMP ·Load64(SB)
176
177// func Loadp(ptr unsafe.Pointer) unsafe.Pointer
178TEXT ·Loadp(SB),NOSPLIT,$0-16
179 JMP ·Load64(SB)
180
181// func StorepNoWB(ptr unsafe.Pointer, val unsafe.Pointer)
182TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
183 JMP ·Store64(SB)
184
185TEXT ·StoreRel(SB), NOSPLIT, $0-12
186 JMP ·Store(SB)
187
188TEXT ·StoreRel64(SB), NOSPLIT, $0-16
189 JMP ·Store64(SB)
190
191TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
192 JMP ·Store64(SB)
193
194// func Xchg(ptr *uint32, new uint32) uint32
195TEXT ·Xchg(SB), NOSPLIT, $0-20
196 MOV ptr+0(FP), A0
197 MOVW new+8(FP), A1
198 AMOSWAPW A1, (A0), A1
199 MOVW A1, ret+16(FP)
200 RET
201
202// func Xchg8(ptr *uint8, new uint8) uint8
203TEXT ·Xchg8(SB), NOSPLIT, $0-17
204 MOV ptr+0(FP), A0
205 MOVBU new+8(FP), A1
206 AND $3, A0, A2
207 SLL $3, A2
208 MOV $255, A4
209 SLL A2, A4
210 NOT A4
211 AND $~3, A0
212 SLL A2, A1
213xchg8_again:
214 LRW (A0), A5
215 AND A4, A5, A3
216 OR A1, A3
217 SCW A3, (A0), A6
218 BNEZ A6, xchg8_again
219 SRL A2, A5
220 MOVB A5, ret+16(FP)
221 RET
222
223// func Xchg64(ptr *uint64, new uint64) uint64
224TEXT ·Xchg64(SB), NOSPLIT, $0-24
225 MOV ptr+0(FP), A0
226 MOV new+8(FP), A1
227 AMOSWAPD A1, (A0), A1
228 MOV A1, ret+16(FP)
229 RET
230
231// Atomically:
232// *val += delta;
233// return *val;
234
235// func Xadd(ptr *uint32, delta int32) uint32
236TEXT ·Xadd(SB), NOSPLIT, $0-20
237 MOV ptr+0(FP), A0
238 MOVW delta+8(FP), A1
239 AMOADDW A1, (A0), A2
240 ADD A2,A1,A0
241 MOVW A0, ret+16(FP)
242 RET
243
244// func Xadd64(ptr *uint64, delta int64) uint64
245TEXT ·Xadd64(SB), NOSPLIT, $0-24
246 MOV ptr+0(FP), A0
247 MOV delta+8(FP), A1
248 AMOADDD A1, (A0), A2
249 ADD A2, A1, A0
250 MOV A0, ret+16(FP)
251 RET
252
253// func Xadduintptr(ptr *uintptr, delta uintptr) uintptr
254TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
255 JMP ·Xadd64(SB)
256
257// func Xchgint32(ptr *int32, new int32) int32
258TEXT ·Xchgint32(SB), NOSPLIT, $0-20
259 JMP ·Xchg(SB)
260
261// func Xchgint64(ptr *int64, new int64) int64
262TEXT ·Xchgint64(SB), NOSPLIT, $0-24
263 JMP ·Xchg64(SB)
264
265// func Xchguintptr(ptr *uintptr, new uintptr) uintptr
266TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
267 JMP ·Xchg64(SB)
268
269// func And8(ptr *uint8, val uint8)
270TEXT ·And8(SB), NOSPLIT, $0-9
271 MOV ptr+0(FP), A0
272 MOVBU val+8(FP), A1
273 AND $3, A0, A2
274 AND $-4, A0
275 SLL $3, A2
276 XOR $255, A1
277 SLL A2, A1
278 XOR $-1, A1
279 AMOANDW A1, (A0), ZERO
280 RET
281
282// func Or8(ptr *uint8, val uint8)
283TEXT ·Or8(SB), NOSPLIT, $0-9
284 MOV ptr+0(FP), A0
285 MOVBU val+8(FP), A1
286 AND $3, A0, A2
287 AND $-4, A0
288 SLL $3, A2
289 SLL A2, A1
290 AMOORW A1, (A0), ZERO
291 RET
292
293// func And(ptr *uint32, val uint32)
294TEXT ·And(SB), NOSPLIT, $0-12
295 MOV ptr+0(FP), A0
296 MOVW val+8(FP), A1
297 AMOANDW A1, (A0), ZERO
298 RET
299
300// func Or(ptr *uint32, val uint32)
301TEXT ·Or(SB), NOSPLIT, $0-12
302 MOV ptr+0(FP), A0
303 MOVW val+8(FP), A1
304 AMOORW A1, (A0), ZERO
305 RET
306
307// func Or32(ptr *uint32, val uint32) uint32
308TEXT ·Or32(SB), NOSPLIT, $0-20
309 MOV ptr+0(FP), A0
310 MOVW val+8(FP), A1
311 AMOORW A1, (A0), A2
312 MOVW A2, ret+16(FP)
313 RET
314
315// func And32(ptr *uint32, val uint32) uint32
316TEXT ·And32(SB), NOSPLIT, $0-20
317 MOV ptr+0(FP), A0
318 MOVW val+8(FP), A1
319 AMOANDW A1, (A0), A2
320 MOVW A2, ret+16(FP)
321 RET
322
323// func Or64(ptr *uint64, val uint64) uint64
324TEXT ·Or64(SB), NOSPLIT, $0-24
325 MOV ptr+0(FP), A0
326 MOV val+8(FP), A1
327 AMOORD A1, (A0), A2
328 MOV A2, ret+16(FP)
329 RET
330
331// func And64(ptr *uint64, val uint64) uint64
332TEXT ·And64(SB), NOSPLIT, $0-24
333 MOV ptr+0(FP), A0
334 MOV val+8(FP), A1
335 AMOANDD A1, (A0), A2
336 MOV A2, ret+16(FP)
337 RET
338
339// func Anduintptr(ptr *uintptr, val uintptr) uintptr
340TEXT ·Anduintptr(SB), NOSPLIT, $0-24
341 JMP ·And64(SB)
342
343// func Oruintptr(ptr *uintptr, val uintptr) uintptr
344TEXT ·Oruintptr(SB), NOSPLIT, $0-24
345 JMP ·Or64(SB)
View as plain text