1// Copyright 2014 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5//go:build ppc64 || ppc64le
6
7#include "textflag.h"
8
9// For more details about how various memory models are
10// enforced on POWER, the following paper provides more
11// details about how they enforce C/C++ like models. This
12// gives context about why the strange looking code
13// sequences below work.
14//
15// http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html
16
17// uint32 ·Load(uint32 volatile* ptr)
18TEXT ·Load(SB),NOSPLIT|NOFRAME,$-8-12
19 MOVD ptr+0(FP), R3
20 SYNC
21 MOVWZ 0(R3), R3
22 CMPW R3, R3, CR7
23 BC 4, 30, 1(PC) // bne- cr7,0x4
24 ISYNC
25 MOVW R3, ret+8(FP)
26 RET
27
28// uint8 ·Load8(uint8 volatile* ptr)
29TEXT ·Load8(SB),NOSPLIT|NOFRAME,$-8-9
30 MOVD ptr+0(FP), R3
31 SYNC
32 MOVBZ 0(R3), R3
33 CMP R3, R3, CR7
34 BC 4, 30, 1(PC) // bne- cr7,0x4
35 ISYNC
36 MOVB R3, ret+8(FP)
37 RET
38
39// uint64 ·Load64(uint64 volatile* ptr)
40TEXT ·Load64(SB),NOSPLIT|NOFRAME,$-8-16
41 MOVD ptr+0(FP), R3
42 SYNC
43 MOVD 0(R3), R3
44 CMP R3, R3, CR7
45 BC 4, 30, 1(PC) // bne- cr7,0x4
46 ISYNC
47 MOVD R3, ret+8(FP)
48 RET
49
50// void *·Loadp(void *volatile *ptr)
51TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$-8-16
52 MOVD ptr+0(FP), R3
53 SYNC
54 MOVD 0(R3), R3
55 CMP R3, R3, CR7
56 BC 4, 30, 1(PC) // bne- cr7,0x4
57 ISYNC
58 MOVD R3, ret+8(FP)
59 RET
60
61// uint32 ·LoadAcq(uint32 volatile* ptr)
62TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$-8-12
63 MOVD ptr+0(FP), R3
64 MOVWZ 0(R3), R3
65 CMPW R3, R3, CR7
66 BC 4, 30, 1(PC) // bne- cr7, 0x4
67 ISYNC
68 MOVW R3, ret+8(FP)
69 RET
70
71// uint64 ·LoadAcq64(uint64 volatile* ptr)
72TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$-8-16
73 MOVD ptr+0(FP), R3
74 MOVD 0(R3), R3
75 CMP R3, R3, CR7
76 BC 4, 30, 1(PC) // bne- cr7, 0x4
77 ISYNC
78 MOVD R3, ret+8(FP)
79 RET
80
81// func Cas(ptr *int32, old, new int32) bool
82// Atomically:
83// if *ptr == old {
84// *ptr = new
85// return true
86// } else {
87// return false
88// }
89TEXT ·Cas(SB), NOSPLIT, $0-17
90 MOVD ptr+0(FP), R3
91 MOVWZ old+8(FP), R4
92 MOVWZ new+12(FP), R5
93 LWSYNC
94cas_again:
95 LWAR (R3), R6
96 CMPW R6, R4
97 BNE cas_fail
98 STWCCC R5, (R3)
99 BNE cas_again
100 MOVD $1, R3
101 LWSYNC
102 MOVB R3, ret+16(FP)
103 RET
104cas_fail:
105 LWSYNC
106 MOVB R0, ret+16(FP)
107 RET
108
109// func Cas64(ptr *uint64, old, new uint64) bool
110// Atomically:
111// if *ptr == old {
112// *ptr = new
113// return true
114// } else {
115// return false
116// }
117TEXT ·Cas64(SB), NOSPLIT, $0-25
118 MOVD ptr+0(FP), R3
119 MOVD old+8(FP), R4
120 MOVD new+16(FP), R5
121 LWSYNC
122cas64_again:
123 LDAR (R3), R6
124 CMP R6, R4
125 BNE cas64_fail
126 STDCCC R5, (R3)
127 BNE cas64_again
128 MOVD $1, R3
129 LWSYNC
130 MOVB R3, ret+24(FP)
131 RET
132cas64_fail:
133 LWSYNC
134 MOVB R0, ret+24(FP)
135 RET
136
137TEXT ·CasRel(SB), NOSPLIT, $0-17
138 MOVD ptr+0(FP), R3
139 MOVWZ old+8(FP), R4
140 MOVWZ new+12(FP), R5
141 LWSYNC
142cas_again:
143 LWAR (R3), $0, R6 // 0 = Mutex release hint
144 CMPW R6, R4
145 BNE cas_fail
146 STWCCC R5, (R3)
147 BNE cas_again
148 MOVD $1, R3
149 MOVB R3, ret+16(FP)
150 RET
151cas_fail:
152 MOVB R0, ret+16(FP)
153 RET
154
155TEXT ·Casint32(SB), NOSPLIT, $0-17
156 BR ·Cas(SB)
157
158TEXT ·Casint64(SB), NOSPLIT, $0-25
159 BR ·Cas64(SB)
160
161TEXT ·Casuintptr(SB), NOSPLIT, $0-25
162 BR ·Cas64(SB)
163
164TEXT ·Loaduintptr(SB), NOSPLIT|NOFRAME, $0-16
165 BR ·Load64(SB)
166
167TEXT ·LoadAcquintptr(SB), NOSPLIT|NOFRAME, $0-16
168 BR ·LoadAcq64(SB)
169
170TEXT ·Loaduint(SB), NOSPLIT|NOFRAME, $0-16
171 BR ·Load64(SB)
172
173TEXT ·Storeint32(SB), NOSPLIT, $0-12
174 BR ·Store(SB)
175
176TEXT ·Storeint64(SB), NOSPLIT, $0-16
177 BR ·Store64(SB)
178
179TEXT ·Storeuintptr(SB), NOSPLIT, $0-16
180 BR ·Store64(SB)
181
182TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
183 BR ·StoreRel64(SB)
184
185TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
186 BR ·Xadd64(SB)
187
188TEXT ·Loadint32(SB), NOSPLIT, $0-12
189 BR ·Load(SB)
190
191TEXT ·Loadint64(SB), NOSPLIT, $0-16
192 BR ·Load64(SB)
193
194TEXT ·Xaddint32(SB), NOSPLIT, $0-20
195 BR ·Xadd(SB)
196
197TEXT ·Xaddint64(SB), NOSPLIT, $0-24
198 BR ·Xadd64(SB)
199
200// func Casp1(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool
201// Atomically:
202// if *ptr == old {
203// *ptr = new
204// return true
205// } else {
206// return false
207// }
208TEXT ·Casp1(SB), NOSPLIT, $0-25
209 BR ·Cas64(SB)
210
211// uint32 xadd(uint32 volatile *ptr, int32 delta)
212// Atomically:
213// *val += delta;
214// return *val;
215TEXT ·Xadd(SB), NOSPLIT, $0-20
216 MOVD ptr+0(FP), R4
217 MOVW delta+8(FP), R5
218 LWSYNC
219 LWAR (R4), R3
220 ADD R5, R3
221 STWCCC R3, (R4)
222 BNE -3(PC)
223 MOVW R3, ret+16(FP)
224 RET
225
226// uint64 Xadd64(uint64 volatile *val, int64 delta)
227// Atomically:
228// *val += delta;
229// return *val;
230TEXT ·Xadd64(SB), NOSPLIT, $0-24
231 MOVD ptr+0(FP), R4
232 MOVD delta+8(FP), R5
233 LWSYNC
234 LDAR (R4), R3
235 ADD R5, R3
236 STDCCC R3, (R4)
237 BNE -3(PC)
238 MOVD R3, ret+16(FP)
239 RET
240
241// uint8 Xchg(ptr *uint8, new uint8)
242// Atomically:
243// old := *ptr;
244// *ptr = new;
245// return old;
246TEXT ·Xchg8(SB), NOSPLIT, $0-17
247 MOVD ptr+0(FP), R4
248 MOVB new+8(FP), R5
249 LWSYNC
250 LBAR (R4), R3
251 STBCCC R5, (R4)
252 BNE -2(PC)
253 ISYNC
254 MOVB R3, ret+16(FP)
255 RET
256
257// uint32 Xchg(ptr *uint32, new uint32)
258// Atomically:
259// old := *ptr;
260// *ptr = new;
261// return old;
262TEXT ·Xchg(SB), NOSPLIT, $0-20
263 MOVD ptr+0(FP), R4
264 MOVW new+8(FP), R5
265 LWSYNC
266 LWAR (R4), R3
267 STWCCC R5, (R4)
268 BNE -2(PC)
269 ISYNC
270 MOVW R3, ret+16(FP)
271 RET
272
273// uint64 Xchg64(ptr *uint64, new uint64)
274// Atomically:
275// old := *ptr;
276// *ptr = new;
277// return old;
278TEXT ·Xchg64(SB), NOSPLIT, $0-24
279 MOVD ptr+0(FP), R4
280 MOVD new+8(FP), R5
281 LWSYNC
282 LDAR (R4), R3
283 STDCCC R5, (R4)
284 BNE -2(PC)
285 ISYNC
286 MOVD R3, ret+16(FP)
287 RET
288
289TEXT ·Xchgint32(SB), NOSPLIT, $0-20
290 BR ·Xchg(SB)
291
292TEXT ·Xchgint64(SB), NOSPLIT, $0-24
293 BR ·Xchg64(SB)
294
295TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
296 BR ·Xchg64(SB)
297
298TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
299 BR ·Store64(SB)
300
301TEXT ·Store(SB), NOSPLIT, $0-12
302 MOVD ptr+0(FP), R3
303 MOVW val+8(FP), R4
304 SYNC
305 MOVW R4, 0(R3)
306 RET
307
308TEXT ·Store8(SB), NOSPLIT, $0-9
309 MOVD ptr+0(FP), R3
310 MOVB val+8(FP), R4
311 SYNC
312 MOVB R4, 0(R3)
313 RET
314
315TEXT ·Store64(SB), NOSPLIT, $0-16
316 MOVD ptr+0(FP), R3
317 MOVD val+8(FP), R4
318 SYNC
319 MOVD R4, 0(R3)
320 RET
321
322TEXT ·StoreRel(SB), NOSPLIT, $0-12
323 MOVD ptr+0(FP), R3
324 MOVW val+8(FP), R4
325 LWSYNC
326 MOVW R4, 0(R3)
327 RET
328
329TEXT ·StoreRel64(SB), NOSPLIT, $0-16
330 MOVD ptr+0(FP), R3
331 MOVD val+8(FP), R4
332 LWSYNC
333 MOVD R4, 0(R3)
334 RET
335
336// void ·Or8(byte volatile*, byte);
337TEXT ·Or8(SB), NOSPLIT, $0-9
338 MOVD ptr+0(FP), R3
339 MOVBZ val+8(FP), R4
340 LWSYNC
341again:
342 LBAR (R3), R6
343 OR R4, R6
344 STBCCC R6, (R3)
345 BNE again
346 RET
347
348// void ·And8(byte volatile*, byte);
349TEXT ·And8(SB), NOSPLIT, $0-9
350 MOVD ptr+0(FP), R3
351 MOVBZ val+8(FP), R4
352 LWSYNC
353again:
354 LBAR (R3), R6
355 AND R4, R6
356 STBCCC R6, (R3)
357 BNE again
358 RET
359
360// func Or(addr *uint32, v uint32)
361TEXT ·Or(SB), NOSPLIT, $0-12
362 MOVD ptr+0(FP), R3
363 MOVW val+8(FP), R4
364 LWSYNC
365again:
366 LWAR (R3), R6
367 OR R4, R6
368 STWCCC R6, (R3)
369 BNE again
370 RET
371
372// func And(addr *uint32, v uint32)
373TEXT ·And(SB), NOSPLIT, $0-12
374 MOVD ptr+0(FP), R3
375 MOVW val+8(FP), R4
376 LWSYNC
377again:
378 LWAR (R3),R6
379 AND R4, R6
380 STWCCC R6, (R3)
381 BNE again
382 RET
383
384// func Or32(addr *uint32, v uint32) old uint32
385TEXT ·Or32(SB), NOSPLIT, $0-20
386 MOVD ptr+0(FP), R3
387 MOVW val+8(FP), R4
388 LWSYNC
389again:
390 LWAR (R3), R6
391 OR R4, R6, R7
392 STWCCC R7, (R3)
393 BNE again
394 MOVW R6, ret+16(FP)
395 RET
396
397// func And32(addr *uint32, v uint32) old uint32
398TEXT ·And32(SB), NOSPLIT, $0-20
399 MOVD ptr+0(FP), R3
400 MOVW val+8(FP), R4
401 LWSYNC
402again:
403 LWAR (R3),R6
404 AND R4, R6, R7
405 STWCCC R7, (R3)
406 BNE again
407 MOVW R6, ret+16(FP)
408 RET
409
410// func Or64(addr *uint64, v uint64) old uint64
411TEXT ·Or64(SB), NOSPLIT, $0-24
412 MOVD ptr+0(FP), R3
413 MOVD val+8(FP), R4
414 LWSYNC
415again:
416 LDAR (R3), R6
417 OR R4, R6, R7
418 STDCCC R7, (R3)
419 BNE again
420 MOVD R6, ret+16(FP)
421 RET
422
423// func And64(addr *uint64, v uint64) old uint64
424TEXT ·And64(SB), NOSPLIT, $0-24
425 MOVD ptr+0(FP), R3
426 MOVD val+8(FP), R4
427 LWSYNC
428again:
429 LDAR (R3),R6
430 AND R4, R6, R7
431 STDCCC R7, (R3)
432 BNE again
433 MOVD R6, ret+16(FP)
434 RET
435
436// func Anduintptr(addr *uintptr, v uintptr) old uintptr
437TEXT ·Anduintptr(SB), NOSPLIT, $0-24
438 JMP ·And64(SB)
439
440// func Oruintptr(addr *uintptr, v uintptr) old uintptr
441TEXT ·Oruintptr(SB), NOSPLIT, $0-24
442 JMP ·Or64(SB)
View as plain text