1// Copyright 2015 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "textflag.h"
7#include "funcdata.h"
8
9// func armcas(ptr *int32, old, new int32) bool
10// Atomically:
11// if *ptr == old {
12// *ptr = new
13// return true
14// } else {
15// return false
16// }
17//
18// To implement ·cas in sys_$GOOS_arm.s
19// using the native instructions, use:
20//
21// TEXT ·cas(SB),NOSPLIT,$0
22// B ·armcas(SB)
23//
24TEXT ·armcas(SB),NOSPLIT,$0-13
25 MOVW ptr+0(FP), R1
26 MOVW old+4(FP), R2
27 MOVW new+8(FP), R3
28casl:
29 LDREX (R1), R0
30 CMP R0, R2
31 BNE casfail
32
33#ifndef GOARM_7
34 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
35 CMP $0, R11
36 BEQ 2(PC)
37#endif
38 DMB MB_ISHST
39
40 STREX R3, (R1), R0
41 CMP $0, R0
42 BNE casl
43 MOVW $1, R0
44
45#ifndef GOARM_7
46 CMP $0, R11
47 BEQ 2(PC)
48#endif
49 DMB MB_ISH
50
51 MOVB R0, ret+12(FP)
52 RET
53casfail:
54 MOVW $0, R0
55 MOVB R0, ret+12(FP)
56 RET
57
58// stubs
59
60TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-8
61 B ·Load(SB)
62
63TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-8
64 B ·Load(SB)
65
66TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-8
67 B ·Load(SB)
68
69TEXT ·Casint32(SB),NOSPLIT,$0-13
70 B ·Cas(SB)
71
72TEXT ·Casint64(SB),NOSPLIT,$-4-21
73 B ·Cas64(SB)
74
75TEXT ·Casuintptr(SB),NOSPLIT,$0-13
76 B ·Cas(SB)
77
78TEXT ·Casp1(SB),NOSPLIT,$0-13
79 B ·Cas(SB)
80
81TEXT ·CasRel(SB),NOSPLIT,$0-13
82 B ·Cas(SB)
83
84TEXT ·Loadint32(SB),NOSPLIT,$0-8
85 B ·Load(SB)
86
87TEXT ·Loadint64(SB),NOSPLIT,$-4-12
88 B ·Load64(SB)
89
90TEXT ·Loaduintptr(SB),NOSPLIT,$0-8
91 B ·Load(SB)
92
93TEXT ·Loaduint(SB),NOSPLIT,$0-8
94 B ·Load(SB)
95
96TEXT ·Storeint32(SB),NOSPLIT,$0-8
97 B ·Store(SB)
98
99TEXT ·Storeint64(SB),NOSPLIT,$0-12
100 B ·Store64(SB)
101
102TEXT ·Storeuintptr(SB),NOSPLIT,$0-8
103 B ·Store(SB)
104
105TEXT ·StorepNoWB(SB),NOSPLIT,$0-8
106 B ·Store(SB)
107
108TEXT ·StoreRel(SB),NOSPLIT,$0-8
109 B ·Store(SB)
110
111TEXT ·StoreReluintptr(SB),NOSPLIT,$0-8
112 B ·Store(SB)
113
114TEXT ·Xaddint32(SB),NOSPLIT,$0-12
115 B ·Xadd(SB)
116
117TEXT ·Xaddint64(SB),NOSPLIT,$-4-20
118 B ·Xadd64(SB)
119
120TEXT ·Xadduintptr(SB),NOSPLIT,$0-12
121 B ·Xadd(SB)
122
123TEXT ·Xchgint32(SB),NOSPLIT,$0-12
124 B ·Xchg(SB)
125
126TEXT ·Xchgint64(SB),NOSPLIT,$-4-20
127 B ·Xchg64(SB)
128
129// 64-bit atomics
130// The native ARM implementations use LDREXD/STREXD, which are
131// available on ARMv6k or later. We use them only on ARMv7.
132// On older ARM, we use Go implementations which simulate 64-bit
133// atomics with locks.
134TEXT armCas64<>(SB),NOSPLIT,$0-21
135 // addr is already in R1
136 MOVW old_lo+4(FP), R2
137 MOVW old_hi+8(FP), R3
138 MOVW new_lo+12(FP), R4
139 MOVW new_hi+16(FP), R5
140cas64loop:
141 LDREXD (R1), R6 // loads R6 and R7
142 CMP R2, R6
143 BNE cas64fail
144 CMP R3, R7
145 BNE cas64fail
146
147 DMB MB_ISHST
148
149 STREXD R4, (R1), R0 // stores R4 and R5
150 CMP $0, R0
151 BNE cas64loop
152 MOVW $1, R0
153
154 DMB MB_ISH
155
156 MOVBU R0, swapped+20(FP)
157 RET
158cas64fail:
159 MOVW $0, R0
160 MOVBU R0, swapped+20(FP)
161 RET
162
163TEXT armXadd64<>(SB),NOSPLIT,$0-20
164 // addr is already in R1
165 MOVW delta_lo+4(FP), R2
166 MOVW delta_hi+8(FP), R3
167
168add64loop:
169 LDREXD (R1), R4 // loads R4 and R5
170 ADD.S R2, R4
171 ADC R3, R5
172
173 DMB MB_ISHST
174
175 STREXD R4, (R1), R0 // stores R4 and R5
176 CMP $0, R0
177 BNE add64loop
178
179 DMB MB_ISH
180
181 MOVW R4, new_lo+12(FP)
182 MOVW R5, new_hi+16(FP)
183 RET
184
185TEXT armXchg64<>(SB),NOSPLIT,$0-20
186 // addr is already in R1
187 MOVW new_lo+4(FP), R2
188 MOVW new_hi+8(FP), R3
189
190swap64loop:
191 LDREXD (R1), R4 // loads R4 and R5
192
193 DMB MB_ISHST
194
195 STREXD R2, (R1), R0 // stores R2 and R3
196 CMP $0, R0
197 BNE swap64loop
198
199 DMB MB_ISH
200
201 MOVW R4, old_lo+12(FP)
202 MOVW R5, old_hi+16(FP)
203 RET
204
205TEXT armLoad64<>(SB),NOSPLIT,$0-12
206 // addr is already in R1
207
208 LDREXD (R1), R2 // loads R2 and R3
209 DMB MB_ISH
210
211 MOVW R2, val_lo+4(FP)
212 MOVW R3, val_hi+8(FP)
213 RET
214
215TEXT armStore64<>(SB),NOSPLIT,$0-12
216 // addr is already in R1
217 MOVW val_lo+4(FP), R2
218 MOVW val_hi+8(FP), R3
219
220store64loop:
221 LDREXD (R1), R4 // loads R4 and R5
222
223 DMB MB_ISHST
224
225 STREXD R2, (R1), R0 // stores R2 and R3
226 CMP $0, R0
227 BNE store64loop
228
229 DMB MB_ISH
230 RET
231
232TEXT armAnd8<>(SB),NOSPLIT,$0-5
233 // addr is already in R1
234 MOVB v+4(FP), R2
235
236and8loop:
237 LDREXB (R1), R6
238
239 DMB MB_ISHST
240
241 AND R2, R6
242 STREXB R6, (R1), R0
243 CMP $0, R0
244 BNE and8loop
245
246 DMB MB_ISH
247
248 RET
249
250TEXT armOr8<>(SB),NOSPLIT,$0-5
251 // addr is already in R1
252 MOVB v+4(FP), R2
253
254or8loop:
255 LDREXB (R1), R6
256
257 DMB MB_ISHST
258
259 ORR R2, R6
260 STREXB R6, (R1), R0
261 CMP $0, R0
262 BNE or8loop
263
264 DMB MB_ISH
265
266 RET
267
268TEXT armXchg8<>(SB),NOSPLIT,$0-9
269 // addr is already in R1
270 MOVB v+4(FP), R2
271xchg8loop:
272 LDREXB (R1), R6
273
274 DMB MB_ISHST
275
276 STREXB R2, (R1), R0
277 CMP $0, R0
278 BNE xchg8loop
279
280 DMB MB_ISH
281
282 MOVB R6, ret+8(FP)
283 RET
284
285// The following functions all panic if their address argument isn't
286// 8-byte aligned. Since we're calling back into Go code to do this,
287// we have to cooperate with stack unwinding. In the normal case, the
288// functions tail-call into the appropriate implementation, which
289// means they must not open a frame. Hence, when they go down the
290// panic path, at that point they push the LR to create a real frame
291// (they don't need to pop it because panic won't return; however, we
292// do need to set the SP delta back).
293
294// Check if R1 is 8-byte aligned, panic if not.
295// Clobbers R2.
296#define CHECK_ALIGN \
297 AND.S $7, R1, R2 \
298 BEQ 4(PC) \
299 MOVW.W R14, -4(R13) /* prepare a real frame */ \
300 BL ·panicUnaligned(SB) \
301 ADD $4, R13 /* compensate SP delta */
302
303TEXT ·Cas64(SB),NOSPLIT,$-4-21
304 NO_LOCAL_POINTERS
305 MOVW addr+0(FP), R1
306 CHECK_ALIGN
307
308#ifndef GOARM_7
309 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
310 CMP $1, R11
311 BEQ 2(PC)
312 JMP ·goCas64(SB)
313#endif
314 JMP armCas64<>(SB)
315
316TEXT ·Xadd64(SB),NOSPLIT,$-4-20
317 NO_LOCAL_POINTERS
318 MOVW addr+0(FP), R1
319 CHECK_ALIGN
320
321#ifndef GOARM_7
322 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
323 CMP $1, R11
324 BEQ 2(PC)
325 JMP ·goXadd64(SB)
326#endif
327 JMP armXadd64<>(SB)
328
329TEXT ·Xchg64(SB),NOSPLIT,$-4-20
330 NO_LOCAL_POINTERS
331 MOVW addr+0(FP), R1
332 CHECK_ALIGN
333
334#ifndef GOARM_7
335 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
336 CMP $1, R11
337 BEQ 2(PC)
338 JMP ·goXchg64(SB)
339#endif
340 JMP armXchg64<>(SB)
341
342TEXT ·Load64(SB),NOSPLIT,$-4-12
343 NO_LOCAL_POINTERS
344 MOVW addr+0(FP), R1
345 CHECK_ALIGN
346
347#ifndef GOARM_7
348 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
349 CMP $1, R11
350 BEQ 2(PC)
351 JMP ·goLoad64(SB)
352#endif
353 JMP armLoad64<>(SB)
354
355TEXT ·Store64(SB),NOSPLIT,$-4-12
356 NO_LOCAL_POINTERS
357 MOVW addr+0(FP), R1
358 CHECK_ALIGN
359
360#ifndef GOARM_7
361 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
362 CMP $1, R11
363 BEQ 2(PC)
364 JMP ·goStore64(SB)
365#endif
366 JMP armStore64<>(SB)
367
368TEXT ·And8(SB),NOSPLIT,$-4-5
369 NO_LOCAL_POINTERS
370 MOVW addr+0(FP), R1
371
372// Uses STREXB/LDREXB that is armv6k or later.
373// For simplicity we only enable this on armv7.
374#ifndef GOARM_7
375 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
376 CMP $1, R11
377 BEQ 2(PC)
378 JMP ·goAnd8(SB)
379#endif
380 JMP armAnd8<>(SB)
381
382TEXT ·Or8(SB),NOSPLIT,$-4-5
383 NO_LOCAL_POINTERS
384 MOVW addr+0(FP), R1
385
386// Uses STREXB/LDREXB that is armv6k or later.
387// For simplicity we only enable this on armv7.
388#ifndef GOARM_7
389 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
390 CMP $1, R11
391 BEQ 2(PC)
392 JMP ·goOr8(SB)
393#endif
394 JMP armOr8<>(SB)
395
396TEXT ·Xchg8(SB),NOSPLIT,$-4-9
397 NO_LOCAL_POINTERS
398 MOVW addr+0(FP), R1
399
400 // Uses STREXB/LDREXB that is armv6k or later.
401 // For simplicity we only enable this on armv7.
402#ifndef GOARM_7
403 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
404 CMP $1, R11
405 BEQ 2(PC)
406 JMP ·goXchg8(SB)
407#endif
408 JMP armXchg8<>(SB)
View as plain text