Text file
src/runtime/asm_amd64.s
Documentation: runtime
1// Copyright 2009 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "go_tls.h"
7#include "funcdata.h"
8#include "textflag.h"
9#include "cgo/abi_amd64.h"
10
11// _rt0_amd64 is common startup code for most amd64 systems when using
12// internal linking. This is the entry point for the program from the
13// kernel for an ordinary -buildmode=exe program. The stack holds the
14// number of arguments and the C-style argv.
15TEXT _rt0_amd64(SB),NOSPLIT,$-8
16 MOVQ 0(SP), DI // argc
17 LEAQ 8(SP), SI // argv
18 JMP runtime·rt0_go(SB)
19
20// main is common startup code for most amd64 systems when using
21// external linking. The C startup code will call the symbol "main"
22// passing argc and argv in the usual C ABI registers DI and SI.
23TEXT main(SB),NOSPLIT,$-8
24 JMP runtime·rt0_go(SB)
25
26// _rt0_amd64_lib is common startup code for most amd64 systems when
27// using -buildmode=c-archive or -buildmode=c-shared. The linker will
28// arrange to invoke this function as a global constructor (for
29// c-archive) or when the shared library is loaded (for c-shared).
30// We expect argc and argv to be passed in the usual C ABI registers
31// DI and SI.
32TEXT _rt0_amd64_lib(SB),NOSPLIT|NOFRAME,$0
33 // Transition from C ABI to Go ABI.
34 PUSH_REGS_HOST_TO_ABI0()
35
36 MOVQ DI, _rt0_amd64_lib_argc<>(SB)
37 MOVQ SI, _rt0_amd64_lib_argv<>(SB)
38
39 // Synchronous initialization.
40 CALL runtime·libpreinit(SB)
41
42 // Create a new thread to finish Go runtime initialization.
43 MOVQ _cgo_sys_thread_create(SB), AX
44 TESTQ AX, AX
45 JZ nocgo
46
47 // We're calling back to C.
48 // Align stack per ELF ABI requirements.
49 MOVQ SP, BX // Callee-save in C ABI
50 ANDQ $~15, SP
51 MOVQ $_rt0_amd64_lib_go(SB), DI
52 MOVQ $0, SI
53 CALL AX
54 MOVQ BX, SP
55 JMP restore
56
57nocgo:
58 ADJSP $16
59 MOVQ $0x800000, 0(SP) // stacksize
60 MOVQ $_rt0_amd64_lib_go(SB), AX
61 MOVQ AX, 8(SP) // fn
62 CALL runtime·newosproc0(SB)
63 ADJSP $-16
64
65restore:
66 POP_REGS_HOST_TO_ABI0()
67 RET
68
69// _rt0_amd64_lib_go initializes the Go runtime.
70// This is started in a separate thread by _rt0_amd64_lib.
71TEXT _rt0_amd64_lib_go(SB),NOSPLIT,$0
72 MOVQ _rt0_amd64_lib_argc<>(SB), DI
73 MOVQ _rt0_amd64_lib_argv<>(SB), SI
74 JMP runtime·rt0_go(SB)
75
76DATA _rt0_amd64_lib_argc<>(SB)/8, $0
77GLOBL _rt0_amd64_lib_argc<>(SB),NOPTR, $8
78DATA _rt0_amd64_lib_argv<>(SB)/8, $0
79GLOBL _rt0_amd64_lib_argv<>(SB),NOPTR, $8
80
81#ifdef GOAMD64_v2
82DATA bad_cpu_msg<>+0x00(SB)/84, $"This program can only be run on AMD64 processors with v2 microarchitecture support.\n"
83#endif
84
85#ifdef GOAMD64_v3
86DATA bad_cpu_msg<>+0x00(SB)/84, $"This program can only be run on AMD64 processors with v3 microarchitecture support.\n"
87#endif
88
89#ifdef GOAMD64_v4
90DATA bad_cpu_msg<>+0x00(SB)/84, $"This program can only be run on AMD64 processors with v4 microarchitecture support.\n"
91#endif
92
93GLOBL bad_cpu_msg<>(SB), RODATA, $84
94
95// Define a list of AMD64 microarchitecture level features
96// https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels
97
98 // SSE3 SSSE3 CMPXCHNG16 SSE4.1 SSE4.2 POPCNT
99#define V2_FEATURES_CX (1 << 0 | 1 << 9 | 1 << 13 | 1 << 19 | 1 << 20 | 1 << 23)
100 // LAHF/SAHF
101#define V2_EXT_FEATURES_CX (1 << 0)
102 // FMA MOVBE OSXSAVE AVX F16C
103#define V3_FEATURES_CX (V2_FEATURES_CX | 1 << 12 | 1 << 22 | 1 << 27 | 1 << 28 | 1 << 29)
104 // ABM (FOR LZNCT)
105#define V3_EXT_FEATURES_CX (V2_EXT_FEATURES_CX | 1 << 5)
106 // BMI1 AVX2 BMI2
107#define V3_EXT_FEATURES_BX (1 << 3 | 1 << 5 | 1 << 8)
108 // XMM YMM
109#define V3_OS_SUPPORT_AX (1 << 1 | 1 << 2)
110
111#define V4_FEATURES_CX V3_FEATURES_CX
112
113#define V4_EXT_FEATURES_CX V3_EXT_FEATURES_CX
114 // AVX512F AVX512DQ AVX512CD AVX512BW AVX512VL
115#define V4_EXT_FEATURES_BX (V3_EXT_FEATURES_BX | 1 << 16 | 1 << 17 | 1 << 28 | 1 << 30 | 1 << 31)
116 // OPMASK ZMM
117#define V4_OS_SUPPORT_AX (V3_OS_SUPPORT_AX | 1 << 5 | (1 << 6 | 1 << 7))
118
119#ifdef GOAMD64_v2
120#define NEED_MAX_CPUID 0x80000001
121#define NEED_FEATURES_CX V2_FEATURES_CX
122#define NEED_EXT_FEATURES_CX V2_EXT_FEATURES_CX
123#endif
124
125#ifdef GOAMD64_v3
126#define NEED_MAX_CPUID 0x80000001
127#define NEED_FEATURES_CX V3_FEATURES_CX
128#define NEED_EXT_FEATURES_CX V3_EXT_FEATURES_CX
129#define NEED_EXT_FEATURES_BX V3_EXT_FEATURES_BX
130#define NEED_OS_SUPPORT_AX V3_OS_SUPPORT_AX
131#endif
132
133#ifdef GOAMD64_v4
134#define NEED_MAX_CPUID 0x80000001
135#define NEED_FEATURES_CX V4_FEATURES_CX
136#define NEED_EXT_FEATURES_CX V4_EXT_FEATURES_CX
137#define NEED_EXT_FEATURES_BX V4_EXT_FEATURES_BX
138
139// Darwin requires a different approach to check AVX512 support, see CL 285572.
140#ifdef GOOS_darwin
141#define NEED_OS_SUPPORT_AX V3_OS_SUPPORT_AX
142// These values are from:
143// https://github.com/apple/darwin-xnu/blob/xnu-4570.1.46/osfmk/i386/cpu_capabilities.h
144#define commpage64_base_address 0x00007fffffe00000
145#define commpage64_cpu_capabilities64 (commpage64_base_address+0x010)
146#define commpage64_version (commpage64_base_address+0x01E)
147#define AVX512F 0x0000004000000000
148#define AVX512CD 0x0000008000000000
149#define AVX512DQ 0x0000010000000000
150#define AVX512BW 0x0000020000000000
151#define AVX512VL 0x0000100000000000
152#define NEED_DARWIN_SUPPORT (AVX512F | AVX512DQ | AVX512CD | AVX512BW | AVX512VL)
153#else
154#define NEED_OS_SUPPORT_AX V4_OS_SUPPORT_AX
155#endif
156
157#endif
158
159TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0
160 // copy arguments forward on an even stack
161 MOVQ DI, AX // argc
162 MOVQ SI, BX // argv
163 SUBQ $(5*8), SP // 3args 2auto
164 ANDQ $~15, SP
165 MOVQ AX, 24(SP)
166 MOVQ BX, 32(SP)
167
168 // create istack out of the given (operating system) stack.
169 // _cgo_init may update stackguard.
170 MOVQ $runtime·g0(SB), DI
171 LEAQ (-64*1024)(SP), BX
172 MOVQ BX, g_stackguard0(DI)
173 MOVQ BX, g_stackguard1(DI)
174 MOVQ BX, (g_stack+stack_lo)(DI)
175 MOVQ SP, (g_stack+stack_hi)(DI)
176
177 // find out information about the processor we're on
178 MOVL $0, AX
179 CPUID
180 CMPL AX, $0
181 JE nocpuinfo
182
183 CMPL BX, $0x756E6547 // "Genu"
184 JNE notintel
185 CMPL DX, $0x49656E69 // "ineI"
186 JNE notintel
187 CMPL CX, $0x6C65746E // "ntel"
188 JNE notintel
189 MOVB $1, runtime·isIntel(SB)
190
191notintel:
192 // Load EAX=1 cpuid flags
193 MOVL $1, AX
194 CPUID
195 MOVL AX, runtime·processorVersionInfo(SB)
196
197nocpuinfo:
198 // if there is an _cgo_init, call it.
199 MOVQ _cgo_init(SB), AX
200 TESTQ AX, AX
201 JZ needtls
202 // arg 1: g0, already in DI
203 MOVQ $setg_gcc<>(SB), SI // arg 2: setg_gcc
204 MOVQ $0, DX // arg 3, 4: not used when using platform's TLS
205 MOVQ $0, CX
206#ifdef GOOS_android
207 MOVQ $runtime·tls_g(SB), DX // arg 3: &tls_g
208 // arg 4: TLS base, stored in slot 0 (Android's TLS_SLOT_SELF).
209 // Compensate for tls_g (+16).
210 MOVQ -16(TLS), CX
211#endif
212#ifdef GOOS_windows
213 MOVQ $runtime·tls_g(SB), DX // arg 3: &tls_g
214 // Adjust for the Win64 calling convention.
215 MOVQ CX, R9 // arg 4
216 MOVQ DX, R8 // arg 3
217 MOVQ SI, DX // arg 2
218 MOVQ DI, CX // arg 1
219#endif
220 CALL AX
221
222 // update stackguard after _cgo_init
223 MOVQ $runtime·g0(SB), CX
224 MOVQ (g_stack+stack_lo)(CX), AX
225 ADDQ $const_stackGuard, AX
226 MOVQ AX, g_stackguard0(CX)
227 MOVQ AX, g_stackguard1(CX)
228
229#ifndef GOOS_windows
230 JMP ok
231#endif
232needtls:
233#ifdef GOOS_plan9
234 // skip TLS setup on Plan 9
235 JMP ok
236#endif
237#ifdef GOOS_solaris
238 // skip TLS setup on Solaris
239 JMP ok
240#endif
241#ifdef GOOS_illumos
242 // skip TLS setup on illumos
243 JMP ok
244#endif
245#ifdef GOOS_darwin
246 // skip TLS setup on Darwin
247 JMP ok
248#endif
249#ifdef GOOS_openbsd
250 // skip TLS setup on OpenBSD
251 JMP ok
252#endif
253
254#ifdef GOOS_windows
255 CALL runtime·wintls(SB)
256#endif
257
258 LEAQ runtime·m0+m_tls(SB), DI
259 CALL runtime·settls(SB)
260
261 // store through it, to make sure it works
262 get_tls(BX)
263 MOVQ $0x123, g(BX)
264 MOVQ runtime·m0+m_tls(SB), AX
265 CMPQ AX, $0x123
266 JEQ 2(PC)
267 CALL runtime·abort(SB)
268ok:
269 // set the per-goroutine and per-mach "registers"
270 get_tls(BX)
271 LEAQ runtime·g0(SB), CX
272 MOVQ CX, g(BX)
273 LEAQ runtime·m0(SB), AX
274
275 // save m->g0 = g0
276 MOVQ CX, m_g0(AX)
277 // save m0 to g0->m
278 MOVQ AX, g_m(CX)
279
280 CLD // convention is D is always left cleared
281
282 // Check GOAMD64 requirements
283 // We need to do this after setting up TLS, so that
284 // we can report an error if there is a failure. See issue 49586.
285#ifdef NEED_FEATURES_CX
286 MOVL $0, AX
287 CPUID
288 CMPL AX, $0
289 JE bad_cpu
290 MOVL $1, AX
291 CPUID
292 ANDL $NEED_FEATURES_CX, CX
293 CMPL CX, $NEED_FEATURES_CX
294 JNE bad_cpu
295#endif
296
297#ifdef NEED_MAX_CPUID
298 MOVL $0x80000000, AX
299 CPUID
300 CMPL AX, $NEED_MAX_CPUID
301 JL bad_cpu
302#endif
303
304#ifdef NEED_EXT_FEATURES_BX
305 MOVL $7, AX
306 MOVL $0, CX
307 CPUID
308 ANDL $NEED_EXT_FEATURES_BX, BX
309 CMPL BX, $NEED_EXT_FEATURES_BX
310 JNE bad_cpu
311#endif
312
313#ifdef NEED_EXT_FEATURES_CX
314 MOVL $0x80000001, AX
315 CPUID
316 ANDL $NEED_EXT_FEATURES_CX, CX
317 CMPL CX, $NEED_EXT_FEATURES_CX
318 JNE bad_cpu
319#endif
320
321#ifdef NEED_OS_SUPPORT_AX
322 XORL CX, CX
323 XGETBV
324 ANDL $NEED_OS_SUPPORT_AX, AX
325 CMPL AX, $NEED_OS_SUPPORT_AX
326 JNE bad_cpu
327#endif
328
329#ifdef NEED_DARWIN_SUPPORT
330 MOVQ $commpage64_version, BX
331 CMPW (BX), $13 // cpu_capabilities64 undefined in versions < 13
332 JL bad_cpu
333 MOVQ $commpage64_cpu_capabilities64, BX
334 MOVQ (BX), BX
335 MOVQ $NEED_DARWIN_SUPPORT, CX
336 ANDQ CX, BX
337 CMPQ BX, CX
338 JNE bad_cpu
339#endif
340
341 CALL runtime·check(SB)
342
343 MOVL 24(SP), AX // copy argc
344 MOVL AX, 0(SP)
345 MOVQ 32(SP), AX // copy argv
346 MOVQ AX, 8(SP)
347 CALL runtime·args(SB)
348 CALL runtime·osinit(SB)
349 CALL runtime·schedinit(SB)
350
351 // create a new goroutine to start program
352 MOVQ $runtime·mainPC(SB), AX // entry
353 PUSHQ AX
354 CALL runtime·newproc(SB)
355 POPQ AX
356
357 // start this M
358 CALL runtime·mstart(SB)
359
360 CALL runtime·abort(SB) // mstart should never return
361 RET
362
363bad_cpu: // show that the program requires a certain microarchitecture level.
364 MOVQ $2, 0(SP)
365 MOVQ $bad_cpu_msg<>(SB), AX
366 MOVQ AX, 8(SP)
367 MOVQ $84, 16(SP)
368 CALL runtime·write(SB)
369 MOVQ $1, 0(SP)
370 CALL runtime·exit(SB)
371 CALL runtime·abort(SB)
372 RET
373
374 // Prevent dead-code elimination of debugCallV2 and debugPinnerV1, which are
375 // intended to be called by debuggers.
376 MOVQ $runtime·debugPinnerV1<ABIInternal>(SB), AX
377 MOVQ $runtime·debugCallV2<ABIInternal>(SB), AX
378 RET
379
380// mainPC is a function value for runtime.main, to be passed to newproc.
381// The reference to runtime.main is made via ABIInternal, since the
382// actual function (not the ABI0 wrapper) is needed by newproc.
383DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
384GLOBL runtime·mainPC(SB),RODATA,$8
385
386TEXT runtime·breakpoint(SB),NOSPLIT,$0-0
387 BYTE $0xcc
388 RET
389
390TEXT runtime·asminit(SB),NOSPLIT,$0-0
391 // No per-thread init.
392 RET
393
394TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME|NOFRAME,$0
395 CALL runtime·mstart0(SB)
396 RET // not reached
397
398/*
399 * go-routine
400 */
401
402// func gogo(buf *gobuf)
403// restore state from Gobuf; longjmp
404TEXT runtime·gogo(SB), NOSPLIT, $0-8
405 MOVQ buf+0(FP), BX // gobuf
406 MOVQ gobuf_g(BX), DX
407 MOVQ 0(DX), CX // make sure g != nil
408 JMP gogo<>(SB)
409
410TEXT gogo<>(SB), NOSPLIT, $0
411 get_tls(CX)
412 MOVQ DX, g(CX)
413 MOVQ DX, R14 // set the g register
414 MOVQ gobuf_sp(BX), SP // restore SP
415 MOVQ gobuf_ctxt(BX), DX
416 MOVQ gobuf_bp(BX), BP
417 MOVQ $0, gobuf_sp(BX) // clear to help garbage collector
418 MOVQ $0, gobuf_ctxt(BX)
419 MOVQ $0, gobuf_bp(BX)
420 MOVQ gobuf_pc(BX), BX
421 JMP BX
422
423// func mcall(fn func(*g))
424// Switch to m->g0's stack, call fn(g).
425// Fn must never return. It should gogo(&g->sched)
426// to keep running g.
427TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT, $0-8
428 MOVQ AX, DX // DX = fn
429
430 // Save state in g->sched. The caller's SP and PC are restored by gogo to
431 // resume execution in the caller's frame (implicit return). The caller's BP
432 // is also restored to support frame pointer unwinding.
433 MOVQ SP, BX // hide (SP) reads from vet
434 MOVQ 8(BX), BX // caller's PC
435 MOVQ BX, (g_sched+gobuf_pc)(R14)
436 LEAQ fn+0(FP), BX // caller's SP
437 MOVQ BX, (g_sched+gobuf_sp)(R14)
438 // Get the caller's frame pointer by dereferencing BP. Storing BP as it is
439 // can cause a frame pointer cycle, see CL 476235.
440 MOVQ (BP), BX // caller's BP
441 MOVQ BX, (g_sched+gobuf_bp)(R14)
442
443 // switch to m->g0 & its stack, call fn
444 MOVQ g_m(R14), BX
445 MOVQ m_g0(BX), SI // SI = g.m.g0
446 CMPQ SI, R14 // if g == m->g0 call badmcall
447 JNE goodm
448 JMP runtime·badmcall(SB)
449goodm:
450 MOVQ R14, AX // AX (and arg 0) = g
451 MOVQ SI, R14 // g = g.m.g0
452 get_tls(CX) // Set G in TLS
453 MOVQ R14, g(CX)
454 MOVQ (g_sched+gobuf_sp)(R14), SP // sp = g0.sched.sp
455 MOVQ $0, BP // clear frame pointer, as caller may execute on another M
456 PUSHQ AX // open up space for fn's arg spill slot
457 MOVQ 0(DX), R12
458 CALL R12 // fn(g)
459 // The Windows native stack unwinder incorrectly classifies the next instruction
460 // as part of the function epilogue, producing a wrong call stack.
461 // Add a NOP to work around this issue. See go.dev/issue/67007.
462 BYTE $0x90
463 POPQ AX
464 JMP runtime·badmcall2(SB)
465 RET
466
467// systemstack_switch is a dummy routine that systemstack leaves at the bottom
468// of the G stack. We need to distinguish the routine that
469// lives at the bottom of the G stack from the one that lives
470// at the top of the system stack because the one at the top of
471// the system stack terminates the stack walk (see topofstack()).
472// The frame layout needs to match systemstack
473// so that it can pretend to be systemstack_switch.
474TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
475 UNDEF
476 // Make sure this function is not leaf,
477 // so the frame is saved.
478 CALL runtime·abort(SB)
479 RET
480
481// func systemstack(fn func())
482TEXT runtime·systemstack(SB), NOSPLIT, $0-8
483 MOVQ fn+0(FP), DI // DI = fn
484 get_tls(CX)
485 MOVQ g(CX), AX // AX = g
486 MOVQ g_m(AX), BX // BX = m
487
488 CMPQ AX, m_gsignal(BX)
489 JEQ noswitch
490
491 MOVQ m_g0(BX), DX // DX = g0
492 CMPQ AX, DX
493 JEQ noswitch
494
495 CMPQ AX, m_curg(BX)
496 JNE bad
497
498 // Switch stacks.
499 // The original frame pointer is stored in BP,
500 // which is useful for stack unwinding.
501 // Save our state in g->sched. Pretend to
502 // be systemstack_switch if the G stack is scanned.
503 CALL gosave_systemstack_switch<>(SB)
504
505 // switch to g0
506 MOVQ DX, g(CX)
507 MOVQ DX, R14 // set the g register
508 MOVQ (g_sched+gobuf_sp)(DX), SP
509
510 // call target function
511 MOVQ DI, DX
512 MOVQ 0(DI), DI
513 CALL DI
514
515 // switch back to g
516 get_tls(CX)
517 MOVQ g(CX), AX
518 MOVQ g_m(AX), BX
519 MOVQ m_curg(BX), AX
520 MOVQ AX, g(CX)
521 MOVQ (g_sched+gobuf_sp)(AX), SP
522 MOVQ (g_sched+gobuf_bp)(AX), BP
523 MOVQ $0, (g_sched+gobuf_sp)(AX)
524 MOVQ $0, (g_sched+gobuf_bp)(AX)
525 RET
526
527noswitch:
528 // already on m stack; tail call the function
529 // Using a tail call here cleans up tracebacks since we won't stop
530 // at an intermediate systemstack.
531 MOVQ DI, DX
532 MOVQ 0(DI), DI
533 // The function epilogue is not called on a tail call.
534 // Pop BP from the stack to simulate it.
535 POPQ BP
536 JMP DI
537
538bad:
539 // Bad: g is not gsignal, not g0, not curg. What is it?
540 MOVQ $runtime·badsystemstack(SB), AX
541 CALL AX
542 INT $3
543
544// func switchToCrashStack0(fn func())
545TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
546 MOVQ g_m(R14), BX // curm
547
548 // set g to gcrash
549 LEAQ runtime·gcrash(SB), R14 // g = &gcrash
550 MOVQ BX, g_m(R14) // g.m = curm
551 MOVQ R14, m_g0(BX) // curm.g0 = g
552 get_tls(CX)
553 MOVQ R14, g(CX)
554
555 // switch to crashstack
556 MOVQ (g_stack+stack_hi)(R14), BX
557 SUBQ $(4*8), BX
558 MOVQ BX, SP
559
560 // call target function
561 MOVQ AX, DX
562 MOVQ 0(AX), AX
563 CALL AX
564
565 // should never return
566 CALL runtime·abort(SB)
567 UNDEF
568
569/*
570 * support for morestack
571 */
572
573// Called during function prolog when more stack is needed.
574//
575// The traceback routines see morestack on a g0 as being
576// the top of a stack (for example, morestack calling newstack
577// calling the scheduler calling newm calling gc), so we must
578// record an argument size. For that purpose, it has no arguments.
579TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
580 // Cannot grow scheduler stack (m->g0).
581 get_tls(CX)
582 MOVQ g(CX), DI // DI = g
583 MOVQ g_m(DI), BX // BX = m
584
585 // Set g->sched to context in f.
586 MOVQ 0(SP), AX // f's PC
587 MOVQ AX, (g_sched+gobuf_pc)(DI)
588 LEAQ 8(SP), AX // f's SP
589 MOVQ AX, (g_sched+gobuf_sp)(DI)
590 MOVQ BP, (g_sched+gobuf_bp)(DI)
591 MOVQ DX, (g_sched+gobuf_ctxt)(DI)
592
593 MOVQ m_g0(BX), SI // SI = m.g0
594 CMPQ DI, SI
595 JNE 3(PC)
596 CALL runtime·badmorestackg0(SB)
597 CALL runtime·abort(SB)
598
599 // Cannot grow signal stack (m->gsignal).
600 MOVQ m_gsignal(BX), SI
601 CMPQ DI, SI
602 JNE 3(PC)
603 CALL runtime·badmorestackgsignal(SB)
604 CALL runtime·abort(SB)
605
606 // Called from f.
607 // Set m->morebuf to f's caller.
608 NOP SP // tell vet SP changed - stop checking offsets
609 MOVQ 8(SP), AX // f's caller's PC
610 MOVQ AX, (m_morebuf+gobuf_pc)(BX)
611 LEAQ 16(SP), AX // f's caller's SP
612 MOVQ AX, (m_morebuf+gobuf_sp)(BX)
613 MOVQ DI, (m_morebuf+gobuf_g)(BX)
614
615 // Call newstack on m->g0's stack.
616 MOVQ m_g0(BX), BX
617 MOVQ BX, g(CX)
618 MOVQ (g_sched+gobuf_sp)(BX), SP
619 MOVQ $0, BP // clear frame pointer, as caller may execute on another M
620 CALL runtime·newstack(SB)
621 CALL runtime·abort(SB) // crash if newstack returns
622 RET
623
624// morestack but not preserving ctxt.
625TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
626 MOVL $0, DX
627 JMP runtime·morestack(SB)
628
629// spillArgs stores return values from registers to a *internal/abi.RegArgs in R12.
630TEXT ·spillArgs(SB),NOSPLIT,$0-0
631 MOVQ AX, 0(R12)
632 MOVQ BX, 8(R12)
633 MOVQ CX, 16(R12)
634 MOVQ DI, 24(R12)
635 MOVQ SI, 32(R12)
636 MOVQ R8, 40(R12)
637 MOVQ R9, 48(R12)
638 MOVQ R10, 56(R12)
639 MOVQ R11, 64(R12)
640 MOVQ X0, 72(R12)
641 MOVQ X1, 80(R12)
642 MOVQ X2, 88(R12)
643 MOVQ X3, 96(R12)
644 MOVQ X4, 104(R12)
645 MOVQ X5, 112(R12)
646 MOVQ X6, 120(R12)
647 MOVQ X7, 128(R12)
648 MOVQ X8, 136(R12)
649 MOVQ X9, 144(R12)
650 MOVQ X10, 152(R12)
651 MOVQ X11, 160(R12)
652 MOVQ X12, 168(R12)
653 MOVQ X13, 176(R12)
654 MOVQ X14, 184(R12)
655 RET
656
657// unspillArgs loads args into registers from a *internal/abi.RegArgs in R12.
658TEXT ·unspillArgs(SB),NOSPLIT,$0-0
659 MOVQ 0(R12), AX
660 MOVQ 8(R12), BX
661 MOVQ 16(R12), CX
662 MOVQ 24(R12), DI
663 MOVQ 32(R12), SI
664 MOVQ 40(R12), R8
665 MOVQ 48(R12), R9
666 MOVQ 56(R12), R10
667 MOVQ 64(R12), R11
668 MOVQ 72(R12), X0
669 MOVQ 80(R12), X1
670 MOVQ 88(R12), X2
671 MOVQ 96(R12), X3
672 MOVQ 104(R12), X4
673 MOVQ 112(R12), X5
674 MOVQ 120(R12), X6
675 MOVQ 128(R12), X7
676 MOVQ 136(R12), X8
677 MOVQ 144(R12), X9
678 MOVQ 152(R12), X10
679 MOVQ 160(R12), X11
680 MOVQ 168(R12), X12
681 MOVQ 176(R12), X13
682 MOVQ 184(R12), X14
683 RET
684
685// reflectcall: call a function with the given argument list
686// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
687// we don't have variable-sized frames, so we use a small number
688// of constant-sized-frame functions to encode a few bits of size in the pc.
689// Caution: ugly multiline assembly macros in your future!
690
691#define DISPATCH(NAME,MAXSIZE) \
692 CMPQ CX, $MAXSIZE; \
693 JA 3(PC); \
694 MOVQ $NAME(SB), AX; \
695 JMP AX
696// Note: can't just "JMP NAME(SB)" - bad inlining results.
697
698TEXT ·reflectcall(SB), NOSPLIT, $0-48
699 MOVLQZX frameSize+32(FP), CX
700 DISPATCH(runtime·call16, 16)
701 DISPATCH(runtime·call32, 32)
702 DISPATCH(runtime·call64, 64)
703 DISPATCH(runtime·call128, 128)
704 DISPATCH(runtime·call256, 256)
705 DISPATCH(runtime·call512, 512)
706 DISPATCH(runtime·call1024, 1024)
707 DISPATCH(runtime·call2048, 2048)
708 DISPATCH(runtime·call4096, 4096)
709 DISPATCH(runtime·call8192, 8192)
710 DISPATCH(runtime·call16384, 16384)
711 DISPATCH(runtime·call32768, 32768)
712 DISPATCH(runtime·call65536, 65536)
713 DISPATCH(runtime·call131072, 131072)
714 DISPATCH(runtime·call262144, 262144)
715 DISPATCH(runtime·call524288, 524288)
716 DISPATCH(runtime·call1048576, 1048576)
717 DISPATCH(runtime·call2097152, 2097152)
718 DISPATCH(runtime·call4194304, 4194304)
719 DISPATCH(runtime·call8388608, 8388608)
720 DISPATCH(runtime·call16777216, 16777216)
721 DISPATCH(runtime·call33554432, 33554432)
722 DISPATCH(runtime·call67108864, 67108864)
723 DISPATCH(runtime·call134217728, 134217728)
724 DISPATCH(runtime·call268435456, 268435456)
725 DISPATCH(runtime·call536870912, 536870912)
726 DISPATCH(runtime·call1073741824, 1073741824)
727 MOVQ $runtime·badreflectcall(SB), AX
728 JMP AX
729
730#define CALLFN(NAME,MAXSIZE) \
731TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
732 NO_LOCAL_POINTERS; \
733 /* copy arguments to stack */ \
734 MOVQ stackArgs+16(FP), SI; \
735 MOVLQZX stackArgsSize+24(FP), CX; \
736 MOVQ SP, DI; \
737 REP;MOVSB; \
738 /* set up argument registers */ \
739 MOVQ regArgs+40(FP), R12; \
740 CALL ·unspillArgs(SB); \
741 /* call function */ \
742 MOVQ f+8(FP), DX; \
743 PCDATA $PCDATA_StackMapIndex, $0; \
744 MOVQ (DX), R12; \
745 CALL R12; \
746 /* copy register return values back */ \
747 MOVQ regArgs+40(FP), R12; \
748 CALL ·spillArgs(SB); \
749 MOVLQZX stackArgsSize+24(FP), CX; \
750 MOVLQZX stackRetOffset+28(FP), BX; \
751 MOVQ stackArgs+16(FP), DI; \
752 MOVQ stackArgsType+0(FP), DX; \
753 MOVQ SP, SI; \
754 ADDQ BX, DI; \
755 ADDQ BX, SI; \
756 SUBQ BX, CX; \
757 CALL callRet<>(SB); \
758 RET
759
760// callRet copies return values back at the end of call*. This is a
761// separate function so it can allocate stack space for the arguments
762// to reflectcallmove. It does not follow the Go ABI; it expects its
763// arguments in registers.
764TEXT callRet<>(SB), NOSPLIT, $40-0
765 NO_LOCAL_POINTERS
766 MOVQ DX, 0(SP)
767 MOVQ DI, 8(SP)
768 MOVQ SI, 16(SP)
769 MOVQ CX, 24(SP)
770 MOVQ R12, 32(SP)
771 CALL runtime·reflectcallmove(SB)
772 RET
773
774CALLFN(·call16, 16)
775CALLFN(·call32, 32)
776CALLFN(·call64, 64)
777CALLFN(·call128, 128)
778CALLFN(·call256, 256)
779CALLFN(·call512, 512)
780CALLFN(·call1024, 1024)
781CALLFN(·call2048, 2048)
782CALLFN(·call4096, 4096)
783CALLFN(·call8192, 8192)
784CALLFN(·call16384, 16384)
785CALLFN(·call32768, 32768)
786CALLFN(·call65536, 65536)
787CALLFN(·call131072, 131072)
788CALLFN(·call262144, 262144)
789CALLFN(·call524288, 524288)
790CALLFN(·call1048576, 1048576)
791CALLFN(·call2097152, 2097152)
792CALLFN(·call4194304, 4194304)
793CALLFN(·call8388608, 8388608)
794CALLFN(·call16777216, 16777216)
795CALLFN(·call33554432, 33554432)
796CALLFN(·call67108864, 67108864)
797CALLFN(·call134217728, 134217728)
798CALLFN(·call268435456, 268435456)
799CALLFN(·call536870912, 536870912)
800CALLFN(·call1073741824, 1073741824)
801
802TEXT runtime·procyield(SB),NOSPLIT,$0-0
803 MOVL cycles+0(FP), AX
804again:
805 PAUSE
806 SUBL $1, AX
807 JNZ again
808 RET
809
810
811TEXT ·publicationBarrier<ABIInternal>(SB),NOSPLIT,$0-0
812 // Stores are already ordered on x86, so this is just a
813 // compile barrier.
814 RET
815
816// Save state of caller into g->sched,
817// but using fake PC from systemstack_switch.
818// Must only be called from functions with frame pointer
819// and without locals ($0) or else unwinding from
820// systemstack_switch is incorrect.
821// Smashes R9.
822TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
823 // Take systemstack_switch PC and add 8 bytes to skip
824 // the prologue. The final location does not matter
825 // as long as we are between the prologue and the epilogue.
826 MOVQ $runtime·systemstack_switch+8(SB), R9
827 MOVQ R9, (g_sched+gobuf_pc)(R14)
828 LEAQ 8(SP), R9
829 MOVQ R9, (g_sched+gobuf_sp)(R14)
830 MOVQ BP, (g_sched+gobuf_bp)(R14)
831 // Assert ctxt is zero. See func save.
832 MOVQ (g_sched+gobuf_ctxt)(R14), R9
833 TESTQ R9, R9
834 JZ 2(PC)
835 CALL runtime·abort(SB)
836 RET
837
838// func asmcgocall_no_g(fn, arg unsafe.Pointer)
839// Call fn(arg) aligned appropriately for the gcc ABI.
840// Called on a system stack, and there may be no g yet (during needm).
841TEXT ·asmcgocall_no_g(SB),NOSPLIT,$32-16
842 MOVQ fn+0(FP), AX
843 MOVQ arg+8(FP), BX
844 MOVQ SP, DX
845 ANDQ $~15, SP // alignment
846 MOVQ DX, 8(SP)
847 MOVQ BX, DI // DI = first argument in AMD64 ABI
848 MOVQ BX, CX // CX = first argument in Win64
849 CALL AX
850 MOVQ 8(SP), DX
851 MOVQ DX, SP
852 RET
853
854// asmcgocall_landingpad calls AX with BX as argument.
855// Must be called on the system stack.
856TEXT ·asmcgocall_landingpad(SB),NOSPLIT,$0-0
857#ifdef GOOS_windows
858 // Make sure we have enough room for 4 stack-backed fast-call
859 // registers as per Windows amd64 calling convention.
860 ADJSP $32
861 // On Windows, asmcgocall_landingpad acts as landing pad for exceptions
862 // thrown in the cgo call. Exceptions that reach this function will be
863 // handled by runtime.sehtramp thanks to the SEH metadata added
864 // by the compiler.
865 // Note that runtime.sehtramp can't be attached directly to asmcgocall
866 // because its initial stack pointer can be outside the system stack bounds,
867 // and Windows stops the stack unwinding without calling the exception handler
868 // when it reaches that point.
869 MOVQ BX, CX // CX = first argument in Win64
870 CALL AX
871 // The exception handler is not called if the next instruction is part of
872 // the epilogue, which includes the RET instruction, so we need to add a NOP here.
873 BYTE $0x90
874 ADJSP $-32
875 RET
876#endif
877 // Tail call AX on non-Windows, as the extra stack frame is not needed.
878 MOVQ BX, DI // DI = first argument in AMD64 ABI
879 JMP AX
880
881// func asmcgocall(fn, arg unsafe.Pointer) int32
882// Call fn(arg) on the scheduler stack,
883// aligned appropriately for the gcc ABI.
884// See cgocall.go for more details.
885TEXT ·asmcgocall(SB),NOSPLIT,$0-20
886 MOVQ fn+0(FP), AX
887 MOVQ arg+8(FP), BX
888
889 MOVQ SP, DX
890
891 // Figure out if we need to switch to m->g0 stack.
892 // We get called to create new OS threads too, and those
893 // come in on the m->g0 stack already. Or we might already
894 // be on the m->gsignal stack.
895 get_tls(CX)
896 MOVQ g(CX), DI
897 CMPQ DI, $0
898 JEQ nosave
899 MOVQ g_m(DI), R8
900 MOVQ m_gsignal(R8), SI
901 CMPQ DI, SI
902 JEQ nosave
903 MOVQ m_g0(R8), SI
904 CMPQ DI, SI
905 JEQ nosave
906
907 // Switch to system stack.
908 // The original frame pointer is stored in BP,
909 // which is useful for stack unwinding.
910 CALL gosave_systemstack_switch<>(SB)
911 MOVQ SI, g(CX)
912 MOVQ (g_sched+gobuf_sp)(SI), SP
913
914 // Now on a scheduling stack (a pthread-created stack).
915 SUBQ $16, SP
916 ANDQ $~15, SP // alignment for gcc ABI
917 MOVQ DI, 8(SP) // save g
918 MOVQ (g_stack+stack_hi)(DI), DI
919 SUBQ DX, DI
920 MOVQ DI, 0(SP) // save depth in stack (can't just save SP, as stack might be copied during a callback)
921 CALL runtime·asmcgocall_landingpad(SB)
922
923 // Restore registers, g, stack pointer.
924 get_tls(CX)
925 MOVQ 8(SP), DI
926 MOVQ (g_stack+stack_hi)(DI), SI
927 SUBQ 0(SP), SI
928 MOVQ DI, g(CX)
929 MOVQ SI, SP
930
931 MOVL AX, ret+16(FP)
932 RET
933
934nosave:
935 // Running on a system stack, perhaps even without a g.
936 // Having no g can happen during thread creation or thread teardown
937 // (see needm/dropm on Solaris, for example).
938 // This code is like the above sequence but without saving/restoring g
939 // and without worrying about the stack moving out from under us
940 // (because we're on a system stack, not a goroutine stack).
941 // The above code could be used directly if already on a system stack,
942 // but then the only path through this code would be a rare case on Solaris.
943 // Using this code for all "already on system stack" calls exercises it more,
944 // which should help keep it correct.
945 SUBQ $16, SP
946 ANDQ $~15, SP
947 MOVQ $0, 8(SP) // where above code stores g, in case someone looks during debugging
948 MOVQ DX, 0(SP) // save original stack pointer
949 CALL runtime·asmcgocall_landingpad(SB)
950 MOVQ 0(SP), SI // restore original stack pointer
951 MOVQ SI, SP
952 MOVL AX, ret+16(FP)
953 RET
954
955#ifdef GOOS_windows
956// Dummy TLS that's used on Windows so that we don't crash trying
957// to restore the G register in needm. needm and its callees are
958// very careful never to actually use the G, the TLS just can't be
959// unset since we're in Go code.
960GLOBL zeroTLS<>(SB),RODATA,$const_tlsSize
961#endif
962
963// func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
964// See cgocall.go for more details.
965TEXT ·cgocallback(SB),NOSPLIT,$24-24
966 NO_LOCAL_POINTERS
967
968 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
969 // It is used to dropm while thread is exiting.
970 MOVQ fn+0(FP), AX
971 CMPQ AX, $0
972 JNE loadg
973 // Restore the g from frame.
974 get_tls(CX)
975 MOVQ frame+8(FP), BX
976 MOVQ BX, g(CX)
977 JMP dropm
978
979loadg:
980 // If g is nil, Go did not create the current thread,
981 // or if this thread never called into Go on pthread platforms.
982 // Call needm to obtain one m for temporary use.
983 // In this case, we're running on the thread stack, so there's
984 // lots of space, but the linker doesn't know. Hide the call from
985 // the linker analysis by using an indirect call through AX.
986 get_tls(CX)
987#ifdef GOOS_windows
988 MOVL $0, BX
989 CMPQ CX, $0
990 JEQ 2(PC)
991#endif
992 MOVQ g(CX), BX
993 CMPQ BX, $0
994 JEQ needm
995 MOVQ g_m(BX), BX
996 MOVQ BX, savedm-8(SP) // saved copy of oldm
997 JMP havem
998needm:
999#ifdef GOOS_windows
1000 // Set up a dummy TLS value. needm is careful not to use it,
1001 // but it needs to be there to prevent autogenerated code from
1002 // crashing when it loads from it.
1003 // We don't need to clear it or anything later because needm
1004 // will set up TLS properly.
1005 MOVQ $zeroTLS<>(SB), DI
1006 CALL runtime·settls(SB)
1007#endif
1008 // On some platforms (Windows) we cannot call needm through
1009 // an ABI wrapper because there's no TLS set up, and the ABI
1010 // wrapper will try to restore the G register (R14) from TLS.
1011 // Clear X15 because Go expects it and we're not calling
1012 // through a wrapper, but otherwise avoid setting the G
1013 // register in the wrapper and call needm directly. It
1014 // takes no arguments and doesn't return any values so
1015 // there's no need to handle that. Clear R14 so that there's
1016 // a bad value in there, in case needm tries to use it.
1017 XORPS X15, X15
1018 XORQ R14, R14
1019 MOVQ $runtime·needAndBindM<ABIInternal>(SB), AX
1020 CALL AX
1021 MOVQ $0, savedm-8(SP)
1022 get_tls(CX)
1023 MOVQ g(CX), BX
1024 MOVQ g_m(BX), BX
1025
1026 // Set m->sched.sp = SP, so that if a panic happens
1027 // during the function we are about to execute, it will
1028 // have a valid SP to run on the g0 stack.
1029 // The next few lines (after the havem label)
1030 // will save this SP onto the stack and then write
1031 // the same SP back to m->sched.sp. That seems redundant,
1032 // but if an unrecovered panic happens, unwindm will
1033 // restore the g->sched.sp from the stack location
1034 // and then systemstack will try to use it. If we don't set it here,
1035 // that restored SP will be uninitialized (typically 0) and
1036 // will not be usable.
1037 MOVQ m_g0(BX), SI
1038 MOVQ SP, (g_sched+gobuf_sp)(SI)
1039
1040havem:
1041 // Now there's a valid m, and we're running on its m->g0.
1042 // Save current m->g0->sched.sp on stack and then set it to SP.
1043 // Save current sp in m->g0->sched.sp in preparation for
1044 // switch back to m->curg stack.
1045 // NOTE: unwindm knows that the saved g->sched.sp is at 0(SP).
1046 MOVQ m_g0(BX), SI
1047 MOVQ (g_sched+gobuf_sp)(SI), AX
1048 MOVQ AX, 0(SP)
1049 MOVQ SP, (g_sched+gobuf_sp)(SI)
1050
1051 // Switch to m->curg stack and call runtime.cgocallbackg.
1052 // Because we are taking over the execution of m->curg
1053 // but *not* resuming what had been running, we need to
1054 // save that information (m->curg->sched) so we can restore it.
1055 // We can restore m->curg->sched.sp easily, because calling
1056 // runtime.cgocallbackg leaves SP unchanged upon return.
1057 // To save m->curg->sched.pc, we push it onto the curg stack and
1058 // open a frame the same size as cgocallback's g0 frame.
1059 // Once we switch to the curg stack, the pushed PC will appear
1060 // to be the return PC of cgocallback, so that the traceback
1061 // will seamlessly trace back into the earlier calls.
1062 MOVQ m_curg(BX), SI
1063 MOVQ SI, g(CX)
1064 MOVQ (g_sched+gobuf_sp)(SI), DI // prepare stack as DI
1065 MOVQ (g_sched+gobuf_pc)(SI), BX
1066 MOVQ BX, -8(DI) // "push" return PC on the g stack
1067 // Gather our arguments into registers.
1068 MOVQ fn+0(FP), BX
1069 MOVQ frame+8(FP), CX
1070 MOVQ ctxt+16(FP), DX
1071 // Compute the size of the frame, including return PC and, if
1072 // GOEXPERIMENT=framepointer, the saved base pointer
1073 LEAQ fn+0(FP), AX
1074 SUBQ SP, AX // AX is our actual frame size
1075 SUBQ AX, DI // Allocate the same frame size on the g stack
1076 MOVQ DI, SP
1077
1078 MOVQ BX, 0(SP)
1079 MOVQ CX, 8(SP)
1080 MOVQ DX, 16(SP)
1081 MOVQ $runtime·cgocallbackg(SB), AX
1082 CALL AX // indirect call to bypass nosplit check. We're on a different stack now.
1083
1084 // Compute the size of the frame again. FP and SP have
1085 // completely different values here than they did above,
1086 // but only their difference matters.
1087 LEAQ fn+0(FP), AX
1088 SUBQ SP, AX
1089
1090 // Restore g->sched (== m->curg->sched) from saved values.
1091 get_tls(CX)
1092 MOVQ g(CX), SI
1093 MOVQ SP, DI
1094 ADDQ AX, DI
1095 MOVQ -8(DI), BX
1096 MOVQ BX, (g_sched+gobuf_pc)(SI)
1097 MOVQ DI, (g_sched+gobuf_sp)(SI)
1098
1099 // Switch back to m->g0's stack and restore m->g0->sched.sp.
1100 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
1101 // so we do not have to restore it.)
1102 MOVQ g(CX), BX
1103 MOVQ g_m(BX), BX
1104 MOVQ m_g0(BX), SI
1105 MOVQ SI, g(CX)
1106 MOVQ (g_sched+gobuf_sp)(SI), SP
1107 MOVQ 0(SP), AX
1108 MOVQ AX, (g_sched+gobuf_sp)(SI)
1109
1110 // If the m on entry was nil, we called needm above to borrow an m,
1111 // 1. for the duration of the call on non-pthread platforms,
1112 // 2. or the duration of the C thread alive on pthread platforms.
1113 // If the m on entry wasn't nil,
1114 // 1. the thread might be a Go thread,
1115 // 2. or it wasn't the first call from a C thread on pthread platforms,
1116 // since then we skip dropm to reuse the m in the first call.
1117 MOVQ savedm-8(SP), BX
1118 CMPQ BX, $0
1119 JNE done
1120
1121 // Skip dropm to reuse it in the next call, when a pthread key has been created.
1122 MOVQ _cgo_pthread_key_created(SB), AX
1123 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
1124 CMPQ AX, $0
1125 JEQ dropm
1126 CMPQ (AX), $0
1127 JNE done
1128
1129dropm:
1130 MOVQ $runtime·dropm(SB), AX
1131 CALL AX
1132#ifdef GOOS_windows
1133 // We need to clear the TLS pointer in case the next
1134 // thread that comes into Go tries to reuse that space
1135 // but uses the same M.
1136 XORQ DI, DI
1137 CALL runtime·settls(SB)
1138#endif
1139done:
1140
1141 // Done!
1142 RET
1143
1144// func setg(gg *g)
1145// set g. for use by needm.
1146TEXT runtime·setg(SB), NOSPLIT, $0-8
1147 MOVQ gg+0(FP), BX
1148 get_tls(CX)
1149 MOVQ BX, g(CX)
1150 RET
1151
1152// void setg_gcc(G*); set g called from gcc.
1153TEXT setg_gcc<>(SB),NOSPLIT,$0
1154 get_tls(AX)
1155 MOVQ DI, g(AX)
1156 MOVQ DI, R14 // set the g register
1157 RET
1158
1159TEXT runtime·abort(SB),NOSPLIT,$0-0
1160 INT $3
1161loop:
1162 JMP loop
1163
1164// check that SP is in range [g->stack.lo, g->stack.hi)
1165TEXT runtime·stackcheck(SB), NOSPLIT|NOFRAME, $0-0
1166 get_tls(CX)
1167 MOVQ g(CX), AX
1168 CMPQ (g_stack+stack_hi)(AX), SP
1169 JHI 2(PC)
1170 CALL runtime·abort(SB)
1171 CMPQ SP, (g_stack+stack_lo)(AX)
1172 JHI 2(PC)
1173 CALL runtime·abort(SB)
1174 RET
1175
1176// func cputicks() int64
1177TEXT runtime·cputicks(SB),NOSPLIT,$0-0
1178 CMPB internal∕cpu·X86+const_offsetX86HasRDTSCP(SB), $1
1179 JNE fences
1180 // Instruction stream serializing RDTSCP is supported.
1181 // RDTSCP is supported by Intel Nehalem (2008) and
1182 // AMD K8 Rev. F (2006) and newer.
1183 RDTSCP
1184done:
1185 SHLQ $32, DX
1186 ADDQ DX, AX
1187 MOVQ AX, ret+0(FP)
1188 RET
1189fences:
1190 // MFENCE is instruction stream serializing and flushes the
1191 // store buffers on AMD. The serialization semantics of LFENCE on AMD
1192 // are dependent on MSR C001_1029 and CPU generation.
1193 // LFENCE on Intel does wait for all previous instructions to have executed.
1194 // Intel recommends MFENCE;LFENCE in its manuals before RDTSC to have all
1195 // previous instructions executed and all previous loads and stores to globally visible.
1196 // Using MFENCE;LFENCE here aligns the serializing properties without
1197 // runtime detection of CPU manufacturer.
1198 MFENCE
1199 LFENCE
1200 RDTSC
1201 JMP done
1202
1203// func memhash(p unsafe.Pointer, h, s uintptr) uintptr
1204// hash function using AES hardware instructions
1205TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT,$0-32
1206 // AX = ptr to data
1207 // BX = seed
1208 // CX = size
1209 CMPB runtime·useAeshash(SB), $0
1210 JEQ noaes
1211 JMP aeshashbody<>(SB)
1212noaes:
1213 JMP runtime·memhashFallback<ABIInternal>(SB)
1214
1215// func strhash(p unsafe.Pointer, h uintptr) uintptr
1216TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT,$0-24
1217 // AX = ptr to string struct
1218 // BX = seed
1219 CMPB runtime·useAeshash(SB), $0
1220 JEQ noaes
1221 MOVQ 8(AX), CX // length of string
1222 MOVQ (AX), AX // string data
1223 JMP aeshashbody<>(SB)
1224noaes:
1225 JMP runtime·strhashFallback<ABIInternal>(SB)
1226
1227// AX: data
1228// BX: hash seed
1229// CX: length
1230// At return: AX = return value
1231TEXT aeshashbody<>(SB),NOSPLIT,$0-0
1232 // Fill an SSE register with our seeds.
1233 MOVQ BX, X0 // 64 bits of per-table hash seed
1234 PINSRW $4, CX, X0 // 16 bits of length
1235 PSHUFHW $0, X0, X0 // repeat length 4 times total
1236 MOVO X0, X1 // save unscrambled seed
1237 PXOR runtime·aeskeysched(SB), X0 // xor in per-process seed
1238 AESENC X0, X0 // scramble seed
1239
1240 CMPQ CX, $16
1241 JB aes0to15
1242 JE aes16
1243 CMPQ CX, $32
1244 JBE aes17to32
1245 CMPQ CX, $64
1246 JBE aes33to64
1247 CMPQ CX, $128
1248 JBE aes65to128
1249 JMP aes129plus
1250
1251aes0to15:
1252 TESTQ CX, CX
1253 JE aes0
1254
1255 ADDQ $16, AX
1256 TESTW $0xff0, AX
1257 JE endofpage
1258
1259 // 16 bytes loaded at this address won't cross
1260 // a page boundary, so we can load it directly.
1261 MOVOU -16(AX), X1
1262 ADDQ CX, CX
1263 MOVQ $masks<>(SB), AX
1264 PAND (AX)(CX*8), X1
1265final1:
1266 PXOR X0, X1 // xor data with seed
1267 AESENC X1, X1 // scramble combo 3 times
1268 AESENC X1, X1
1269 AESENC X1, X1
1270 MOVQ X1, AX // return X1
1271 RET
1272
1273endofpage:
1274 // address ends in 1111xxxx. Might be up against
1275 // a page boundary, so load ending at last byte.
1276 // Then shift bytes down using pshufb.
1277 MOVOU -32(AX)(CX*1), X1
1278 ADDQ CX, CX
1279 MOVQ $shifts<>(SB), AX
1280 PSHUFB (AX)(CX*8), X1
1281 JMP final1
1282
1283aes0:
1284 // Return scrambled input seed
1285 AESENC X0, X0
1286 MOVQ X0, AX // return X0
1287 RET
1288
1289aes16:
1290 MOVOU (AX), X1
1291 JMP final1
1292
1293aes17to32:
1294 // make second starting seed
1295 PXOR runtime·aeskeysched+16(SB), X1
1296 AESENC X1, X1
1297
1298 // load data to be hashed
1299 MOVOU (AX), X2
1300 MOVOU -16(AX)(CX*1), X3
1301
1302 // xor with seed
1303 PXOR X0, X2
1304 PXOR X1, X3
1305
1306 // scramble 3 times
1307 AESENC X2, X2
1308 AESENC X3, X3
1309 AESENC X2, X2
1310 AESENC X3, X3
1311 AESENC X2, X2
1312 AESENC X3, X3
1313
1314 // combine results
1315 PXOR X3, X2
1316 MOVQ X2, AX // return X2
1317 RET
1318
1319aes33to64:
1320 // make 3 more starting seeds
1321 MOVO X1, X2
1322 MOVO X1, X3
1323 PXOR runtime·aeskeysched+16(SB), X1
1324 PXOR runtime·aeskeysched+32(SB), X2
1325 PXOR runtime·aeskeysched+48(SB), X3
1326 AESENC X1, X1
1327 AESENC X2, X2
1328 AESENC X3, X3
1329
1330 MOVOU (AX), X4
1331 MOVOU 16(AX), X5
1332 MOVOU -32(AX)(CX*1), X6
1333 MOVOU -16(AX)(CX*1), X7
1334
1335 PXOR X0, X4
1336 PXOR X1, X5
1337 PXOR X2, X6
1338 PXOR X3, X7
1339
1340 AESENC X4, X4
1341 AESENC X5, X5
1342 AESENC X6, X6
1343 AESENC X7, X7
1344
1345 AESENC X4, X4
1346 AESENC X5, X5
1347 AESENC X6, X6
1348 AESENC X7, X7
1349
1350 AESENC X4, X4
1351 AESENC X5, X5
1352 AESENC X6, X6
1353 AESENC X7, X7
1354
1355 PXOR X6, X4
1356 PXOR X7, X5
1357 PXOR X5, X4
1358 MOVQ X4, AX // return X4
1359 RET
1360
1361aes65to128:
1362 // make 7 more starting seeds
1363 MOVO X1, X2
1364 MOVO X1, X3
1365 MOVO X1, X4
1366 MOVO X1, X5
1367 MOVO X1, X6
1368 MOVO X1, X7
1369 PXOR runtime·aeskeysched+16(SB), X1
1370 PXOR runtime·aeskeysched+32(SB), X2
1371 PXOR runtime·aeskeysched+48(SB), X3
1372 PXOR runtime·aeskeysched+64(SB), X4
1373 PXOR runtime·aeskeysched+80(SB), X5
1374 PXOR runtime·aeskeysched+96(SB), X6
1375 PXOR runtime·aeskeysched+112(SB), X7
1376 AESENC X1, X1
1377 AESENC X2, X2
1378 AESENC X3, X3
1379 AESENC X4, X4
1380 AESENC X5, X5
1381 AESENC X6, X6
1382 AESENC X7, X7
1383
1384 // load data
1385 MOVOU (AX), X8
1386 MOVOU 16(AX), X9
1387 MOVOU 32(AX), X10
1388 MOVOU 48(AX), X11
1389 MOVOU -64(AX)(CX*1), X12
1390 MOVOU -48(AX)(CX*1), X13
1391 MOVOU -32(AX)(CX*1), X14
1392 MOVOU -16(AX)(CX*1), X15
1393
1394 // xor with seed
1395 PXOR X0, X8
1396 PXOR X1, X9
1397 PXOR X2, X10
1398 PXOR X3, X11
1399 PXOR X4, X12
1400 PXOR X5, X13
1401 PXOR X6, X14
1402 PXOR X7, X15
1403
1404 // scramble 3 times
1405 AESENC X8, X8
1406 AESENC X9, X9
1407 AESENC X10, X10
1408 AESENC X11, X11
1409 AESENC X12, X12
1410 AESENC X13, X13
1411 AESENC X14, X14
1412 AESENC X15, X15
1413
1414 AESENC X8, X8
1415 AESENC X9, X9
1416 AESENC X10, X10
1417 AESENC X11, X11
1418 AESENC X12, X12
1419 AESENC X13, X13
1420 AESENC X14, X14
1421 AESENC X15, X15
1422
1423 AESENC X8, X8
1424 AESENC X9, X9
1425 AESENC X10, X10
1426 AESENC X11, X11
1427 AESENC X12, X12
1428 AESENC X13, X13
1429 AESENC X14, X14
1430 AESENC X15, X15
1431
1432 // combine results
1433 PXOR X12, X8
1434 PXOR X13, X9
1435 PXOR X14, X10
1436 PXOR X15, X11
1437 PXOR X10, X8
1438 PXOR X11, X9
1439 PXOR X9, X8
1440 // X15 must be zero on return
1441 PXOR X15, X15
1442 MOVQ X8, AX // return X8
1443 RET
1444
1445aes129plus:
1446 // make 7 more starting seeds
1447 MOVO X1, X2
1448 MOVO X1, X3
1449 MOVO X1, X4
1450 MOVO X1, X5
1451 MOVO X1, X6
1452 MOVO X1, X7
1453 PXOR runtime·aeskeysched+16(SB), X1
1454 PXOR runtime·aeskeysched+32(SB), X2
1455 PXOR runtime·aeskeysched+48(SB), X3
1456 PXOR runtime·aeskeysched+64(SB), X4
1457 PXOR runtime·aeskeysched+80(SB), X5
1458 PXOR runtime·aeskeysched+96(SB), X6
1459 PXOR runtime·aeskeysched+112(SB), X7
1460 AESENC X1, X1
1461 AESENC X2, X2
1462 AESENC X3, X3
1463 AESENC X4, X4
1464 AESENC X5, X5
1465 AESENC X6, X6
1466 AESENC X7, X7
1467
1468 // start with last (possibly overlapping) block
1469 MOVOU -128(AX)(CX*1), X8
1470 MOVOU -112(AX)(CX*1), X9
1471 MOVOU -96(AX)(CX*1), X10
1472 MOVOU -80(AX)(CX*1), X11
1473 MOVOU -64(AX)(CX*1), X12
1474 MOVOU -48(AX)(CX*1), X13
1475 MOVOU -32(AX)(CX*1), X14
1476 MOVOU -16(AX)(CX*1), X15
1477
1478 // xor in seed
1479 PXOR X0, X8
1480 PXOR X1, X9
1481 PXOR X2, X10
1482 PXOR X3, X11
1483 PXOR X4, X12
1484 PXOR X5, X13
1485 PXOR X6, X14
1486 PXOR X7, X15
1487
1488 // compute number of remaining 128-byte blocks
1489 DECQ CX
1490 SHRQ $7, CX
1491
1492 PCALIGN $16
1493aesloop:
1494 // scramble state
1495 AESENC X8, X8
1496 AESENC X9, X9
1497 AESENC X10, X10
1498 AESENC X11, X11
1499 AESENC X12, X12
1500 AESENC X13, X13
1501 AESENC X14, X14
1502 AESENC X15, X15
1503
1504 // scramble state, xor in a block
1505 MOVOU (AX), X0
1506 MOVOU 16(AX), X1
1507 MOVOU 32(AX), X2
1508 MOVOU 48(AX), X3
1509 AESENC X0, X8
1510 AESENC X1, X9
1511 AESENC X2, X10
1512 AESENC X3, X11
1513 MOVOU 64(AX), X4
1514 MOVOU 80(AX), X5
1515 MOVOU 96(AX), X6
1516 MOVOU 112(AX), X7
1517 AESENC X4, X12
1518 AESENC X5, X13
1519 AESENC X6, X14
1520 AESENC X7, X15
1521
1522 ADDQ $128, AX
1523 DECQ CX
1524 JNE aesloop
1525
1526 // 3 more scrambles to finish
1527 AESENC X8, X8
1528 AESENC X9, X9
1529 AESENC X10, X10
1530 AESENC X11, X11
1531 AESENC X12, X12
1532 AESENC X13, X13
1533 AESENC X14, X14
1534 AESENC X15, X15
1535 AESENC X8, X8
1536 AESENC X9, X9
1537 AESENC X10, X10
1538 AESENC X11, X11
1539 AESENC X12, X12
1540 AESENC X13, X13
1541 AESENC X14, X14
1542 AESENC X15, X15
1543 AESENC X8, X8
1544 AESENC X9, X9
1545 AESENC X10, X10
1546 AESENC X11, X11
1547 AESENC X12, X12
1548 AESENC X13, X13
1549 AESENC X14, X14
1550 AESENC X15, X15
1551
1552 PXOR X12, X8
1553 PXOR X13, X9
1554 PXOR X14, X10
1555 PXOR X15, X11
1556 PXOR X10, X8
1557 PXOR X11, X9
1558 PXOR X9, X8
1559 // X15 must be zero on return
1560 PXOR X15, X15
1561 MOVQ X8, AX // return X8
1562 RET
1563
1564// func memhash32(p unsafe.Pointer, h uintptr) uintptr
1565// ABIInternal for performance.
1566TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT,$0-24
1567 // AX = ptr to data
1568 // BX = seed
1569 CMPB runtime·useAeshash(SB), $0
1570 JEQ noaes
1571 MOVQ BX, X0 // X0 = seed
1572 PINSRD $2, (AX), X0 // data
1573 AESENC runtime·aeskeysched+0(SB), X0
1574 AESENC runtime·aeskeysched+16(SB), X0
1575 AESENC runtime·aeskeysched+32(SB), X0
1576 MOVQ X0, AX // return X0
1577 RET
1578noaes:
1579 JMP runtime·memhash32Fallback<ABIInternal>(SB)
1580
1581// func memhash64(p unsafe.Pointer, h uintptr) uintptr
1582// ABIInternal for performance.
1583TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT,$0-24
1584 // AX = ptr to data
1585 // BX = seed
1586 CMPB runtime·useAeshash(SB), $0
1587 JEQ noaes
1588 MOVQ BX, X0 // X0 = seed
1589 PINSRQ $1, (AX), X0 // data
1590 AESENC runtime·aeskeysched+0(SB), X0
1591 AESENC runtime·aeskeysched+16(SB), X0
1592 AESENC runtime·aeskeysched+32(SB), X0
1593 MOVQ X0, AX // return X0
1594 RET
1595noaes:
1596 JMP runtime·memhash64Fallback<ABIInternal>(SB)
1597
1598// simple mask to get rid of data in the high part of the register.
1599DATA masks<>+0x00(SB)/8, $0x0000000000000000
1600DATA masks<>+0x08(SB)/8, $0x0000000000000000
1601DATA masks<>+0x10(SB)/8, $0x00000000000000ff
1602DATA masks<>+0x18(SB)/8, $0x0000000000000000
1603DATA masks<>+0x20(SB)/8, $0x000000000000ffff
1604DATA masks<>+0x28(SB)/8, $0x0000000000000000
1605DATA masks<>+0x30(SB)/8, $0x0000000000ffffff
1606DATA masks<>+0x38(SB)/8, $0x0000000000000000
1607DATA masks<>+0x40(SB)/8, $0x00000000ffffffff
1608DATA masks<>+0x48(SB)/8, $0x0000000000000000
1609DATA masks<>+0x50(SB)/8, $0x000000ffffffffff
1610DATA masks<>+0x58(SB)/8, $0x0000000000000000
1611DATA masks<>+0x60(SB)/8, $0x0000ffffffffffff
1612DATA masks<>+0x68(SB)/8, $0x0000000000000000
1613DATA masks<>+0x70(SB)/8, $0x00ffffffffffffff
1614DATA masks<>+0x78(SB)/8, $0x0000000000000000
1615DATA masks<>+0x80(SB)/8, $0xffffffffffffffff
1616DATA masks<>+0x88(SB)/8, $0x0000000000000000
1617DATA masks<>+0x90(SB)/8, $0xffffffffffffffff
1618DATA masks<>+0x98(SB)/8, $0x00000000000000ff
1619DATA masks<>+0xa0(SB)/8, $0xffffffffffffffff
1620DATA masks<>+0xa8(SB)/8, $0x000000000000ffff
1621DATA masks<>+0xb0(SB)/8, $0xffffffffffffffff
1622DATA masks<>+0xb8(SB)/8, $0x0000000000ffffff
1623DATA masks<>+0xc0(SB)/8, $0xffffffffffffffff
1624DATA masks<>+0xc8(SB)/8, $0x00000000ffffffff
1625DATA masks<>+0xd0(SB)/8, $0xffffffffffffffff
1626DATA masks<>+0xd8(SB)/8, $0x000000ffffffffff
1627DATA masks<>+0xe0(SB)/8, $0xffffffffffffffff
1628DATA masks<>+0xe8(SB)/8, $0x0000ffffffffffff
1629DATA masks<>+0xf0(SB)/8, $0xffffffffffffffff
1630DATA masks<>+0xf8(SB)/8, $0x00ffffffffffffff
1631GLOBL masks<>(SB),RODATA,$256
1632
1633// func checkASM() bool
1634TEXT ·checkASM(SB),NOSPLIT,$0-1
1635 // check that masks<>(SB) and shifts<>(SB) are aligned to 16-byte
1636 MOVQ $masks<>(SB), AX
1637 MOVQ $shifts<>(SB), BX
1638 ORQ BX, AX
1639 TESTQ $15, AX
1640 SETEQ ret+0(FP)
1641 RET
1642
1643// these are arguments to pshufb. They move data down from
1644// the high bytes of the register to the low bytes of the register.
1645// index is how many bytes to move.
1646DATA shifts<>+0x00(SB)/8, $0x0000000000000000
1647DATA shifts<>+0x08(SB)/8, $0x0000000000000000
1648DATA shifts<>+0x10(SB)/8, $0xffffffffffffff0f
1649DATA shifts<>+0x18(SB)/8, $0xffffffffffffffff
1650DATA shifts<>+0x20(SB)/8, $0xffffffffffff0f0e
1651DATA shifts<>+0x28(SB)/8, $0xffffffffffffffff
1652DATA shifts<>+0x30(SB)/8, $0xffffffffff0f0e0d
1653DATA shifts<>+0x38(SB)/8, $0xffffffffffffffff
1654DATA shifts<>+0x40(SB)/8, $0xffffffff0f0e0d0c
1655DATA shifts<>+0x48(SB)/8, $0xffffffffffffffff
1656DATA shifts<>+0x50(SB)/8, $0xffffff0f0e0d0c0b
1657DATA shifts<>+0x58(SB)/8, $0xffffffffffffffff
1658DATA shifts<>+0x60(SB)/8, $0xffff0f0e0d0c0b0a
1659DATA shifts<>+0x68(SB)/8, $0xffffffffffffffff
1660DATA shifts<>+0x70(SB)/8, $0xff0f0e0d0c0b0a09
1661DATA shifts<>+0x78(SB)/8, $0xffffffffffffffff
1662DATA shifts<>+0x80(SB)/8, $0x0f0e0d0c0b0a0908
1663DATA shifts<>+0x88(SB)/8, $0xffffffffffffffff
1664DATA shifts<>+0x90(SB)/8, $0x0e0d0c0b0a090807
1665DATA shifts<>+0x98(SB)/8, $0xffffffffffffff0f
1666DATA shifts<>+0xa0(SB)/8, $0x0d0c0b0a09080706
1667DATA shifts<>+0xa8(SB)/8, $0xffffffffffff0f0e
1668DATA shifts<>+0xb0(SB)/8, $0x0c0b0a0908070605
1669DATA shifts<>+0xb8(SB)/8, $0xffffffffff0f0e0d
1670DATA shifts<>+0xc0(SB)/8, $0x0b0a090807060504
1671DATA shifts<>+0xc8(SB)/8, $0xffffffff0f0e0d0c
1672DATA shifts<>+0xd0(SB)/8, $0x0a09080706050403
1673DATA shifts<>+0xd8(SB)/8, $0xffffff0f0e0d0c0b
1674DATA shifts<>+0xe0(SB)/8, $0x0908070605040302
1675DATA shifts<>+0xe8(SB)/8, $0xffff0f0e0d0c0b0a
1676DATA shifts<>+0xf0(SB)/8, $0x0807060504030201
1677DATA shifts<>+0xf8(SB)/8, $0xff0f0e0d0c0b0a09
1678GLOBL shifts<>(SB),RODATA,$256
1679
1680// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
1681// Must obey the gcc calling convention.
1682TEXT _cgo_topofstack(SB),NOSPLIT,$0
1683 get_tls(CX)
1684 MOVQ g(CX), AX
1685 MOVQ g_m(AX), AX
1686 MOVQ m_curg(AX), AX
1687 MOVQ (g_stack+stack_hi)(AX), AX
1688 RET
1689
1690// The top-most function running on a goroutine
1691// returns to goexit+PCQuantum.
1692TEXT runtime·goexit(SB),NOSPLIT|TOPFRAME|NOFRAME,$0-0
1693 BYTE $0x90 // NOP
1694 CALL runtime·goexit1(SB) // does not return
1695 // traceback from goexit1 must hit code range of goexit
1696 BYTE $0x90 // NOP
1697
1698// This is called from .init_array and follows the platform, not Go, ABI.
1699TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
1700 PUSHQ R15 // The access to global variables below implicitly uses R15, which is callee-save
1701 MOVQ runtime·lastmoduledatap(SB), AX
1702 MOVQ DI, moduledata_next(AX)
1703 MOVQ DI, runtime·lastmoduledatap(SB)
1704 POPQ R15
1705 RET
1706
1707// Initialize special registers then jump to sigpanic.
1708// This function is injected from the signal handler for panicking
1709// signals. It is quite painful to set X15 in the signal context,
1710// so we do it here.
1711TEXT ·sigpanic0(SB),NOSPLIT,$0-0
1712 get_tls(R14)
1713 MOVQ g(R14), R14
1714 XORPS X15, X15
1715 JMP ·sigpanic<ABIInternal>(SB)
1716
1717// gcWriteBarrier informs the GC about heap pointer writes.
1718//
1719// gcWriteBarrier returns space in a write barrier buffer which
1720// should be filled in by the caller.
1721// gcWriteBarrier does NOT follow the Go ABI. It accepts the
1722// number of bytes of buffer needed in R11, and returns a pointer
1723// to the buffer space in R11.
1724// It clobbers FLAGS. It does not clobber any general-purpose registers,
1725// but may clobber others (e.g., SSE registers).
1726// Typical use would be, when doing *(CX+88) = AX
1727// CMPL $0, runtime.writeBarrier(SB)
1728// JEQ dowrite
1729// CALL runtime.gcBatchBarrier2(SB)
1730// MOVQ AX, (R11)
1731// MOVQ 88(CX), DX
1732// MOVQ DX, 8(R11)
1733// dowrite:
1734// MOVQ AX, 88(CX)
1735TEXT gcWriteBarrier<>(SB),NOSPLIT,$112
1736 // Save the registers clobbered by the fast path. This is slightly
1737 // faster than having the caller spill these.
1738 MOVQ R12, 96(SP)
1739 MOVQ R13, 104(SP)
1740retry:
1741 // TODO: Consider passing g.m.p in as an argument so they can be shared
1742 // across a sequence of write barriers.
1743 MOVQ g_m(R14), R13
1744 MOVQ m_p(R13), R13
1745 // Get current buffer write position.
1746 MOVQ (p_wbBuf+wbBuf_next)(R13), R12 // original next position
1747 ADDQ R11, R12 // new next position
1748 // Is the buffer full?
1749 CMPQ R12, (p_wbBuf+wbBuf_end)(R13)
1750 JA flush
1751 // Commit to the larger buffer.
1752 MOVQ R12, (p_wbBuf+wbBuf_next)(R13)
1753 // Make return value (the original next position)
1754 SUBQ R11, R12
1755 MOVQ R12, R11
1756 // Restore registers.
1757 MOVQ 96(SP), R12
1758 MOVQ 104(SP), R13
1759 RET
1760
1761flush:
1762 // Save all general purpose registers since these could be
1763 // clobbered by wbBufFlush and were not saved by the caller.
1764 // It is possible for wbBufFlush to clobber other registers
1765 // (e.g., SSE registers), but the compiler takes care of saving
1766 // those in the caller if necessary. This strikes a balance
1767 // with registers that are likely to be used.
1768 //
1769 // We don't have type information for these, but all code under
1770 // here is NOSPLIT, so nothing will observe these.
1771 //
1772 // TODO: We could strike a different balance; e.g., saving X0
1773 // and not saving GP registers that are less likely to be used.
1774 MOVQ DI, 0(SP)
1775 MOVQ AX, 8(SP)
1776 MOVQ BX, 16(SP)
1777 MOVQ CX, 24(SP)
1778 MOVQ DX, 32(SP)
1779 // DI already saved
1780 MOVQ SI, 40(SP)
1781 MOVQ BP, 48(SP)
1782 MOVQ R8, 56(SP)
1783 MOVQ R9, 64(SP)
1784 MOVQ R10, 72(SP)
1785 MOVQ R11, 80(SP)
1786 // R12 already saved
1787 // R13 already saved
1788 // R14 is g
1789 MOVQ R15, 88(SP)
1790
1791 CALL runtime·wbBufFlush(SB)
1792
1793 MOVQ 0(SP), DI
1794 MOVQ 8(SP), AX
1795 MOVQ 16(SP), BX
1796 MOVQ 24(SP), CX
1797 MOVQ 32(SP), DX
1798 MOVQ 40(SP), SI
1799 MOVQ 48(SP), BP
1800 MOVQ 56(SP), R8
1801 MOVQ 64(SP), R9
1802 MOVQ 72(SP), R10
1803 MOVQ 80(SP), R11
1804 MOVQ 88(SP), R15
1805 JMP retry
1806
1807TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1808 MOVL $8, R11
1809 JMP gcWriteBarrier<>(SB)
1810TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1811 MOVL $16, R11
1812 JMP gcWriteBarrier<>(SB)
1813TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1814 MOVL $24, R11
1815 JMP gcWriteBarrier<>(SB)
1816TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1817 MOVL $32, R11
1818 JMP gcWriteBarrier<>(SB)
1819TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1820 MOVL $40, R11
1821 JMP gcWriteBarrier<>(SB)
1822TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1823 MOVL $48, R11
1824 JMP gcWriteBarrier<>(SB)
1825TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1826 MOVL $56, R11
1827 JMP gcWriteBarrier<>(SB)
1828TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
1829 MOVL $64, R11
1830 JMP gcWriteBarrier<>(SB)
1831
1832DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
1833GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
1834
1835// debugCallV2 is the entry point for debugger-injected function
1836// calls on running goroutines. It informs the runtime that a
1837// debug call has been injected and creates a call frame for the
1838// debugger to fill in.
1839//
1840// To inject a function call, a debugger should:
1841// 1. Check that the goroutine is in state _Grunning and that
1842// there are at least 256 bytes free on the stack.
1843// 2. Push the current PC on the stack (updating SP).
1844// 3. Write the desired argument frame size at SP-16 (using the SP
1845// after step 2).
1846// 4. Save all machine registers (including flags and XMM registers)
1847// so they can be restored later by the debugger.
1848// 5. Set the PC to debugCallV2 and resume execution.
1849//
1850// If the goroutine is in state _Grunnable, then it's not generally
1851// safe to inject a call because it may return out via other runtime
1852// operations. Instead, the debugger should unwind the stack to find
1853// the return to non-runtime code, add a temporary breakpoint there,
1854// and inject the call once that breakpoint is hit.
1855//
1856// If the goroutine is in any other state, it's not safe to inject a call.
1857//
1858// This function communicates back to the debugger by setting R12 and
1859// invoking INT3 to raise a breakpoint signal. See the comments in the
1860// implementation for the protocol the debugger is expected to
1861// follow. InjectDebugCall in the runtime tests demonstrates this protocol.
1862//
1863// The debugger must ensure that any pointers passed to the function
1864// obey escape analysis requirements. Specifically, it must not pass
1865// a stack pointer to an escaping argument. debugCallV2 cannot check
1866// this invariant.
1867//
1868// This is ABIInternal because Go code injects its PC directly into new
1869// goroutine stacks.
1870TEXT runtime·debugCallV2<ABIInternal>(SB),NOSPLIT,$152-0
1871 // Save all registers that may contain pointers so they can be
1872 // conservatively scanned.
1873 //
1874 // We can't do anything that might clobber any of these
1875 // registers before this.
1876 MOVQ R15, r15-(14*8+8)(SP)
1877 MOVQ R14, r14-(13*8+8)(SP)
1878 MOVQ R13, r13-(12*8+8)(SP)
1879 MOVQ R12, r12-(11*8+8)(SP)
1880 MOVQ R11, r11-(10*8+8)(SP)
1881 MOVQ R10, r10-(9*8+8)(SP)
1882 MOVQ R9, r9-(8*8+8)(SP)
1883 MOVQ R8, r8-(7*8+8)(SP)
1884 MOVQ DI, di-(6*8+8)(SP)
1885 MOVQ SI, si-(5*8+8)(SP)
1886 MOVQ BP, bp-(4*8+8)(SP)
1887 MOVQ BX, bx-(3*8+8)(SP)
1888 MOVQ DX, dx-(2*8+8)(SP)
1889 // Save the frame size before we clobber it. Either of the last
1890 // saves could clobber this depending on whether there's a saved BP.
1891 MOVQ frameSize-24(FP), DX // aka -16(RSP) before prologue
1892 MOVQ CX, cx-(1*8+8)(SP)
1893 MOVQ AX, ax-(0*8+8)(SP)
1894
1895 // Save the argument frame size.
1896 MOVQ DX, frameSize-128(SP)
1897
1898 // Perform a safe-point check.
1899 MOVQ retpc-8(FP), AX // Caller's PC
1900 MOVQ AX, 0(SP)
1901 CALL runtime·debugCallCheck(SB)
1902 MOVQ 8(SP), AX
1903 TESTQ AX, AX
1904 JZ good
1905 // The safety check failed. Put the reason string at the top
1906 // of the stack.
1907 MOVQ AX, 0(SP)
1908 MOVQ 16(SP), AX
1909 MOVQ AX, 8(SP)
1910 // Set R12 to 8 and invoke INT3. The debugger should get the
1911 // reason a call can't be injected from the top of the stack
1912 // and resume execution.
1913 MOVQ $8, R12
1914 BYTE $0xcc
1915 JMP restore
1916
1917good:
1918 // Registers are saved and it's safe to make a call.
1919 // Open up a call frame, moving the stack if necessary.
1920 //
1921 // Once the frame is allocated, this will set R12 to 0 and
1922 // invoke INT3. The debugger should write the argument
1923 // frame for the call at SP, set up argument registers, push
1924 // the trapping PC on the stack, set the PC to the function to
1925 // call, set RDX to point to the closure (if a closure call),
1926 // and resume execution.
1927 //
1928 // If the function returns, this will set R12 to 1 and invoke
1929 // INT3. The debugger can then inspect any return value saved
1930 // on the stack at SP and in registers and resume execution again.
1931 //
1932 // If the function panics, this will set R12 to 2 and invoke INT3.
1933 // The interface{} value of the panic will be at SP. The debugger
1934 // can inspect the panic value and resume execution again.
1935#define DEBUG_CALL_DISPATCH(NAME,MAXSIZE) \
1936 CMPQ AX, $MAXSIZE; \
1937 JA 5(PC); \
1938 MOVQ $NAME(SB), AX; \
1939 MOVQ AX, 0(SP); \
1940 CALL runtime·debugCallWrap(SB); \
1941 JMP restore
1942
1943 MOVQ frameSize-128(SP), AX
1944 DEBUG_CALL_DISPATCH(debugCall32<>, 32)
1945 DEBUG_CALL_DISPATCH(debugCall64<>, 64)
1946 DEBUG_CALL_DISPATCH(debugCall128<>, 128)
1947 DEBUG_CALL_DISPATCH(debugCall256<>, 256)
1948 DEBUG_CALL_DISPATCH(debugCall512<>, 512)
1949 DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)
1950 DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)
1951 DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)
1952 DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)
1953 DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)
1954 DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)
1955 DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)
1956 // The frame size is too large. Report the error.
1957 MOVQ $debugCallFrameTooLarge<>(SB), AX
1958 MOVQ AX, 0(SP)
1959 MOVQ $20, 8(SP) // length of debugCallFrameTooLarge string
1960 MOVQ $8, R12
1961 BYTE $0xcc
1962 JMP restore
1963
1964restore:
1965 // Calls and failures resume here.
1966 //
1967 // Set R12 to 16 and invoke INT3. The debugger should restore
1968 // all registers except RIP and RSP and resume execution.
1969 MOVQ $16, R12
1970 BYTE $0xcc
1971 // We must not modify flags after this point.
1972
1973 // Restore pointer-containing registers, which may have been
1974 // modified from the debugger's copy by stack copying.
1975 MOVQ ax-(0*8+8)(SP), AX
1976 MOVQ cx-(1*8+8)(SP), CX
1977 MOVQ dx-(2*8+8)(SP), DX
1978 MOVQ bx-(3*8+8)(SP), BX
1979 MOVQ bp-(4*8+8)(SP), BP
1980 MOVQ si-(5*8+8)(SP), SI
1981 MOVQ di-(6*8+8)(SP), DI
1982 MOVQ r8-(7*8+8)(SP), R8
1983 MOVQ r9-(8*8+8)(SP), R9
1984 MOVQ r10-(9*8+8)(SP), R10
1985 MOVQ r11-(10*8+8)(SP), R11
1986 MOVQ r12-(11*8+8)(SP), R12
1987 MOVQ r13-(12*8+8)(SP), R13
1988 MOVQ r14-(13*8+8)(SP), R14
1989 MOVQ r15-(14*8+8)(SP), R15
1990
1991 RET
1992
1993// runtime.debugCallCheck assumes that functions defined with the
1994// DEBUG_CALL_FN macro are safe points to inject calls.
1995#define DEBUG_CALL_FN(NAME,MAXSIZE) \
1996TEXT NAME(SB),WRAPPER,$MAXSIZE-0; \
1997 NO_LOCAL_POINTERS; \
1998 MOVQ $0, R12; \
1999 BYTE $0xcc; \
2000 MOVQ $1, R12; \
2001 BYTE $0xcc; \
2002 RET
2003DEBUG_CALL_FN(debugCall32<>, 32)
2004DEBUG_CALL_FN(debugCall64<>, 64)
2005DEBUG_CALL_FN(debugCall128<>, 128)
2006DEBUG_CALL_FN(debugCall256<>, 256)
2007DEBUG_CALL_FN(debugCall512<>, 512)
2008DEBUG_CALL_FN(debugCall1024<>, 1024)
2009DEBUG_CALL_FN(debugCall2048<>, 2048)
2010DEBUG_CALL_FN(debugCall4096<>, 4096)
2011DEBUG_CALL_FN(debugCall8192<>, 8192)
2012DEBUG_CALL_FN(debugCall16384<>, 16384)
2013DEBUG_CALL_FN(debugCall32768<>, 32768)
2014DEBUG_CALL_FN(debugCall65536<>, 65536)
2015
2016// func debugCallPanicked(val interface{})
2017TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16
2018 // Copy the panic value to the top of stack.
2019 MOVQ val_type+0(FP), AX
2020 MOVQ AX, 0(SP)
2021 MOVQ val_data+8(FP), AX
2022 MOVQ AX, 8(SP)
2023 MOVQ $2, R12
2024 BYTE $0xcc
2025 RET
2026
2027// Note: these functions use a special calling convention to save generated code space.
2028// Arguments are passed in registers, but the space for those arguments are allocated
2029// in the caller's stack frame. These stubs write the args into that stack space and
2030// then tail call to the corresponding runtime handler.
2031// The tail call makes these stubs disappear in backtraces.
2032// Defined as ABIInternal since they do not use the stack-based Go ABI.
2033TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
2034 MOVQ CX, BX
2035 JMP runtime·goPanicIndex<ABIInternal>(SB)
2036TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
2037 MOVQ CX, BX
2038 JMP runtime·goPanicIndexU<ABIInternal>(SB)
2039TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
2040 MOVQ CX, AX
2041 MOVQ DX, BX
2042 JMP runtime·goPanicSliceAlen<ABIInternal>(SB)
2043TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
2044 MOVQ CX, AX
2045 MOVQ DX, BX
2046 JMP runtime·goPanicSliceAlenU<ABIInternal>(SB)
2047TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
2048 MOVQ CX, AX
2049 MOVQ DX, BX
2050 JMP runtime·goPanicSliceAcap<ABIInternal>(SB)
2051TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
2052 MOVQ CX, AX
2053 MOVQ DX, BX
2054 JMP runtime·goPanicSliceAcapU<ABIInternal>(SB)
2055TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
2056 MOVQ CX, BX
2057 JMP runtime·goPanicSliceB<ABIInternal>(SB)
2058TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
2059 MOVQ CX, BX
2060 JMP runtime·goPanicSliceBU<ABIInternal>(SB)
2061TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
2062 MOVQ DX, AX
2063 JMP runtime·goPanicSlice3Alen<ABIInternal>(SB)
2064TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
2065 MOVQ DX, AX
2066 JMP runtime·goPanicSlice3AlenU<ABIInternal>(SB)
2067TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
2068 MOVQ DX, AX
2069 JMP runtime·goPanicSlice3Acap<ABIInternal>(SB)
2070TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
2071 MOVQ DX, AX
2072 JMP runtime·goPanicSlice3AcapU<ABIInternal>(SB)
2073TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
2074 MOVQ CX, AX
2075 MOVQ DX, BX
2076 JMP runtime·goPanicSlice3B<ABIInternal>(SB)
2077TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
2078 MOVQ CX, AX
2079 MOVQ DX, BX
2080 JMP runtime·goPanicSlice3BU<ABIInternal>(SB)
2081TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
2082 MOVQ CX, BX
2083 JMP runtime·goPanicSlice3C<ABIInternal>(SB)
2084TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
2085 MOVQ CX, BX
2086 JMP runtime·goPanicSlice3CU<ABIInternal>(SB)
2087TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16
2088 MOVQ DX, AX
2089 JMP runtime·goPanicSliceConvert<ABIInternal>(SB)
2090
2091#ifdef GOOS_android
2092// Use the free TLS_SLOT_APP slot #2 on Android Q.
2093// Earlier androids are set up in gcc_android.c.
2094DATA runtime·tls_g+0(SB)/8, $16
2095GLOBL runtime·tls_g+0(SB), NOPTR, $8
2096#endif
2097#ifdef GOOS_windows
2098GLOBL runtime·tls_g+0(SB), NOPTR, $8
2099#endif
2100
2101// The compiler and assembler's -spectre=ret mode rewrites
2102// all indirect CALL AX / JMP AX instructions to be
2103// CALL retpolineAX / JMP retpolineAX.
2104// See https://support.google.com/faqs/answer/7625886.
2105#define RETPOLINE(reg) \
2106 /* CALL setup */ BYTE $0xE8; BYTE $(2+2); BYTE $0; BYTE $0; BYTE $0; \
2107 /* nospec: */ \
2108 /* PAUSE */ BYTE $0xF3; BYTE $0x90; \
2109 /* JMP nospec */ BYTE $0xEB; BYTE $-(2+2); \
2110 /* setup: */ \
2111 /* MOVQ AX, 0(SP) */ BYTE $0x48|((reg&8)>>1); BYTE $0x89; \
2112 BYTE $0x04|((reg&7)<<3); BYTE $0x24; \
2113 /* RET */ BYTE $0xC3
2114
2115TEXT runtime·retpolineAX(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(0)
2116TEXT runtime·retpolineCX(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(1)
2117TEXT runtime·retpolineDX(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(2)
2118TEXT runtime·retpolineBX(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(3)
2119/* SP is 4, can't happen / magic encodings */
2120TEXT runtime·retpolineBP(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(5)
2121TEXT runtime·retpolineSI(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(6)
2122TEXT runtime·retpolineDI(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(7)
2123TEXT runtime·retpolineR8(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(8)
2124TEXT runtime·retpolineR9(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(9)
2125TEXT runtime·retpolineR10(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(10)
2126TEXT runtime·retpolineR11(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(11)
2127TEXT runtime·retpolineR12(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(12)
2128TEXT runtime·retpolineR13(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(13)
2129TEXT runtime·retpolineR14(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(14)
2130TEXT runtime·retpolineR15(SB),NOSPLIT|NOFRAME,$0; RETPOLINE(15)
2131
2132TEXT ·getfp<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
2133 MOVQ BP, AX
2134 RET
View as plain text