Go's defer and panic: Open-Coded Defers and the Runtime's Unwind Loop

// src/runtime/panic.go:38
// We have two different ways of doing defers. The older way involves creating a
// defer record at the time that a defer statement is executing and adding it to a
// defer chain. This chain is inspected by the deferreturn call at all function
// exits in order to run the appropriate defer calls. A cheaper way (which we call
// open-coded defers) is used for functions in which no defer statements occur in
// loops. In that case, we simply store the defer function/arg information into
// specific stack slots at the point of each defer statement, as well as setting a
// bit in a bitmask. At each function exit, we add inline code to directly make
// the appropriate defer calls based on the bitmask and fn/arg information stored
// on the stack. During panic/Goexit processing, the appropriate defer calls are
// made using extra funcdata info that indicates the exact stack slots that
// contain the bitmask and defer fn/args.

// src/runtime/runtime2.go:1150
// A _defer holds an entry on the list of deferred calls.
// If you add a field here, add code to clear it in deferProcStack.
// This struct must match the code in cmd/compile/internal/ssagen/ssa.go:deferstruct
// and cmd/compile/internal/ssagen/ssa.go:(*state).call.
// Some defers will be allocated on the stack and some on the heap.
// All defers are logically part of the stack, so write barriers to
// initialize them are not required. All defers must be manually scanned,
// and for heap defers, marked.
type _defer struct {
	heap      bool
	rangefunc bool    // true for rangefunc list
	sp        uintptr // sp at time of defer
	pc        uintptr // pc at time of defer
	fn        func()  // can be nil for open-coded defers
	link      *_defer // next defer on G; can point to either heap or stack!

	// If rangefunc is true, *head is the head of the atomic linked list
	// during a range-over-func execution.
	head *atomic.Pointer[_defer]
}

// src/cmd/compile/internal/ssagen/ssa.go:449
	if s.hasOpenDefers {
		// Create the deferBits variable and stack slot.  deferBits is a
		// bitmask showing which of the open-coded defers in this function
		// have been activated.
		deferBitsTemp := typecheck.TempAt(src.NoXPos, s.curfn, types.Types[types.TUINT8])
		deferBitsTemp.SetAddrtaken(true)
		s.deferBitsTemp = deferBitsTemp
		// For this value, AuxInt is initialized to zero by default
		startDeferBits := s.entryNewValue0(ssaop.OpConst8, types.Types[types.TUINT8])
		s.vars[deferBitsVar] = startDeferBits
		s.deferBitsAddr = s.addr(deferBitsTemp)
		s.store(types.Types[types.TUINT8], s.deferBitsAddr, startDeferBits)

// src/cmd/compile/internal/walk/walk.go:306
// The max number of defers in a function using open-coded defers. We enforce this
// limit because the deferBits bitmask is currently a single byte (to minimize code size)
const maxOpenDefers = 8

// src/cmd/compile/internal/ssagen/ssa.go:4868
	// Update deferBits only after evaluation and storage to stack of
	// the function is successful.
	bitvalue := s.constInt8(types.Types[types.TUINT8], 1<<uint(index))
	newDeferBits := s.newValue2(ssaop.OpOr8, types.Types[types.TUINT8], s.variable(deferBitsVar, types.Types[types.TUINT8]), bitvalue)
	s.vars[deferBitsVar] = newDeferBits
	s.store(types.Types[types.TUINT8], s.deferBitsAddr, newDeferBits)

   frame of f()
   +----------------------+
   | deferBits: uint8     |   bit 0 = defer #1 armed
   |                      |   bit 1 = defer #2 armed
   |                      |   ...
   +----------------------+
   | slot 0: func()       |   closure for defer #1
   | slot 1: func()       |   closure for defer #2
   | slot 2: func()       |   closure for defer #3
   | slot 3: func()       |   closure for defer #4
   +----------------------+

// src/runtime/panic.go:655
// deferreturn runs deferred functions for the caller's frame.
// The compiler inserts a call to this at the end of any
// function which calls defer.
func deferreturn() {
	var p _panic
	p.deferreturn = true

	p.start(sys.GetCallerPC(), unsafe.Pointer(sys.GetCallerSP()))
	for {
		fn, ok := p.nextDefer()
		if !ok {
			break
		}
		fn()
	}
}

// src/cmd/compile/internal/ssagen/ssa.go:7282
	if openDeferInfo != nil {
		// When doing open-coded defers, generate a disconnected call to
		// deferreturn and a return. This will be used to during panic
		// recovery to unwind the stack and return back to the runtime.

// src/cmd/compile/internal/ssagen/ssa.go:1738
		if s.hasOpenDefers {
			s.openDeferRecord(n.Call.(*ir.CallExpr))
		} else {
			d := callDefer
			if n.Esc() == ir.EscNever && n.DeferAt == nil {
				d = callDeferStack
			}
			s.call(n.Call.(*ir.CallExpr), d, false, n.DeferAt)
		}

// src/runtime/panic.go:538
// deferprocStack queues a new deferred function with a defer record on the stack.
// The defer record must have its fn field initialized.
// All other fields can contain junk.
// Nosplit because of the uninitialized pointer fields on the stack.
//
//go:nosplit
func deferprocStack(d *_defer) {
	gp := getg()
	if gp.m.curg != gp {
		// go code on the system stack can't defer
		throw("defer on system stack")
	}

	// fn is already set.
	// The other fields are junk on entry to deferprocStack and
	// are initialized here.
	d.heap = false
	d.rangefunc = false
	d.sp = sys.GetCallerSP()
	d.pc = sys.GetCallerPC()
	// The lines below implement:
	//   d.link = gp._defer
	//   d.head = nil
	//   gp._defer = d
	// But without write barriers. The first two are writes to
	// the stack so they don't need a write barrier, and furthermore
	// are to uninitialized memory, so they must not use a write barrier.
	// The third write does not require a write barrier because we
	// explicitly mark all the defer structures, so we don't need to
	// keep track of pointers to them with a write barrier.
	*(*uintptr)(unsafe.Pointer(&d.link)) = uintptr(unsafe.Pointer(gp._defer))
	*(*uintptr)(unsafe.Pointer(&d.head)) = 0
	*(*uintptr)(unsafe.Pointer(&gp._defer)) = uintptr(unsafe.Pointer(d))
}

// src/runtime/panic.go:351
// Create a new deferred function fn, which has no arguments and results.
// The compiler turns a defer statement into a call to this.
func deferproc(fn func()) {
	gp := getg()
	if gp.m.curg != gp {
		// go code on the system stack can't defer
		throw("defer on system stack")
	}

	d := newdefer()
	d.link = gp._defer
	gp._defer = d
	d.fn = fn
	d.pc = sys.GetCallerPC()
	// We must not be preempted between calling GetCallerSP and
	// storing it to d.sp because GetCallerSP's result is a
	// uintptr stack pointer.
	d.sp = sys.GetCallerSP()
}

// src/cmd/compile/internal/escape/call.go:237
// goDeferStmt analyzes a "go" or "defer" statement.
func (e *escape) goDeferStmt(n *ir.GoDeferStmt) {
	k := e.heapHole()
	if n.Op() == ir.ODEFER && e.loopDepth == 1 && n.DeferAt == nil {
		// Top-level defer arguments don't escape to the heap,
		// but they do need to last until they're invoked.
		k = e.later(e.discardHole())

		// force stack allocation of defer record, unless
		// open-coded defers are used (see ssa.go)
		n.SetEsc(ir.EscNever)
	}

// src/cmd/compile/internal/walk/stmt.go:106
	case ir.ODEFER:
		n := n.(*ir.GoDeferStmt)
		w.curfunc.SetHasDefer(true)
		w.curfunc.NumDefers++
		if w.curfunc.NumDefers > maxOpenDefers || n.DeferAt != nil {
			// Don't allow open-coded defers if there are more than
			// 8 defers in the function, since we use a single
			// byte to record active defers.
			// Also don't allow if we need to use deferprocat.
			w.curfunc.SetOpenCodedDeferDisallowed(true)
		}
		if n.Esc() != ir.EscNever {
			// If n.Esc is not EscNever, then this defer occurs in a loop,
			// so open-coded defers cannot be used in this function.
			w.curfunc.SetOpenCodedDeferDisallowed(true)
		}

// src/runtime/panic.go:799
// The implementation of the predeclared function panic.
// The compiler emits calls to this function.
//
// gopanic should be an internal detail,
// but historically, widely used packages access it using linkname.
//
// Do not remove or change the type signature.
// See go.dev/issue/67401.
//
//go:linkname gopanic
func gopanic(e any) {
	if e == nil {
		if debug.panicnil.Load() != 1 {
			e = new(PanicNilError)
		} else {
			panicnil.IncNonDefault()
		}
	}

// src/runtime/panic.go:848
	var p _panic
	p.arg = e

	runningPanicDefers.Add(1)

	p.start(sys.GetCallerPC(), unsafe.Pointer(sys.GetCallerSP()))
	for {
		fn, ok := p.nextDefer()
		if !ok {
			break
		}
		fn()
	}

// src/runtime/panic.go:944
	for {
		for p.deferBitsPtr != nil {
			bits := *p.deferBitsPtr

			// Check whether any open-coded defers are still pending.
			//
			// Note: We need to check this upfront (rather than after
			// clearing the top bit) because it's possible that Goexit
			// invokes a deferred call, and there were still more pending
			// open-coded defers in the frame; but then the deferred call
			// panic and invoked the remaining defers in the frame, before
			// recovering and restarting the Goexit loop.
			if bits == 0 {
				p.deferBitsPtr = nil
				break
			}

			// Find index of top bit set.
			i := 7 - uintptr(sys.LeadingZeros8(bits))

			// Clear bit and store it back.
			bits &^= 1 << i
			*p.deferBitsPtr = bits

			return *(*func())(add(p.slotsPtr, i*goarch.PtrSize)), true
		}

	Recheck:
		if d := gp._defer; d != nil && d.sp == uintptr(p.sp) {
			if d.rangefunc {
				deferconvert(d)
				popDefer(gp)
				goto Recheck
			}

			fn := d.fn

			p.retpc = d.pc

			// Unlink and free.
			popDefer(gp)

			return fn, true
		}

		if !p.nextFrame() {
			return nil, false
		}
	}

// src/runtime/panic.go:1059
func (p *_panic) initOpenCodedDefers(fn funcInfo, varp unsafe.Pointer) bool {
	fd := funcdata(fn, abi.FUNCDATA_OpenCodedDeferInfo)
	if fd == nil {
		return false
	}

	if fn.deferreturn == 0 {
		throw("missing deferreturn")
	}

	deferBitsOffset, fd := readvarintUnsafe(fd)
	deferBitsPtr := (*uint8)(add(varp, -uintptr(deferBitsOffset)))
	if *deferBitsPtr == 0 {
		return false // has open-coded defers, but none pending
	}

	slotsOffset, fd := readvarintUnsafe(fd)

	p.retpc = fn.entry() + uintptr(fn.deferreturn)
	p.deferBitsPtr = deferBitsPtr
	p.slotsPtr = add(varp, -uintptr(slotsOffset))

	return true
}

// src/runtime/panic.go:1084
// The implementation of the predeclared function recover.
func gorecover() any {
	p := recoverablePanic(3) // skip systemstack_switch, recoverablePanic, gorecover
	if p == nil {
		return nil
	}
	p.recovered = true
	return p.arg
}

// src/runtime/panic.go:1179
// So our criteria is, there must be exactly one non-wrapper
// frame between gopanic and gorecover.
//
// We don't recover this:
//
//	defer func() { func() { recover() }() }()
//
// because there are 2 non-wrapper frames.
//
// We don't recover this:
//
//	defer recover()
//
// because there are 0 non-wrapper frames.
//
// It must not be inlined into its callers: they count on its frame being
// there to skip.
//
//go:noinline
func recoverablePanic(skip int) *_panic {
	gp := getg()
	p := gp._panic
	if p == nil || p.goexit || p.recovered {
		return nil
	}

	canRecover := false
	systemstack(func() {
		var u unwinder
		u.init(gp, 0)
		for i := 0; i < skip; i++ {
			u.next()
		}
		nonWrapperFrames := 0
	loop:
		for ; u.valid(); u.next() {
			for iu, f := newInlineUnwinder(u.frame.fn, u.symPC()); f.valid(); f = iu.next(f) {
				sf := iu.srcFunc(f)
				switch sf.funcID {
				case abi.FuncIDWrapper:
					continue
				case abi.FuncID_gopanic:
					if u.frame.sp == uintptr(p.startSP) && nonWrapperFrames > 0 {
						canRecover = true
					}
					break loop
				default:
					nonWrapperFrames++
					if nonWrapperFrames > 1 {
						break loop
					}
				}
			}
		}
	})
	if !canRecover {
		return nil
	}
	return p
}

// src/runtime/panic.go:779
// A PanicNilError happens when code calls panic(nil).
//
// Before Go 1.21, programs that called panic(nil) observed recover returning nil.
// Starting in Go 1.21, programs that call panic(nil) observe recover returning a *PanicNilError.
// Programs can change back to the old behavior by setting GODEBUG=panicnil=1.
type PanicNilError struct {
	// This field makes PanicNilError structurally different from
	// any other struct in this package, and the _ makes it different
	// from any struct in other packages too.
	// This avoids any accidental conversions being possible
	// between this struct and some other struct sharing the same fields,
	// like happened in go.dev/issue/56603.
	_ [0]*PanicNilError
}

func (*PanicNilError) Error() string { return "runtime error: panic called with nil argument" }
func (*PanicNilError) RuntimeError() {}

// src/runtime/panic.go:672
// Goexit terminates the goroutine that calls it. No other goroutine is affected.
// Goexit runs all deferred calls before terminating the goroutine. Because Goexit
// is not a panic, any recover calls in those deferred functions will return nil.
//
// Calling Goexit from the main goroutine terminates that goroutine
// without func main returning. Since func main has not returned,
// the program continues execution of other goroutines.
// If all other goroutines exit, the program crashes.
//
// It crashes if called from a thread not created by the Go runtime.
func Goexit() {
	// Create a panic object for Goexit, so we can recognize when it might be
	// bypassed by a recover().
	var p _panic
	p.goexit = true

	p.start(sys.GetCallerPC(), unsafe.Pointer(sys.GetCallerSP()))
	for {
		fn, ok := p.nextDefer()
		if !ok {
			break
		}
		fn()
	}

	goexit1()
}