Current File : //usr/local/go/src/runtime/lockrank_on.go
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build goexperiment.staticlockranking
// +build goexperiment.staticlockranking
package runtime
import (
"runtime/internal/atomic"
"unsafe"
)
// worldIsStopped is accessed atomically to track world-stops. 1 == world
// stopped.
var worldIsStopped uint32
// lockRankStruct is embedded in mutex
type lockRankStruct struct {
// static lock ranking of the lock
rank lockRank
// pad field to make sure lockRankStruct is a multiple of 8 bytes, even on
// 32-bit systems.
pad int
}
func lockInit(l *mutex, rank lockRank) {
l.rank = rank
}
func getLockRank(l *mutex) lockRank {
return l.rank
}
// lockWithRank is like lock(l), but allows the caller to specify a lock rank
// when acquiring a non-static lock.
//
// Note that we need to be careful about stack splits:
//
// This function is not nosplit, thus it may split at function entry. This may
// introduce a new edge in the lock order, but it is no different from any
// other (nosplit) call before this call (including the call to lock() itself).
//
// However, we switch to the systemstack to record the lock held to ensure that
// we record an accurate lock ordering. e.g., without systemstack, a stack
// split on entry to lock2() would record stack split locks as taken after l,
// even though l is not actually locked yet.
func lockWithRank(l *mutex, rank lockRank) {
if l == &debuglock || l == &paniclk {
// debuglock is only used for println/printlock(). Don't do lock
// rank recording for it, since print/println are used when
// printing out a lock ordering problem below.
//
// paniclk is only used for fatal throw/panic. Don't do lock
// ranking recording for it, since we throw after reporting a
// lock ordering problem. Additionally, paniclk may be taken
// after effectively any lock (anywhere we might panic), which
// the partial order doesn't cover.
lock2(l)
return
}
if rank == 0 {
rank = lockRankLeafRank
}
gp := getg()
// Log the new class.
systemstack(func() {
i := gp.m.locksHeldLen
if i >= len(gp.m.locksHeld) {
throw("too many locks held concurrently for rank checking")
}
gp.m.locksHeld[i].rank = rank
gp.m.locksHeld[i].lockAddr = uintptr(unsafe.Pointer(l))
gp.m.locksHeldLen++
// i is the index of the lock being acquired
if i > 0 {
checkRanks(gp, gp.m.locksHeld[i-1].rank, rank)
}
lock2(l)
})
}
// nosplit to ensure it can be called in as many contexts as possible.
//go:nosplit
func printHeldLocks(gp *g) {
if gp.m.locksHeldLen == 0 {
println("<none>")
return
}
for j, held := range gp.m.locksHeld[:gp.m.locksHeldLen] {
println(j, ":", held.rank.String(), held.rank, unsafe.Pointer(gp.m.locksHeld[j].lockAddr))
}
}
// acquireLockRank acquires a rank which is not associated with a mutex lock
//
// This function may be called in nosplit context and thus must be nosplit.
//go:nosplit
func acquireLockRank(rank lockRank) {
gp := getg()
// Log the new class. See comment on lockWithRank.
systemstack(func() {
i := gp.m.locksHeldLen
if i >= len(gp.m.locksHeld) {
throw("too many locks held concurrently for rank checking")
}
gp.m.locksHeld[i].rank = rank
gp.m.locksHeld[i].lockAddr = 0
gp.m.locksHeldLen++
// i is the index of the lock being acquired
if i > 0 {
checkRanks(gp, gp.m.locksHeld[i-1].rank, rank)
}
})
}
// checkRanks checks if goroutine g, which has mostly recently acquired a lock
// with rank 'prevRank', can now acquire a lock with rank 'rank'.
//
//go:systemstack
func checkRanks(gp *g, prevRank, rank lockRank) {
rankOK := false
if rank < prevRank {
// If rank < prevRank, then we definitely have a rank error
rankOK = false
} else if rank == lockRankLeafRank {
// If new lock is a leaf lock, then the preceding lock can
// be anything except another leaf lock.
rankOK = prevRank < lockRankLeafRank
} else {
// We've now verified the total lock ranking, but we
// also enforce the partial ordering specified by
// lockPartialOrder as well. Two locks with the same rank
// can only be acquired at the same time if explicitly
// listed in the lockPartialOrder table.
list := lockPartialOrder[rank]
for _, entry := range list {
if entry == prevRank {
rankOK = true
break
}
}
}
if !rankOK {
printlock()
println(gp.m.procid, " ======")
printHeldLocks(gp)
throw("lock ordering problem")
}
}
// See comment on lockWithRank regarding stack splitting.
func unlockWithRank(l *mutex) {
if l == &debuglock || l == &paniclk {
// See comment at beginning of lockWithRank.
unlock2(l)
return
}
gp := getg()
systemstack(func() {
found := false
for i := gp.m.locksHeldLen - 1; i >= 0; i-- {
if gp.m.locksHeld[i].lockAddr == uintptr(unsafe.Pointer(l)) {
found = true
copy(gp.m.locksHeld[i:gp.m.locksHeldLen-1], gp.m.locksHeld[i+1:gp.m.locksHeldLen])
gp.m.locksHeldLen--
break
}
}
if !found {
println(gp.m.procid, ":", l.rank.String(), l.rank, l)
throw("unlock without matching lock acquire")
}
unlock2(l)
})
}
// releaseLockRank releases a rank which is not associated with a mutex lock
//
// This function may be called in nosplit context and thus must be nosplit.
//go:nosplit
func releaseLockRank(rank lockRank) {
gp := getg()
systemstack(func() {
found := false
for i := gp.m.locksHeldLen - 1; i >= 0; i-- {
if gp.m.locksHeld[i].rank == rank && gp.m.locksHeld[i].lockAddr == 0 {
found = true
copy(gp.m.locksHeld[i:gp.m.locksHeldLen-1], gp.m.locksHeld[i+1:gp.m.locksHeldLen])
gp.m.locksHeldLen--
break
}
}
if !found {
println(gp.m.procid, ":", rank.String(), rank)
throw("lockRank release without matching lockRank acquire")
}
})
}
// See comment on lockWithRank regarding stack splitting.
func lockWithRankMayAcquire(l *mutex, rank lockRank) {
gp := getg()
if gp.m.locksHeldLen == 0 {
// No possibility of lock ordering problem if no other locks held
return
}
systemstack(func() {
i := gp.m.locksHeldLen
if i >= len(gp.m.locksHeld) {
throw("too many locks held concurrently for rank checking")
}
// Temporarily add this lock to the locksHeld list, so
// checkRanks() will print out list, including this lock, if there
// is a lock ordering problem.
gp.m.locksHeld[i].rank = rank
gp.m.locksHeld[i].lockAddr = uintptr(unsafe.Pointer(l))
gp.m.locksHeldLen++
checkRanks(gp, gp.m.locksHeld[i-1].rank, rank)
gp.m.locksHeldLen--
})
}
// nosplit to ensure it can be called in as many contexts as possible.
//go:nosplit
func checkLockHeld(gp *g, l *mutex) bool {
for i := gp.m.locksHeldLen - 1; i >= 0; i-- {
if gp.m.locksHeld[i].lockAddr == uintptr(unsafe.Pointer(l)) {
return true
}
}
return false
}
// assertLockHeld throws if l is not held by the caller.
//
// nosplit to ensure it can be called in as many contexts as possible.
//go:nosplit
func assertLockHeld(l *mutex) {
gp := getg()
held := checkLockHeld(gp, l)
if held {
return
}
// Crash from system stack to avoid splits that may cause
// additional issues.
systemstack(func() {
printlock()
print("caller requires lock ", l, " (rank ", l.rank.String(), "), holding:\n")
printHeldLocks(gp)
throw("not holding required lock!")
})
}
// assertRankHeld throws if a mutex with rank r is not held by the caller.
//
// This is less precise than assertLockHeld, but can be used in places where a
// pointer to the exact mutex is not available.
//
// nosplit to ensure it can be called in as many contexts as possible.
//go:nosplit
func assertRankHeld(r lockRank) {
gp := getg()
for i := gp.m.locksHeldLen - 1; i >= 0; i-- {
if gp.m.locksHeld[i].rank == r {
return
}
}
// Crash from system stack to avoid splits that may cause
// additional issues.
systemstack(func() {
printlock()
print("caller requires lock with rank ", r.String(), "), holding:\n")
printHeldLocks(gp)
throw("not holding required lock!")
})
}
// worldStopped notes that the world is stopped.
//
// Caller must hold worldsema.
//
// nosplit to ensure it can be called in as many contexts as possible.
//go:nosplit
func worldStopped() {
if stopped := atomic.Xadd(&worldIsStopped, 1); stopped != 1 {
systemstack(func() {
print("world stop count=", stopped, "\n")
throw("recursive world stop")
})
}
}
// worldStarted that the world is starting.
//
// Caller must hold worldsema.
//
// nosplit to ensure it can be called in as many contexts as possible.
//go:nosplit
func worldStarted() {
if stopped := atomic.Xadd(&worldIsStopped, -1); stopped != 0 {
systemstack(func() {
print("world stop count=", stopped, "\n")
throw("released non-stopped world stop")
})
}
}
// nosplit to ensure it can be called in as many contexts as possible.
//go:nosplit
func checkWorldStopped() bool {
stopped := atomic.Load(&worldIsStopped)
if stopped > 1 {
systemstack(func() {
print("inconsistent world stop count=", stopped, "\n")
throw("inconsistent world stop count")
})
}
return stopped == 1
}
// assertWorldStopped throws if the world is not stopped. It does not check
// which M stopped the world.
//
// nosplit to ensure it can be called in as many contexts as possible.
//go:nosplit
func assertWorldStopped() {
if checkWorldStopped() {
return
}
throw("world not stopped")
}
// assertWorldStoppedOrLockHeld throws if the world is not stopped and the
// passed lock is not held.
//
// nosplit to ensure it can be called in as many contexts as possible.
//go:nosplit
func assertWorldStoppedOrLockHeld(l *mutex) {
if checkWorldStopped() {
return
}
gp := getg()
held := checkLockHeld(gp, l)
if held {
return
}
// Crash from system stack to avoid splits that may cause
// additional issues.
systemstack(func() {
printlock()
print("caller requires world stop or lock ", l, " (rank ", l.rank.String(), "), holding:\n")
println("<no world stop>")
printHeldLocks(gp)
throw("no world stop or required lock!")
})
}
Mini Shell