diff --git a/api/go1.14.txt b/api/go1.14.txt index 3af0fee3b4cad..b3d339b81fecc 100644 --- a/api/go1.14.txt +++ b/api/go1.14.txt @@ -174,6 +174,12 @@ pkg net/http, method (Header) Values(string) []string pkg net/http, type Transport struct, DialTLSContext func(context.Context, string, string) (net.Conn, error) pkg net/http/httptest, type Server struct, EnableHTTP2 bool pkg net/textproto, method (MIMEHeader) Values(string) []string +pkg runtime/debug, func ReadGCPolicy(*GCPolicy) +pkg runtime/debug, func SetMaxHeap(uintptr, chan<- struct) uintptr +pkg runtime/debug, type GCPolicy struct +pkg runtime/debug, type GCPolicy struct, AvailGCPercent int +pkg runtime/debug, type GCPolicy struct, GCPercent int +pkg runtime/debug, type GCPolicy struct, MaxHeapBytes uintptr pkg strconv, method (*NumError) Unwrap() error pkg syscall (windows-386), const CTRL_CLOSE_EVENT = 2 pkg syscall (windows-386), const CTRL_CLOSE_EVENT ideal-int diff --git a/src/runtime/debug/garbage.go b/src/runtime/debug/garbage.go index 785e9d4598eac..d4746fb123afc 100644 --- a/src/runtime/debug/garbage.go +++ b/src/runtime/debug/garbage.go @@ -87,7 +87,10 @@ func ReadGCStats(stats *GCStats) { // SetGCPercent returns the previous setting. // The initial setting is the value of the GOGC environment variable // at startup, or 100 if the variable is not set. -// A negative percentage disables garbage collection. +// A negative percentage disables triggering garbage collection +// based on the ratio of fresh allocation to previously live heap. +// However, GC can still be explicitly triggered by runtime.GC and +// similar functions, or by the maximum heap size set by SetMaxHeap. func SetGCPercent(percent int) int { return int(setGCPercent(int32(percent))) } @@ -166,3 +169,114 @@ func WriteHeapDump(fd uintptr) // If SetTraceback is called with a level lower than that of the // environment variable, the call is ignored. func SetTraceback(level string) + +// GCPolicy reports the garbage collector's policy for controlling the +// heap size and scheduling garbage collection work. +type GCPolicy struct { + // GCPercent is the current value of GOGC, as set by the GOGC + // environment variable or SetGCPercent. + // + // If triggering GC by relative heap growth is disabled, this + // will be -1. + GCPercent int + + // MaxHeapBytes is the current soft heap limit set by + // SetMaxHeap, in bytes. + // + // If there is no heap limit set, this will be ^uintptr(0). + MaxHeapBytes uintptr + + // AvailGCPercent is the heap space available for allocation + // before the next GC, as a percent of the heap used at the + // end of the previous garbage collection. It measures memory + // pressure and how hard the garbage collector must work to + // achieve the heap size goals set by GCPercent and + // MaxHeapBytes. + // + // For example, if AvailGCPercent is 100, then at the end of + // the previous garbage collection, the space available for + // allocation before the next GC was the same as the space + // used. If AvailGCPercent is 20, then the space available is + // only a 20% of the space used. + // + // AvailGCPercent is directly comparable with GCPercent. + // + // If AvailGCPercent >= GCPercent, the garbage collector is + // not under pressure and can amortize the cost of garbage + // collection by allowing the heap to grow in proportion to + // how much is used. + // + // If AvailGCPercent < GCPercent, the garbage collector is + // under pressure and must run more frequently to keep the + // heap size under MaxHeapBytes. Smaller values of + // AvailGCPercent indicate greater pressure. In this case, the + // application should shed load and reduce its live heap size + // to relieve memory pressure. + // + // AvailGCPercent is always >= 0. + AvailGCPercent int +} + +// SetMaxHeap sets a soft limit on the size of the Go heap and returns +// the previous setting. By default, there is no limit. +// +// If a max heap is set, the garbage collector will endeavor to keep +// the heap size under the specified size, even if this is lower than +// would normally be determined by GOGC (see SetGCPercent). +// +// Whenever the garbage collector's scheduling policy changes as a +// result of this heap limit (that is, the result that would be +// returned by ReadGCPolicy changes), the garbage collector will send +// to the notify channel. This is a non-blocking send, so this should +// be a single-element buffered channel, though this is not required. +// Only a single channel may be registered for notifications at a +// time; SetMaxHeap replaces any previously registered channel. +// +// The application is strongly encouraged to respond to this +// notification by calling ReadGCPolicy and, if AvailGCPercent is less +// than GCPercent, shedding load to reduce its live heap size. Setting +// a maximum heap size limits the garbage collector's ability to +// amortize the cost of garbage collection when the heap reaches the +// heap size limit. This is particularly important in +// request-processing systems, where increasing pressure on the +// garbage collector reduces CPU time available to the application, +// making it less able to complete work, leading to even more pressure +// on the garbage collector. The application must shed load to avoid +// this "GC death spiral". +// +// The limit set by SetMaxHeap is soft. If the garbage collector would +// consume too much CPU to keep the heap under this limit (leading to +// "thrashing"), it will allow the heap to grow larger than the +// specified max heap. +// +// The heap size does not include everything in the process's memory +// footprint. Notably, it does not include stacks, C-allocated memory, +// or many runtime-internal structures. +// +// To disable the heap limit, pass ^uintptr(0) for the bytes argument. +// In this case, notify can be nil. +// +// To depend only on the heap limit to trigger garbage collection, +// call SetGCPercent(-1) after setting a heap limit. +func SetMaxHeap(bytes uintptr, notify chan<- struct{}) uintptr { + if bytes == ^uintptr(0) { + return gcSetMaxHeap(bytes, nil) + } + if notify == nil { + panic("SetMaxHeap requires a non-nil notify channel") + } + return gcSetMaxHeap(bytes, notify) +} + +// gcSetMaxHeap is provided by package runtime. +func gcSetMaxHeap(bytes uintptr, notify chan<- struct{}) uintptr + +// ReadGCPolicy reads the garbage collector's current policy for +// managing the heap size. This includes static settings controlled by +// the application and dynamic policy determined by heap usage. +func ReadGCPolicy(gcp *GCPolicy) { + gcp.GCPercent, gcp.MaxHeapBytes, gcp.AvailGCPercent = gcReadPolicy() +} + +// gcReadPolicy is provided by package runtime. +func gcReadPolicy() (gogc int, maxHeap uintptr, egogc int) diff --git a/src/runtime/debug/garbage_test.go b/src/runtime/debug/garbage_test.go index 69e769ecf29d3..f1e20b1f58798 100644 --- a/src/runtime/debug/garbage_test.go +++ b/src/runtime/debug/garbage_test.go @@ -179,6 +179,114 @@ func abs64(a int64) int64 { return a } +func TestSetMaxHeap(t *testing.T) { + defer func() { + setGCPercentBallast = nil + setGCPercentSink = nil + SetMaxHeap(^uintptr(0), nil) + runtime.GC() + }() + + runtime.GC() + var m1 runtime.MemStats + runtime.ReadMemStats(&m1) + + // Create 50 MB of live heap as a baseline. + const baseline = 50 << 20 + setGCPercentBallast = make([]byte, baseline-m1.Alloc) + // Disable GOGC-based policy. + defer SetGCPercent(SetGCPercent(-1)) + // Set max heap to 2x baseline. + const limit = 2 * baseline + notify := make(chan struct{}, 1) + prev := SetMaxHeap(limit, notify) + + // Check that that notified us of heap pressure. + select { + case <-notify: + default: + t.Errorf("missing GC pressure notification") + } + + // Test return value. + if prev != ^uintptr(0) { + t.Errorf("want previous limit %d, got %d", ^uintptr(0), prev) + } + prev = SetMaxHeap(limit, notify) + if prev != limit { + t.Errorf("want previous limit %d, got %d", limit, prev) + } + + // Allocate a bunch and check that we stay under the limit. + runtime.ReadMemStats(&m1) + var m2 runtime.MemStats + var gcp GCPolicy + for i := 0; i < 200; i++ { + setGCPercentSink = make([]byte, 1<<20) + runtime.ReadMemStats(&m2) + if m2.HeapAlloc > limit { + t.Fatalf("HeapAlloc %d exceeds heap limit %d", m2.HeapAlloc, limit) + } + ReadGCPolicy(&gcp) + if gcp.GCPercent != -1 { + t.Fatalf("want GCPercent %d, got policy %+v", -1, gcp) + } + if gcp.MaxHeapBytes != limit { + t.Fatalf("want MaxHeapBytes %d, got policy %+v", limit, gcp) + } + const availErr = 10 + availLow := 100*(limit-baseline)/baseline - availErr + availHigh := 100*(limit-baseline)/baseline + availErr + if !(availLow <= gcp.AvailGCPercent && gcp.AvailGCPercent <= availHigh) { + t.Fatalf("AvailGCPercent %d out of range [%d, %d]", gcp.AvailGCPercent, availLow, availHigh) + } + } + if m1.NumGC == m2.NumGC { + t.Fatalf("failed to trigger GC") + } +} + +func TestAvailGCPercent(t *testing.T) { + defer func() { + SetMaxHeap(^uintptr(0), nil) + runtime.GC() + }() + + runtime.GC() + + // Set GOGC=100. + defer SetGCPercent(SetGCPercent(100)) + // Set max heap to 100MB. + const limit = 100 << 20 + SetMaxHeap(limit, make(chan struct{})) + + // Allocate a bunch and monitor AvailGCPercent. + var m runtime.MemStats + var gcp GCPolicy + sl := [][]byte{} + for i := 0; i < 200; i++ { + sl = append(sl, make([]byte, 1<<20)) + runtime.GC() + runtime.ReadMemStats(&m) + ReadGCPolicy(&gcp) + // Use int64 to avoid overflow on 32-bit. + avail := int(100 * (limit - int64(m.HeapAlloc)) / int64(m.HeapAlloc)) + if avail > 100 { + avail = 100 + } + if avail < 10 { + avail = 10 + } + const availErr = 2 // This is more controlled than the test above. + availLow := avail - availErr + availHigh := avail + availErr + if !(availLow <= gcp.AvailGCPercent && gcp.AvailGCPercent <= availHigh) { + t.Logf("MemStats: %+v\nGCPolicy: %+v\n", m, gcp) + t.Fatalf("AvailGCPercent %d out of range [%d, %d]", gcp.AvailGCPercent, availLow, availHigh) + } + } +} + func TestSetMaxThreadsOvf(t *testing.T) { // Verify that a big threads count will not overflow the int32 // maxmcount variable, causing a panic (see Issue 16076). diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index b3499516f65a4..11e6855a2a8cc 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -169,6 +169,9 @@ const defaultHeapMinimum = 4 << 20 // Initialized from $GOGC. GOGC=off means no GC. var gcpercent int32 +// To begin with, maxHeap is infinity. +var maxHeap uintptr = ^uintptr(0) + func gcinit() { if unsafe.Sizeof(workbuf{}) != _WorkbufSize { throw("size of Workbuf is suboptimal") @@ -185,6 +188,9 @@ func gcinit() { // This will go into computing the initial GC goal. memstats.heap_marked = uint64(float64(heapminimum) / (1 + memstats.triggerRatio)) + // Disable heap limit initially. + gcPressure.maxHeap = ^uintptr(0) + // Set gcpercent from the environment. This will also compute // and set the GC trigger and goal. _ = setGCPercent(readgogc()) @@ -224,6 +230,7 @@ func gcenable() { //go:linkname setGCPercent runtime/debug.setGCPercent func setGCPercent(in int32) (out int32) { // Run on the system stack since we grab the heap lock. + var updated bool systemstack(func() { lock(&mheap_.lock) out = gcpercent @@ -231,12 +238,20 @@ func setGCPercent(in int32) (out int32) { in = -1 } gcpercent = in - heapminimum = defaultHeapMinimum * uint64(gcpercent) / 100 + if gcpercent >= 0 { + heapminimum = defaultHeapMinimum * uint64(gcpercent) / 100 + } else { + heapminimum = 0 + } // Update pacing in response to gcpercent change. - gcSetTriggerRatio(memstats.triggerRatio) + updated = gcSetTriggerRatio(memstats.triggerRatio) unlock(&mheap_.lock) }) + if updated { + gcPolicyNotify() + } + // If we just disabled GC, wait for any concurrent GC mark to // finish so we always return with no GC running. if in < 0 { @@ -246,6 +261,59 @@ func setGCPercent(in int32) (out int32) { return out } +var gcPressure struct { + // lock may be acquired while mheap_.lock is held. Hence, it + // must only be acquired from the system stack. + lock mutex + + // notify is a notification channel for GC pressure changes + // with a notification sent after every gcSetTriggerRatio. + // It is provided by package debug. It may be nil. + notify chan<- struct{} + + // Together gogc, maxHeap, and egogc represent the GC policy. + // + // gogc is GOGC, maxHeap is the GC heap limit, and egogc is the effective GOGC. + // + // These are set by the user with debug.SetMaxHeap. GC will + // attempt to keep heap_live under maxHeap, even if it has to + // violate GOGC (up to a point). + gogc int + maxHeap uintptr + egogc int +} + +//go:linkname gcSetMaxHeap runtime/debug.gcSetMaxHeap +func gcSetMaxHeap(bytes uintptr, notify chan<- struct{}) uintptr { + var ( + prev uintptr + updated bool + ) + systemstack(func() { + // gcPressure.notify has a write barrier on it so it must be protected + // by gcPressure's lock instead of mheap's, otherwise we could deadlock. + lock(&gcPressure.lock) + gcPressure.notify = notify + unlock(&gcPressure.lock) + + lock(&mheap_.lock) + + // Update max heap. + prev = maxHeap + maxHeap = bytes + + // Update pacing. This will update gcPressure from the + // globals gcpercent and maxHeap. + updated = gcSetTriggerRatio(memstats.triggerRatio) + + unlock(&mheap_.lock) + }) + if updated { + gcPolicyNotify() + } + return prev +} + // Garbage collector phase. // Indicates to write barrier and synchronization task to perform. var gcphase uint32 @@ -757,27 +825,84 @@ func pollFractionalWorkerExit() bool { // This can be called any time. If GC is the in the middle of a // concurrent phase, it will adjust the pacing of that phase. // -// This depends on gcpercent, memstats.heap_marked, and -// memstats.heap_live. These must be up to date. +// This depends on gcpercent, mheap_.maxHeap, memstats.heap_marked, +// and memstats.heap_live. These must be up to date. +// +// Returns whether or not there was a change in the GC policy. +// If it returns true, the caller must call gcPolicyNotify() after +// releasing the heap lock. // // mheap_.lock must be held or the world must be stopped. -func gcSetTriggerRatio(triggerRatio float64) { +// +// This must be called on the system stack because it acquires +// gcPressure.lock. +// +//go:systemstack +func gcSetTriggerRatio(triggerRatio float64) (changed bool) { + // Since GOGC ratios are in terms of heap_marked, make sure it + // isn't 0. This shouldn't happen, but if it does we want to + // avoid infinities and divide-by-zeroes. + if memstats.heap_marked == 0 { + memstats.heap_marked = 1 + } + // Compute the next GC goal, which is when the allocated heap // has grown by GOGC/100 over the heap marked by the last - // cycle. + // cycle, or maxHeap, whichever is lower. goal := ^uint64(0) if gcpercent >= 0 { goal = memstats.heap_marked + memstats.heap_marked*uint64(gcpercent)/100 } + lock(&gcPressure.lock) + if gcPressure.maxHeap != ^uintptr(0) && goal > uint64(gcPressure.maxHeap) { // Careful of 32-bit uintptr! + // Use maxHeap-based goal. + goal = uint64(gcPressure.maxHeap) + unlock(&gcPressure.lock) + + // Avoid thrashing by not letting the + // effective GOGC drop below 10. + // + // TODO(austin): This heuristic is pulled from + // thin air. It might be better to do + // something to more directly force + // amortization of GC costs, e.g., by limiting + // what fraction of the time GC can be active. + var minGOGC uint64 = 10 + if gcpercent >= 0 && uint64(gcpercent) < minGOGC { + // The user explicitly requested + // GOGC < minGOGC. Use that. + minGOGC = uint64(gcpercent) + } + lowerBound := memstats.heap_marked + memstats.heap_marked*minGOGC/100 + if goal < lowerBound { + goal = lowerBound + } + } else { + unlock(&gcPressure.lock) + } // Set the trigger ratio, capped to reasonable bounds. - if gcpercent >= 0 { - scalingFactor := float64(gcpercent) / 100 + if triggerRatio < 0 { + // This can happen if the mutator is allocating very + // quickly or the GC is scanning very slowly. + triggerRatio = 0 + } else if gcpercent >= 0 && triggerRatio > float64(gcpercent)/100 { + // Cap trigger ratio at GOGC/100. + triggerRatio = float64(gcpercent) / 100 + } + memstats.triggerRatio = triggerRatio + + // Compute the absolute GC trigger from the trigger ratio. + // + // We trigger the next GC cycle when the allocated heap has + // grown by the trigger ratio over the marked heap size. + trigger := ^uint64(0) + if goal != ^uint64(0) { + trigger = uint64(float64(memstats.heap_marked) * (1 + triggerRatio)) // Ensure there's always a little margin so that the // mutator assist ratio isn't infinity. - maxTriggerRatio := 0.95 * scalingFactor - if triggerRatio > maxTriggerRatio { - triggerRatio = maxTriggerRatio + if trigger > goal*95/100 { + trigger = goal * 95 / 100 } // If we let triggerRatio go too low, then if the application @@ -792,30 +917,14 @@ func gcSetTriggerRatio(triggerRatio float64) { // fast/scalable allocator with 48 Ps that could drive the trigger ratio // to <0.05, this constant causes applications to retain the same peak // RSS compared to not having this allocator. - minTriggerRatio := 0.6 * scalingFactor - if triggerRatio < minTriggerRatio { - triggerRatio = minTriggerRatio + const minTriggerRatio = 0.6 + minTrigger := memstats.heap_marked + uint64(minTriggerRatio*float64(goal-memstats.heap_marked)) + if trigger < minTrigger { + trigger = minTrigger } - } else if triggerRatio < 0 { - // gcpercent < 0, so just make sure we're not getting a negative - // triggerRatio. This case isn't expected to happen in practice, - // and doesn't really matter because if gcpercent < 0 then we won't - // ever consume triggerRatio further on in this function, but let's - // just be defensive here; the triggerRatio being negative is almost - // certainly undesirable. - triggerRatio = 0 - } - memstats.triggerRatio = triggerRatio - // Compute the absolute GC trigger from the trigger ratio. - // - // We trigger the next GC cycle when the allocated heap has - // grown by the trigger ratio over the marked heap size. - trigger := ^uint64(0) - if gcpercent >= 0 { - trigger = uint64(float64(memstats.heap_marked) * (1 + triggerRatio)) // Don't trigger below the minimum heap size. - minTrigger := heapminimum + minTrigger = heapminimum if !isSweepDone() { // Concurrent sweep happens in the heap growth // from heap_live to gc_trigger, so ensure @@ -889,6 +998,85 @@ func gcSetTriggerRatio(triggerRatio float64) { } gcPaceScavenger() + + // Update the GC policy due to a GC pressure change. + lock(&gcPressure.lock) + gogc, maxHeap, egogc := gcReadPolicyLocked() + if gogc != gcPressure.gogc || maxHeap != gcPressure.maxHeap || egogc != gcPressure.egogc { + gcPressure.gogc, gcPressure.maxHeap, gcPressure.egogc = gogc, maxHeap, egogc + changed = true + } + unlock(&gcPressure.lock) + return +} + +// Sends a non-blocking notification on gcPressure.notify. +// +// mheap_.lock and gcPressure.lock must not be held. +func gcPolicyNotify() { + // Switch to the system stack to acquire gcPressure.lock. + var n chan<- struct{} + gp := getg() + systemstack(func() { + lock(&gcPressure.lock) + if gcPressure.notify == nil { + unlock(&gcPressure.lock) + return + } + if raceenabled { + // notify is protected by gcPressure.lock, but + // the race detector can't see that. + raceacquireg(gp, unsafe.Pointer(&gcPressure.notify)) + } + // Just grab the channel first so that we're holding as + // few locks as possible when we actually make the channel send. + n = gcPressure.notify + if raceenabled { + racereleaseg(gp, unsafe.Pointer(&gcPressure.notify)) + } + unlock(&gcPressure.lock) + }) + if n == nil { + return + } + + // Perform a non-blocking send on the channel. + select { + case n <- struct{}{}: + default: + } +} + +//go:linkname gcReadPolicy runtime/debug.gcReadPolicy +func gcReadPolicy() (gogc int, maxHeap uintptr, egogc int) { + systemstack(func() { + lock(&mheap_.lock) + gogc, maxHeap, egogc = gcReadPolicyLocked() + unlock(&mheap_.lock) + }) + return +} + +// mheap_.lock must be locked, therefore this must be called on the +// systemstack. +//go:systemstack +func gcReadPolicyLocked() (gogc int, maxHeapOut uintptr, egogc int) { + goal := memstats.next_gc + if goal < uint64(maxHeap) && gcpercent >= 0 { + // We're not up against the max heap size, so just + // return GOGC. + egogc = int(gcpercent) + } else { + // Back out the effective GOGC from the goal. + egogc = int(gcEffectiveGrowthRatio() * 100) + // The effective GOGC may actually be higher than + // gcpercent if the heap is tiny. Avoid that confusion + // and just return the user-set GOGC. + if gcpercent >= 0 && egogc > int(gcpercent) { + egogc = int(gcpercent) + } + } + return int(gcpercent), maxHeap, egogc } // gcEffectiveGrowthRatio returns the current effective heap growth @@ -1209,7 +1397,7 @@ func (t gcTrigger) test() bool { // own write. return memstats.heap_live >= memstats.gc_trigger case gcTriggerTime: - if gcpercent < 0 { + if gcpercent < 0 && gcPressure.maxHeap == ^uintptr(0) { return false } lastgc := int64(atomic.Load64(&memstats.last_gc_nanotime)) @@ -1703,7 +1891,13 @@ func gcMarkTermination(nextTriggerRatio float64) { memstats.last_heap_inuse = memstats.heap_inuse // Update GC trigger and pacing for the next cycle. - gcSetTriggerRatio(nextTriggerRatio) + var notify bool + systemstack(func() { + notify = gcSetTriggerRatio(nextTriggerRatio) + }) + if notify { + gcPolicyNotify() + } // Update timing memstats now := nanotime() diff --git a/src/runtime/mstats.go b/src/runtime/mstats.go index 6a8a34d1edcda..1c6798dfd25ad 100644 --- a/src/runtime/mstats.go +++ b/src/runtime/mstats.go @@ -335,8 +335,8 @@ type MemStats struct { // // The garbage collector's goal is to keep HeapAlloc ≤ NextGC. // At the end of each GC cycle, the target for the next cycle - // is computed based on the amount of reachable data and the - // value of GOGC. + // is computed based on the amount of reachable data, the + // value of GOGC, and the max heap size (if set). NextGC uint64 // LastGC is the time the last garbage collection finished, as