minio/cmd/leak-detect_test.go

164 lines
5.2 KiB
Go
Raw Normal View History

// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
2016-07-25 23:39:14 -04:00
package cmd
2016-07-25 23:39:14 -04:00
import (
"runtime/debug"
2016-07-25 23:39:14 -04:00
"sort"
"strings"
"time"
)
const (
// deadline (in seconds) up to which the go routine leak detection has to be retried.
2016-07-25 23:39:14 -04:00
leakDetectDeadline = 5
// pause time (in milliseconds) between each snapshot at the end of the go routine leak detection.
leakDetectPauseTimeMs = 50
)
// LeakDetect - type with methods for go routine leak detection.
type LeakDetect struct {
relevantRoutines map[string]bool
}
// NewLeakDetect - Initialize a LeakDetector with the snapshot of relevant Go routines.
func NewLeakDetect() LeakDetect {
snapshot := LeakDetect{
relevantRoutines: make(map[string]bool),
}
for _, g := range pickRelevantGoroutines() {
snapshot.relevantRoutines[g] = true
}
return snapshot
}
// CompareCurrentSnapshot - Compares the initial relevant stack trace with the current one (during the time of invocation).
2016-07-25 23:39:14 -04:00
func (initialSnapShot LeakDetect) CompareCurrentSnapshot() []string {
var stackDiff []string
for _, g := range pickRelevantGoroutines() {
// Identify the Go routines those were not present in the initial snapshot.
// In other words a stack diff.
if !initialSnapShot.relevantRoutines[g] {
stackDiff = append(stackDiff, g)
}
}
return stackDiff
}
// DetectLeak - Creates a snapshot of runtime stack and compares it with the initial stack snapshot.
2016-07-25 23:39:14 -04:00
func (initialSnapShot LeakDetect) DetectLeak(t TestErrHandler) {
if t.Failed() {
return
}
// Loop, waiting for goroutines to shut down.
// Wait up to 5 seconds, but finish as quickly as possible.
deadline := UTCNow().Add(leakDetectDeadline * time.Second)
2016-07-25 23:39:14 -04:00
for {
// get sack snapshot of relevant go routines.
leaked := initialSnapShot.CompareCurrentSnapshot()
// current stack snapshot matches the initial one, no leaks, return.
if len(leaked) == 0 {
return
}
// wait a test again will deadline.
if UTCNow().Before(deadline) {
2016-07-25 23:39:14 -04:00
time.Sleep(leakDetectPauseTimeMs * time.Millisecond)
continue
}
// after the deadline time report all the difference in the latest snapshot compared with the initial one.
for _, g := range leaked {
t.Errorf("Leaked goroutine: %v", g)
}
return
}
}
// DetectTestLeak - snapshots the currently running goroutines and returns a
2016-07-25 23:39:14 -04:00
// function to be run at the end of tests to see whether any
// goroutines leaked.
// Usage: `defer DetectTestLeak(t)()` in beginning line of benchmarks or unit tests.
func DetectTestLeak(t TestErrHandler) func() {
initialStackSnapShot := NewLeakDetect()
return func() {
initialStackSnapShot.DetectLeak(t)
}
}
// list of functions to be ignored from the stack trace.
// Leak detection is done when tests are run, should ignore the tests related functions,
// and other runtime functions while identifying leaks.
var ignoredStackFns = []string{
"",
// Below are the stacks ignored by the upstream leaktest code.
"testing.Main(",
"testing.tRunner(",
"testing.tRunner(",
"runtime.goexit",
"created by runtime.gc",
// ignore the snapshot function.
// since the snapshot is taken here the entry will have the current function too.
"pickRelevantGoroutines",
"runtime.MHeap_Scavenger",
"signal.signal_recv",
"sigterm.handler",
"runtime_mcall",
"goroutine in C code",
}
// Identify whether the stack trace entry is part of ignoredStackFn .
func isIgnoredStackFn(stack string) (ok bool) {
ok = true
for _, stackFn := range ignoredStackFns {
if !strings.Contains(stack, stackFn) {
ok = false
continue
}
break
}
return ok
}
// pickRelevantGoroutines returns all goroutines we care about for the purpose
// of leak checking. It excludes testing or runtime ones.
func pickRelevantGoroutines() (gs []string) {
// get runtime stack buffer.
buf := debug.Stack()
2016-07-25 23:39:14 -04:00
// runtime stack of go routines will be listed with 2 blank spaces between each of them, so split on "\n\n" .
for _, g := range strings.Split(string(buf), "\n\n") {
// Again split on a new line, the first line of the second half contains the info about the go routine.
2016-07-25 23:39:14 -04:00
sl := strings.SplitN(g, "\n", 2)
if len(sl) != 2 {
continue
}
stack := strings.TrimSpace(sl[1])
// ignore the testing go routine.
// since the tests will be invoking the leaktest it would contain the test go routine.
if strings.HasPrefix(stack, "testing.RunTests") {
continue
}
// Ignore the following go routines.
// testing and run time go routines should be ignored, only the application generated go routines should be taken into account.
if isIgnoredStackFn(stack) {
continue
}
gs = append(gs, g)
}
sort.Strings(gs)
return
}