Always unfreeze when connection dies (#19634)

Unfreeze as soon as the incoming connection is terminated and don't wait for everything to complete.

We don't want to keep the services frozen if something becomes stuck.
This commit is contained in:
Klaus Post 2024-04-29 10:39:04 -07:00 committed by GitHub
parent 2e38bb5175
commit 3cf8a7c888
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 25 additions and 7 deletions

View File

@ -1576,7 +1576,8 @@ func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request)
// NetperfHandler - perform mesh style network throughput test
func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
ctx, cancel := context.WithCancel(r.Context())
defer cancel()
objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
if objectAPI == nil {
@ -1597,6 +1598,15 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
ctx = lkctx.Context()
defer nsLock.Unlock(lkctx)
// Freeze all incoming S3 API calls before running speedtest.
globalNotificationSys.ServiceFreeze(ctx, true)
// Unfreeze as soon as request context is canceled or when the function returns.
go func() {
<-ctx.Done()
globalNotificationSys.ServiceFreeze(ctx, false)
}()
durationStr := r.Form.Get(peerRESTDuration)
duration, err := time.ParseDuration(durationStr)
if err != nil {
@ -1622,7 +1632,8 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
// increasing concurrency and stopping when we have reached the limits on the
// system.
func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
ctx, cancel := context.WithCancel(r.Context())
defer cancel()
objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
if objectAPI == nil {
@ -1692,8 +1703,11 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
// Freeze all incoming S3 API calls before running speedtest.
globalNotificationSys.ServiceFreeze(ctx, true)
// unfreeze all incoming S3 API calls after speedtest.
defer globalNotificationSys.ServiceFreeze(ctx, false)
// Unfreeze as soon as request context is canceled or when the function returns.
go func() {
<-ctx.Done()
globalNotificationSys.ServiceFreeze(ctx, false)
}()
keepAliveTicker := time.NewTicker(500 * time.Millisecond)
defer keepAliveTicker.Stop()
@ -1793,7 +1807,8 @@ func validateObjPerfOptions(ctx context.Context, storageInfo madmin.StorageInfo,
// DriveSpeedtestHandler - reports throughput of drives available in the cluster
func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
ctx, cancel := context.WithCancel(r.Context())
defer cancel()
objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
if objectAPI == nil {
@ -1803,8 +1818,11 @@ func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.R
// Freeze all incoming S3 API calls before running speedtest.
globalNotificationSys.ServiceFreeze(ctx, true)
// unfreeze all incoming S3 API calls after speedtest.
defer globalNotificationSys.ServiceFreeze(ctx, false)
// Unfreeze as soon as request context is canceled or when the function returns.
go func() {
<-ctx.Done()
globalNotificationSys.ServiceFreeze(ctx, false)
}()
serial := r.Form.Get("serial") == "true"
blockSizeStr := r.Form.Get("blocksize")