diff --git a/storagenode/monitor/monitor.go b/storagenode/monitor/monitor.go index f6b58c76a..989750b7e 100644 --- a/storagenode/monitor/monitor.go +++ b/storagenode/monitor/monitor.go @@ -46,6 +46,7 @@ type Config struct { VerifyDirWritableInterval time.Duration `help:"how frequently to verify writability of storage directory" releaseDefault:"5m" devDefault:"30s"` VerifyDirReadableTimeout time.Duration `help:"how long to wait for a storage directory readability verification to complete" releaseDefault:"1m" devDefault:"10s"` VerifyDirWritableTimeout time.Duration `help:"how long to wait for a storage directory writability verification to complete" releaseDefault:"1m" devDefault:"10s"` + VerifyDirWarnOnly bool `help:"if the storage directory verification check fails, log a warning instead of killing the node" default:"false"` MinimumDiskSpace memory.Size `help:"how much disk space a node at minimum has to advertise" default:"500GB"` MinimumBandwidth memory.Size `help:"how much bandwidth a node at minimum has to advertise (deprecated)" default:"0TB"` NotifyLowDiskCooldown time.Duration `help:"minimum length of time between capacity reports" default:"10m" hidden:"true"` @@ -134,8 +135,16 @@ func (service *Service) Run(ctx context.Context) (err error) { err := service.store.VerifyStorageDirWithTimeout(ctx, service.contact.Local().ID, timeout) if err != nil { if errs.Is(err, context.DeadlineExceeded) { + if service.Config.VerifyDirWarnOnly { + service.log.Error("timed out while verifying readability of storage directory", zap.Duration("timeout", timeout)) + return nil + } return Error.New("timed out after %v while verifying readability of storage directory", timeout) } + if service.Config.VerifyDirWarnOnly { + service.log.Error("error verifying location and/or readability of storage directory", zap.Error(err)) + return nil + } return Error.New("error verifying location and/or readability of storage directory: %v", err) } return nil @@ -147,8 +156,16 @@ func (service *Service) Run(ctx context.Context) (err error) { err := service.store.CheckWritabilityWithTimeout(ctx, timeout) if err != nil { if errs.Is(err, context.DeadlineExceeded) { + if service.Config.VerifyDirWarnOnly { + service.log.Error("timed out while verifying writability of storage directory", zap.Duration("timeout", timeout)) + return nil + } return Error.New("timed out after %v while verifying writability of storage directory", timeout) } + if service.Config.VerifyDirWarnOnly { + service.log.Error("error verifying writability of storage directory", zap.Error(err)) + return nil + } return Error.New("error verifying writability of storage directory: %v", err) } return nil