diff --git a/storage/filestore/dir.go b/storage/filestore/dir.go index 3f15e56af..2ba6b3978 100644 --- a/storage/filestore/dir.go +++ b/storage/filestore/dir.go @@ -8,6 +8,7 @@ import ( "context" "encoding/base32" "errors" + "fmt" "io" "math" "os" @@ -871,6 +872,12 @@ func (info *blobInfo) Stat(ctx context.Context) (os.FileInfo, error) { if info.fileInfo == nil { fileInfo, err := os.Lstat(info.path) if err != nil { + if os.IsNotExist(err) { + return nil, err + } + if isLowLevelCorruptionError(err) { + return nil, &CorruptDataError{path: info.path, error: err} + } return nil, err } if fileInfo.Mode().IsDir() { @@ -884,3 +891,27 @@ func (info *blobInfo) Stat(ctx context.Context) (os.FileInfo, error) { func (info *blobInfo) FullPath(ctx context.Context) (string, error) { return info.path, nil } + +// CorruptDataError represents a filesystem or disk error which indicates data corruption. +// +// We use a custom error type here so that we can add explanatory information and wrap the original +// error at the same time. +type CorruptDataError struct { + path string + error error +} + +// Unwrap unwraps the error. +func (cde CorruptDataError) Unwrap() error { + return cde.error +} + +// Path returns the path at which the error was encountered. +func (cde CorruptDataError) Path() string { + return cde.path +} + +// Error returns an error string describing the condition. +func (cde CorruptDataError) Error() string { + return fmt.Sprintf("unrecoverable error accessing data on the storage file system (path=%v; error=%v). This is most likely due to disk bad sectors or a corrupted file system. Check your disk for bad sectors and integrity", cde.path, cde.error) +} diff --git a/storage/filestore/errors_other.go b/storage/filestore/errors_other.go new file mode 100644 index 000000000..52c8b7ce8 --- /dev/null +++ b/storage/filestore/errors_other.go @@ -0,0 +1,23 @@ +// Copyright (C) 2023 Storj Labs, Inc. +// See LICENSE for copying information. + +//go:build !unix +// +build !unix + +package filestore + +import ( + "errors" + "os" + "strings" +) + +func isLowLevelCorruptionError(err error) bool { + // convert to lowercase the perr.Op because Go returns inconsistently + // "lstat" in Linux and "Lstat" in Windows + var perr *os.PathError + if errors.As(err, &perr) && strings.ToLower(perr.Op) == "lstat" { + return true + } + return false +} diff --git a/storage/filestore/errors_unix.go b/storage/filestore/errors_unix.go new file mode 100644 index 000000000..ab59bbf3a --- /dev/null +++ b/storage/filestore/errors_unix.go @@ -0,0 +1,27 @@ +// Copyright (C) 2023 Storj Labs, Inc. +// See LICENSE for copying information. + +//go:build unix + +package filestore + +import ( + "errors" + "os" + "syscall" +) + +func isLowLevelCorruptionError(err error) bool { + var perr *os.PathError + if errors.As(err, &perr) && perr.Op == "lstat" { + return true + } + var errnoErr syscall.Errno + if errors.As(err, &errnoErr) { + switch errnoErr { + case syscall.EBADMSG, syscall.EIO: + return true + } + } + return false +} diff --git a/storagenode/retain/retain.go b/storagenode/retain/retain.go index e02a3c032..ebd26c427 100644 --- a/storagenode/retain/retain.go +++ b/storagenode/retain/retain.go @@ -398,6 +398,7 @@ func (s *Service) retainPieces(ctx context.Context, req Request) (err error) { // piece was deleted while we were scanning. return nil } + piecesSkipped++ s.log.Warn("failed to determine mtime of blob", zap.Error(err)) // but continue iterating.