JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/audit,storage: increase sleep delay in TestMaxVerifyCount

Browse Source 
Currently TextMaxVerifyCount flakes in some tests, try increasing the
sleep time to ensure that things are slow enough to trigger the error
condition.

Also pass ctx to all the funcs so we can handle sleep better.

Change-Id: I605b6ea8b14a0a66d81a605ce3251f57a1669c00
This commit is contained in:
Egon Elbre 2021-09-10 16:05:29 +03:00
parent b472925c1e
commit 1aec831d98
12 changed files with 114 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/storagenode/main.go
View File

@ -273,7 +273,7 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
return err
}
if err := db.Pieces().CreateVerificationFile(identity.ID); err != nil {
if err := db.Pieces().CreateVerificationFile(ctx, identity.ID); err != nil {
return err
}

16
private/testblobs/bad.go
View File

@ -201,19 +201,19 @@ func (bad *BadBlobs) ListNamespaces(ctx context.Context) ([][]byte, error) {
}
// FreeSpace return how much free space left for writing.
func (bad *BadBlobs) FreeSpace() (int64, error) {
func (bad *BadBlobs) FreeSpace(ctx context.Context) (int64, error) {
if err := bad.err.Err(); err != nil {
return 0, err
}
return bad.blobs.FreeSpace()
return bad.blobs.FreeSpace(ctx)
}
// CheckWritability tests writability of the storage directory by creating and deleting a file.
func (bad *BadBlobs) CheckWritability() error {
func (bad *BadBlobs) CheckWritability(ctx context.Context) error {
if err := bad.err.Err(); err != nil {
return err
}
return bad.blobs.CheckWritability()
return bad.blobs.CheckWritability(ctx)
}
// SpaceUsedForBlobs adds up how much is used in all namespaces.
@ -241,18 +241,18 @@ func (bad *BadBlobs) SpaceUsedForTrash(ctx context.Context) (int64, error) {
}
// CreateVerificationFile creates a file to be used for storage directory verification.
func (bad *BadBlobs) CreateVerificationFile(id storj.NodeID) error {
func (bad *BadBlobs) CreateVerificationFile(ctx context.Context, id storj.NodeID) error {
if err := bad.err.Err(); err != nil {
return err
}
return bad.blobs.CreateVerificationFile(id)
return bad.blobs.CreateVerificationFile(ctx, id)
}
// VerifyStorageDir verifies that the storage directory is correct by checking for the existence and validity
// of the verification file.
func (bad *BadBlobs) VerifyStorageDir(id storj.NodeID) error {
func (bad *BadBlobs) VerifyStorageDir(ctx context.Context, id storj.NodeID) error {
if err := bad.err.Err(); err != nil {
return err
}
return bad.blobs.VerifyStorageDir(id)
return bad.blobs.VerifyStorageDir(ctx, id)
}

4
private/testblobs/limitedspace.go
View File

@ -4,6 +4,8 @@
package testblobs
import (
"context"
"go.uber.org/zap"
"storj.io/storj/storage"
@ -51,6 +53,6 @@ func newLimitedSpaceBlobs(log *zap.Logger, blobs storage.Blobs, freeSpace int64)
}
// FreeSpace returns how much free space left for writing.
func (limspace *LimitedSpaceBlobs) FreeSpace() (int64, error) {
func (limspace *LimitedSpaceBlobs) FreeSpace(ctx context.Context) (int64, error) {
return limspace.freeSpace, nil
}

98
private/testblobs/slow.go
View File

@ -8,6 +8,7 @@ import (
"sync/atomic"
"time"
"github.com/zeebo/errs"
"go.uber.org/zap"
"storj.io/common/storj"
@ -62,7 +63,9 @@ func newSlowBlobs(log *zap.Logger, blobs storage.Blobs) *SlowBlobs {
// Create creates a new blob that can be written optionally takes a size
// argument for performance improvements, -1 is unknown size.
func (slow *SlowBlobs) Create(ctx context.Context, ref storage.BlobRef, size int64) (storage.BlobWriter, error) {
slow.sleep()
if err := slow.sleep(ctx); err != nil {
return nil, errs.Wrap(err)
}
return slow.blobs.Create(ctx, ref, size)
}
@ -73,56 +76,74 @@ func (slow *SlowBlobs) Close() error {
// Open opens a reader with the specified namespace and key.
func (slow *SlowBlobs) Open(ctx context.Context, ref storage.BlobRef) (storage.BlobReader, error) {
slow.sleep()
if err := slow.sleep(ctx); err != nil {
return nil, errs.Wrap(err)
}
return slow.blobs.Open(ctx, ref)
}
// OpenWithStorageFormat opens a reader for the already-located blob, avoiding the potential need
// to check multiple storage formats to find the blob.
func (slow *SlowBlobs) OpenWithStorageFormat(ctx context.Context, ref storage.BlobRef, formatVer storage.FormatVersion) (storage.BlobReader, error) {
slow.sleep()
if err := slow.sleep(ctx); err != nil {
return nil, errs.Wrap(err)
}
return slow.blobs.OpenWithStorageFormat(ctx, ref, formatVer)
}
// Trash deletes the blob with the namespace and key.
func (slow *SlowBlobs) Trash(ctx context.Context, ref storage.BlobRef) error {
slow.sleep()
if err := slow.sleep(ctx); err != nil {
return errs.Wrap(err)
}
return slow.blobs.Trash(ctx, ref)
}
// RestoreTrash restores all files in the trash.
func (slow *SlowBlobs) RestoreTrash(ctx context.Context, namespace []byte) ([][]byte, error) {
slow.sleep()
if err := slow.sleep(ctx); err != nil {
return nil, errs.Wrap(err)
}
return slow.blobs.RestoreTrash(ctx, namespace)
}
// EmptyTrash empties the trash.
func (slow *SlowBlobs) EmptyTrash(ctx context.Context, namespace []byte, trashedBefore time.Time) (int64, [][]byte, error) {
slow.sleep()
if err := slow.sleep(ctx); err != nil {
return 0, nil, errs.Wrap(err)
}
return slow.blobs.EmptyTrash(ctx, namespace, trashedBefore)
}
// Delete deletes the blob with the namespace and key.
func (slow *SlowBlobs) Delete(ctx context.Context, ref storage.BlobRef) error {
slow.sleep()
if err := slow.sleep(ctx); err != nil {
return errs.Wrap(err)
}
return slow.blobs.Delete(ctx, ref)
}
// DeleteWithStorageFormat deletes the blob with the namespace, key, and format version.
func (slow *SlowBlobs) DeleteWithStorageFormat(ctx context.Context, ref storage.BlobRef, formatVer storage.FormatVersion) error {
slow.sleep()
if err := slow.sleep(ctx); err != nil {
return errs.Wrap(err)
}
return slow.blobs.DeleteWithStorageFormat(ctx, ref, formatVer)
}
// DeleteNamespace deletes blobs of specific satellite, used after successful GE only.
func (slow *SlowBlobs) DeleteNamespace(ctx context.Context, ref []byte) (err error) {
slow.sleep()
if err := slow.sleep(ctx); err != nil {
return errs.Wrap(err)
}
return slow.blobs.DeleteNamespace(ctx, ref)
}
// Stat looks up disk metadata on the blob file.
func (slow *SlowBlobs) Stat(ctx context.Context, ref storage.BlobRef) (storage.BlobInfo, error) {
slow.sleep()
if err := slow.sleep(ctx); err != nil {
return nil, errs.Wrap(err)
}
return slow.blobs.Stat(ctx, ref)
}
@ -130,7 +151,9 @@ func (slow *SlowBlobs) Stat(ctx context.Context, ref storage.BlobRef) (storage.B
// version. This avoids the potential need to check multiple storage formats for the blob
// when the format is already known.
func (slow *SlowBlobs) StatWithStorageFormat(ctx context.Context, ref storage.BlobRef, formatVer storage.FormatVersion) (storage.BlobInfo, error) {
slow.sleep()
if err := slow.sleep(ctx); err != nil {
return nil, errs.Wrap(err)
}
return slow.blobs.StatWithStorageFormat(ctx, ref, formatVer)
}
@ -138,7 +161,9 @@ func (slow *SlowBlobs) StatWithStorageFormat(ctx context.Context, ref storage.Bl
// If walkFunc returns a non-nil error, WalkNamespace will stop iterating and return the
// error immediately.
func (slow *SlowBlobs) WalkNamespace(ctx context.Context, namespace []byte, walkFunc func(storage.BlobInfo) error) error {
slow.sleep()
if err := slow.sleep(ctx); err != nil {
return errs.Wrap(err)
}
return slow.blobs.WalkNamespace(ctx, namespace, walkFunc)
}
@ -148,45 +173,57 @@ func (slow *SlowBlobs) ListNamespaces(ctx context.Context) ([][]byte, error) {
}
// FreeSpace return how much free space left for writing.
func (slow *SlowBlobs) FreeSpace() (int64, error) {
slow.sleep()
return slow.blobs.FreeSpace()
func (slow *SlowBlobs) FreeSpace(ctx context.Context) (int64, error) {
if err := slow.sleep(ctx); err != nil {
return 0, errs.Wrap(err)
}
return slow.blobs.FreeSpace(ctx)
}
// CheckWritability tests writability of the storage directory by creating and deleting a file.
func (slow *SlowBlobs) CheckWritability() error {
slow.sleep()
return slow.blobs.CheckWritability()
func (slow *SlowBlobs) CheckWritability(ctx context.Context) error {
if err := slow.sleep(ctx); err != nil {
return errs.Wrap(err)
}
return slow.blobs.CheckWritability(ctx)
}
// SpaceUsedForBlobs adds up how much is used in all namespaces.
func (slow *SlowBlobs) SpaceUsedForBlobs(ctx context.Context) (int64, error) {
slow.sleep()
if err := slow.sleep(ctx); err != nil {
return 0, errs.Wrap(err)
}
return slow.blobs.SpaceUsedForBlobs(ctx)
}
// SpaceUsedForBlobsInNamespace adds up how much is used in the given namespace.
func (slow *SlowBlobs) SpaceUsedForBlobsInNamespace(ctx context.Context, namespace []byte) (int64, error) {
slow.sleep()
if err := slow.sleep(ctx); err != nil {
return 0, errs.Wrap(err)
}
return slow.blobs.SpaceUsedForBlobsInNamespace(ctx, namespace)
}
// SpaceUsedForTrash adds up how much is used in all namespaces.
func (slow *SlowBlobs) SpaceUsedForTrash(ctx context.Context) (int64, error) {
slow.sleep()
if err := slow.sleep(ctx); err != nil {
return 0, errs.Wrap(err)
}
return slow.blobs.SpaceUsedForTrash(ctx)
}
// CreateVerificationFile creates a file to be used for storage directory verification.
func (slow *SlowBlobs) CreateVerificationFile(id storj.NodeID) error {
slow.sleep()
return slow.blobs.CreateVerificationFile(id)
func (slow *SlowBlobs) CreateVerificationFile(ctx context.Context, id storj.NodeID) error {
if err := slow.sleep(ctx); err != nil {
return errs.Wrap(err)
}
return slow.blobs.CreateVerificationFile(ctx, id)
}
// VerifyStorageDir verifies that the storage directory is correct by checking for the existence and validity
// of the verification file.
func (slow *SlowBlobs) VerifyStorageDir(id storj.NodeID) error {
return slow.blobs.VerifyStorageDir(id)
func (slow *SlowBlobs) VerifyStorageDir(ctx context.Context, id storj.NodeID) error {
return slow.blobs.VerifyStorageDir(ctx, id)
}
// SetLatency configures the blob store to sleep for delay duration for all
@ -196,7 +233,12 @@ func (slow *SlowBlobs) SetLatency(delay time.Duration) {
}
// sleep sleeps for the duration set to slow.delay.
func (slow *SlowBlobs) sleep() {
func (slow *SlowBlobs) sleep(ctx context.Context) error {
delay := time.Duration(atomic.LoadInt64(&slow.delay))
time.Sleep(delay)
select {
case <-time.After(delay):
return nil
case <-ctx.Done():
return ctx.Err()
}
}

2
private/testplanet/storagenode.go
View File

@ -220,7 +220,7 @@ func (planet *Planet) newStorageNode(ctx context.Context, prefix string, index,
return nil, err
}
if err := db.Pieces().CreateVerificationFile(identity.ID); err != nil {
if err := db.Pieces().CreateVerificationFile(ctx, identity.ID); err != nil {
return nil, err
}

2
satellite/audit/reverify_test.go
View File

@ -1255,7 +1255,7 @@ func TestMaxReverifyCount(t *testing.T) {
node := planet.FindNode(slowNode)
slowNodeDB := node.DB.(*testblobs.SlowDB)
// make downloads on storage node slower than the timeout on the satellite for downloading shares
delay := 1 * time.Second
delay := 3 * time.Second
slowNodeDB.SetLatency(delay)
oldRep, err := satellite.Reputation.Service.Get(ctx, slowNode)

8
storage/blob.go
View File

@ -96,9 +96,9 @@ type Blobs interface {
// when the format is already known.
StatWithStorageFormat(ctx context.Context, ref BlobRef, formatVer FormatVersion) (BlobInfo, error)
// FreeSpace return how much free space is available to the blobstore.
FreeSpace() (int64, error)
FreeSpace(ctx context.Context) (int64, error)
// CheckWritability tests writability of the storage directory by creating and deleting a file.
CheckWritability() error
CheckWritability(ctx context.Context) error
// SpaceUsedForTrash returns the total space used by the trash.
SpaceUsedForTrash(ctx context.Context) (int64, error)
// SpaceUsedForBlobs adds up how much is used in all namespaces.
@ -113,10 +113,10 @@ type Blobs interface {
// parameter is intended to allow canceling iteration early.
WalkNamespace(ctx context.Context, namespace []byte, walkFunc func(BlobInfo) error) error
// CreateVerificationFile creates a file to be used for storage directory verification.
CreateVerificationFile(id storj.NodeID) error
CreateVerificationFile(ctx context.Context, id storj.NodeID) error
// VerifyStorageDir verifies that the storage directory is correct by checking for the existence and validity
// of the verification file.
VerifyStorageDir(id storj.NodeID) error
VerifyStorageDir(ctx context.Context, id storj.NodeID) error
// Close closes the blob store and any resources associated with it.
Close() error
}

6
storage/filestore/dir.go
View File

@ -99,7 +99,7 @@ func (dir *Dir) garbagedir() string { return filepath.Join(dir.path, "garbage")
func (dir *Dir) trashdir() string { return filepath.Join(dir.path, "trash") }
// CreateVerificationFile creates a file to be used for storage directory verification.
func (dir *Dir) CreateVerificationFile(id storj.NodeID) error {
func (dir *Dir) CreateVerificationFile(ctx context.Context, id storj.NodeID) error {
f, err := os.Create(filepath.Join(dir.path, verificationFileName))
if err != nil {
return err
@ -113,7 +113,7 @@ func (dir *Dir) CreateVerificationFile(id storj.NodeID) error {
// Verify verifies that the storage directory is correct by checking for the existence and validity
// of the verification file.
func (dir *Dir) Verify(id storj.NodeID) error {
func (dir *Dir) Verify(ctx context.Context, id storj.NodeID) error {
content, err := ioutil.ReadFile(filepath.Join(dir.path, verificationFileName))
if err != nil {
return err
@ -836,7 +836,7 @@ type DiskInfo struct {
}
// Info returns information about the current state of the dir.
func (dir *Dir) Info() (DiskInfo, error) {
func (dir *Dir) Info(ctx context.Context) (DiskInfo, error) {
path, err := filepath.Abs(dir.path)
if err != nil {
return DiskInfo{}, err

14
storage/filestore/store.go
View File

@ -251,8 +251,8 @@ func (store *blobStore) SpaceUsedForTrash(ctx context.Context) (total int64, err
}
// FreeSpace returns how much space left in underlying directory.
func (store *blobStore) FreeSpace() (int64, error) {
info, err := store.dir.Info()
func (store *blobStore) FreeSpace(ctx context.Context) (int64, error) {
info, err := store.dir.Info(ctx)
if err != nil {
return 0, err
}
@ -260,7 +260,7 @@ func (store *blobStore) FreeSpace() (int64, error) {
}
// CheckWritability tests writability of the storage directory by creating and deleting a file.
func (store *blobStore) CheckWritability() error {
func (store *blobStore) CheckWritability(ctx context.Context) error {
f, err := ioutil.TempFile(store.dir.Path(), "write-test")
if err != nil {
return err
@ -296,12 +296,12 @@ func (store *blobStore) TestCreateV0(ctx context.Context, ref storage.BlobRef) (
}
// CreateVerificationFile creates a file to be used for storage directory verification.
func (store *blobStore) CreateVerificationFile(id storj.NodeID) error {
return store.dir.CreateVerificationFile(id)
func (store *blobStore) CreateVerificationFile(ctx context.Context, id storj.NodeID) error {
return store.dir.CreateVerificationFile(ctx, id)
}
// VerifyStorageDir verifies that the storage directory is correct by checking for the existence and validity
// of the verification file.
func (store *blobStore) VerifyStorageDir(id storj.NodeID) error {
return store.dir.Verify(id)
func (store *blobStore) VerifyStorageDir(ctx context.Context, id storj.NodeID) error {
return store.dir.Verify(ctx, id)
}

14
storage/filestore/store_test.go
View File

@ -892,15 +892,15 @@ func TestStorageDirVerification(t *testing.T) {
})
// test nonexistent file returns error
require.Error(t, store.VerifyStorageDir(ident0.ID))
require.Error(t, store.VerifyStorageDir(ctx, ident0.ID))
require.NoError(t, dir.CreateVerificationFile(ident0.ID))
require.NoError(t, dir.CreateVerificationFile(ctx, ident0.ID))
// test correct ID returns no error
require.NoError(t, store.VerifyStorageDir(ident0.ID))
require.NoError(t, store.VerifyStorageDir(ctx, ident0.ID))
// test incorrect ID returns error
err = store.VerifyStorageDir(ident1.ID)
err = store.VerifyStorageDir(ctx, ident1.ID)
require.Contains(t, err.Error(), "does not match running node's ID")
// test invalid node ID returns error
@ -911,10 +911,10 @@ func TestStorageDirVerification(t *testing.T) {
}()
_, err = f.Write([]byte{0, 1, 2, 3})
require.NoError(t, err)
err = store.VerifyStorageDir(ident0.ID)
err = store.VerifyStorageDir(ctx, ident0.ID)
require.Contains(t, err.Error(), "content of file is not a valid node ID")
// test file overwrite returns no error
require.NoError(t, dir.CreateVerificationFile(ident0.ID))
require.NoError(t, store.VerifyStorageDir(ident0.ID))
require.NoError(t, dir.CreateVerificationFile(ctx, ident0.ID))
require.NoError(t, store.VerifyStorageDir(ctx, ident0.ID))
}

4
storagenode/monitor/monitor.go
View File

@ -126,7 +126,7 @@ func (service *Service) Run(ctx context.Context) (err error) {
group, ctx := errgroup.WithContext(ctx)
group.Go(func() error {
return service.VerifyDirReadableLoop.Run(ctx, func(ctx context.Context) error {
err := service.store.VerifyStorageDir(service.contact.Local().ID)
err := service.store.VerifyStorageDir(ctx, service.contact.Local().ID)
if err != nil {
return Error.New("error verifying location and/or readability of storage directory: %v", err)
}
@ -135,7 +135,7 @@ func (service *Service) Run(ctx context.Context) (err error) {
})
group.Go(func() error {
return service.VerifyDirWritableLoop.Run(ctx, func(ctx context.Context) error {
err := service.store.CheckWritability()
err := service.store.CheckWritability(ctx)
if err != nil {
return Error.New("error verifying writability of storage directory: %v", err)
}

14
storagenode/pieces/store.go
View File

@ -197,14 +197,14 @@ func NewStore(log *zap.Logger, blobs storage.Blobs, v0PieceInfo V0PieceInfoDB,
}
// CreateVerificationFile creates a file to be used for storage directory verification.
func (store *Store) CreateVerificationFile(id storj.NodeID) error {
return store.blobs.CreateVerificationFile(id)
func (store *Store) CreateVerificationFile(ctx context.Context, id storj.NodeID) error {
return store.blobs.CreateVerificationFile(ctx, id)
}
// VerifyStorageDir verifies that the storage directory is correct by checking for the existence and validity
// of the verification file.
func (store *Store) VerifyStorageDir(id storj.NodeID) error {
return store.blobs.VerifyStorageDir(id)
func (store *Store) VerifyStorageDir(ctx context.Context, id storj.NodeID) error {
return store.blobs.VerifyStorageDir(ctx, id)
}
// Writer returns a new piece writer.
@ -697,7 +697,7 @@ type StorageStatus struct {
// StorageStatus returns information about the disk.
func (store *Store) StorageStatus(ctx context.Context) (_ StorageStatus, err error) {
defer mon.Task()(&ctx)(&err)
diskFree, err := store.blobs.FreeSpace()
diskFree, err := store.blobs.FreeSpace(ctx)
if err != nil {
return StorageStatus{}, err
}
@ -708,8 +708,8 @@ func (store *Store) StorageStatus(ctx context.Context) (_ StorageStatus, err err
}
// CheckWritability tests writability of the storage directory by creating and deleting a file.
func (store *Store) CheckWritability() error {
return store.blobs.CheckWritability()
func (store *Store) CheckWritability(ctx context.Context) error {
return store.blobs.CheckWritability(ctx)
}
type storedPieceAccess struct {