storagenode/blobstore: move blob store logic
The blobstore implementation is entirely related to storagenode, so the rightful place is together with the storagenode implementation. Fixes https://github.com/storj/storj/issues/5754 Change-Id: Ie6637b0262cf37af6c3e558556c7604d9dc3613d
This commit is contained in:
parent
462c16eb9b
commit
f5020de57c
@ -23,10 +23,10 @@ import (
|
|||||||
"storj.io/private/debug"
|
"storj.io/private/debug"
|
||||||
"storj.io/storj/private/revocation"
|
"storj.io/storj/private/revocation"
|
||||||
"storj.io/storj/private/server"
|
"storj.io/storj/private/server"
|
||||||
"storj.io/storj/storage/filestore"
|
|
||||||
"storj.io/storj/storagenode"
|
"storj.io/storj/storagenode"
|
||||||
"storj.io/storj/storagenode/apikeys"
|
"storj.io/storj/storagenode/apikeys"
|
||||||
"storj.io/storj/storagenode/bandwidth"
|
"storj.io/storj/storagenode/bandwidth"
|
||||||
|
"storj.io/storj/storagenode/blobstore/filestore"
|
||||||
"storj.io/storj/storagenode/collector"
|
"storj.io/storj/storagenode/collector"
|
||||||
"storj.io/storj/storagenode/console/consoleserver"
|
"storj.io/storj/storagenode/console/consoleserver"
|
||||||
"storj.io/storj/storagenode/contact"
|
"storj.io/storj/storagenode/contact"
|
||||||
|
@ -19,12 +19,12 @@ import (
|
|||||||
"storj.io/common/sync2"
|
"storj.io/common/sync2"
|
||||||
"storj.io/common/testcontext"
|
"storj.io/common/testcontext"
|
||||||
"storj.io/common/testrand"
|
"storj.io/common/testrand"
|
||||||
"storj.io/storj/private/testblobs"
|
|
||||||
"storj.io/storj/private/testplanet"
|
"storj.io/storj/private/testplanet"
|
||||||
"storj.io/storj/satellite"
|
"storj.io/storj/satellite"
|
||||||
"storj.io/storj/satellite/audit"
|
"storj.io/storj/satellite/audit"
|
||||||
"storj.io/storj/satellite/metabase"
|
"storj.io/storj/satellite/metabase"
|
||||||
"storj.io/storj/storagenode"
|
"storj.io/storj/storagenode"
|
||||||
|
"storj.io/storj/storagenode/blobstore/testblobs"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestReverifySuccess(t *testing.T) {
|
func TestReverifySuccess(t *testing.T) {
|
||||||
|
@ -26,13 +26,13 @@ import (
|
|||||||
"storj.io/common/testcontext"
|
"storj.io/common/testcontext"
|
||||||
"storj.io/common/testrand"
|
"storj.io/common/testrand"
|
||||||
"storj.io/common/uuid"
|
"storj.io/common/uuid"
|
||||||
"storj.io/storj/private/testblobs"
|
|
||||||
"storj.io/storj/private/testplanet"
|
"storj.io/storj/private/testplanet"
|
||||||
"storj.io/storj/satellite"
|
"storj.io/storj/satellite"
|
||||||
"storj.io/storj/satellite/audit"
|
"storj.io/storj/satellite/audit"
|
||||||
"storj.io/storj/satellite/metabase"
|
"storj.io/storj/satellite/metabase"
|
||||||
"storj.io/storj/storage"
|
|
||||||
"storj.io/storj/storagenode"
|
"storj.io/storj/storagenode"
|
||||||
|
"storj.io/storj/storagenode/blobstore"
|
||||||
|
"storj.io/storj/storagenode/blobstore/testblobs"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestDownloadSharesHappyPath checks that the Share.Error field of all shares
|
// TestDownloadSharesHappyPath checks that the Share.Error field of all shares
|
||||||
@ -1108,7 +1108,7 @@ func getRemoteSegment(
|
|||||||
func corruptPieceData(ctx context.Context, t *testing.T, planet *testplanet.Planet, corruptedNode *testplanet.StorageNode, corruptedPieceID storj.PieceID) {
|
func corruptPieceData(ctx context.Context, t *testing.T, planet *testplanet.Planet, corruptedNode *testplanet.StorageNode, corruptedPieceID storj.PieceID) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
blobRef := storage.BlobRef{
|
blobRef := blobstore.BlobRef{
|
||||||
Namespace: planet.Satellites[0].ID().Bytes(),
|
Namespace: planet.Satellites[0].ID().Bytes(),
|
||||||
Key: corruptedPieceID.Bytes(),
|
Key: corruptedPieceID.Bytes(),
|
||||||
}
|
}
|
||||||
@ -1566,7 +1566,7 @@ func newBadBlobsAllowVerify(log *zap.Logger, nodeDB storagenode.DB) storagenode.
|
|||||||
|
|
||||||
type badBlobsAllowVerify struct {
|
type badBlobsAllowVerify struct {
|
||||||
testblobs.ErrorBlobs
|
testblobs.ErrorBlobs
|
||||||
goodBlobs storage.Blobs
|
goodBlobs blobstore.Blobs
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *badBlobsAllowVerify) VerifyStorageDir(ctx context.Context, id storj.NodeID) error {
|
func (b *badBlobsAllowVerify) VerifyStorageDir(ctx context.Context, id storj.NodeID) error {
|
||||||
|
@ -23,8 +23,8 @@ import (
|
|||||||
"storj.io/storj/private/testplanet"
|
"storj.io/storj/private/testplanet"
|
||||||
"storj.io/storj/satellite/gc/bloomfilter"
|
"storj.io/storj/satellite/gc/bloomfilter"
|
||||||
"storj.io/storj/satellite/metabase"
|
"storj.io/storj/satellite/metabase"
|
||||||
"storj.io/storj/storage"
|
|
||||||
"storj.io/storj/storagenode"
|
"storj.io/storj/storagenode"
|
||||||
|
"storj.io/storj/storagenode/blobstore"
|
||||||
"storj.io/uplink/private/eestream"
|
"storj.io/uplink/private/eestream"
|
||||||
"storj.io/uplink/private/testuplink"
|
"storj.io/uplink/private/testuplink"
|
||||||
)
|
)
|
||||||
@ -102,7 +102,7 @@ func TestGarbageCollection(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Check that piece of the deleted object is on the storagenode
|
// Check that piece of the deleted object is on the storagenode
|
||||||
pieceAccess, err := targetNode.DB.Pieces().Stat(ctx, storage.BlobRef{
|
pieceAccess, err := targetNode.DB.Pieces().Stat(ctx, blobstore.BlobRef{
|
||||||
Namespace: satellite.ID().Bytes(),
|
Namespace: satellite.ID().Bytes(),
|
||||||
Key: deletedPieceID.Bytes(),
|
Key: deletedPieceID.Bytes(),
|
||||||
})
|
})
|
||||||
@ -128,7 +128,7 @@ func TestGarbageCollection(t *testing.T) {
|
|||||||
targetNode.Storage2.RetainService.TestWaitUntilEmpty()
|
targetNode.Storage2.RetainService.TestWaitUntilEmpty()
|
||||||
|
|
||||||
// Check that piece of the deleted object is not on the storagenode
|
// Check that piece of the deleted object is not on the storagenode
|
||||||
pieceAccess, err = targetNode.DB.Pieces().Stat(ctx, storage.BlobRef{
|
pieceAccess, err = targetNode.DB.Pieces().Stat(ctx, blobstore.BlobRef{
|
||||||
Namespace: satellite.ID().Bytes(),
|
Namespace: satellite.ID().Bytes(),
|
||||||
Key: deletedPieceID.Bytes(),
|
Key: deletedPieceID.Bytes(),
|
||||||
})
|
})
|
||||||
@ -136,7 +136,7 @@ func TestGarbageCollection(t *testing.T) {
|
|||||||
require.Nil(t, pieceAccess)
|
require.Nil(t, pieceAccess)
|
||||||
|
|
||||||
// Check that piece of the kept object is on the storagenode
|
// Check that piece of the kept object is on the storagenode
|
||||||
pieceAccess, err = targetNode.DB.Pieces().Stat(ctx, storage.BlobRef{
|
pieceAccess, err = targetNode.DB.Pieces().Stat(ctx, blobstore.BlobRef{
|
||||||
Namespace: satellite.ID().Bytes(),
|
Namespace: satellite.ID().Bytes(),
|
||||||
Key: keptPieceID.Bytes(),
|
Key: keptPieceID.Bytes(),
|
||||||
})
|
})
|
||||||
|
@ -26,13 +26,13 @@ import (
|
|||||||
"storj.io/common/sync2"
|
"storj.io/common/sync2"
|
||||||
"storj.io/common/testcontext"
|
"storj.io/common/testcontext"
|
||||||
"storj.io/common/testrand"
|
"storj.io/common/testrand"
|
||||||
"storj.io/storj/private/testblobs"
|
|
||||||
"storj.io/storj/private/testplanet"
|
"storj.io/storj/private/testplanet"
|
||||||
"storj.io/storj/satellite"
|
"storj.io/storj/satellite"
|
||||||
"storj.io/storj/satellite/metabase"
|
"storj.io/storj/satellite/metabase"
|
||||||
"storj.io/storj/satellite/metainfo"
|
"storj.io/storj/satellite/metainfo"
|
||||||
"storj.io/storj/satellite/overlay"
|
"storj.io/storj/satellite/overlay"
|
||||||
"storj.io/storj/storagenode"
|
"storj.io/storj/storagenode"
|
||||||
|
"storj.io/storj/storagenode/blobstore/testblobs"
|
||||||
"storj.io/storj/storagenode/gracefulexit"
|
"storj.io/storj/storagenode/gracefulexit"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -35,7 +35,7 @@ import (
|
|||||||
"storj.io/storj/satellite/internalpb"
|
"storj.io/storj/satellite/internalpb"
|
||||||
"storj.io/storj/satellite/metabase"
|
"storj.io/storj/satellite/metabase"
|
||||||
"storj.io/storj/satellite/metainfo"
|
"storj.io/storj/satellite/metainfo"
|
||||||
"storj.io/storj/storage"
|
"storj.io/storj/storagenode/blobstore"
|
||||||
"storj.io/uplink"
|
"storj.io/uplink"
|
||||||
"storj.io/uplink/private/metaclient"
|
"storj.io/uplink/private/metaclient"
|
||||||
"storj.io/uplink/private/object"
|
"storj.io/uplink/private/object"
|
||||||
@ -1372,7 +1372,7 @@ func TestEndpoint_Object_With_StorageNodes(t *testing.T) {
|
|||||||
node := planet.FindNode(piece.StorageNode)
|
node := planet.FindNode(piece.StorageNode)
|
||||||
pieceID := segments[0].RootPieceID.Derive(piece.StorageNode, int32(piece.Number))
|
pieceID := segments[0].RootPieceID.Derive(piece.StorageNode, int32(piece.Number))
|
||||||
|
|
||||||
piece, err := node.DB.Pieces().Stat(ctx, storage.BlobRef{
|
piece, err := node.DB.Pieces().Stat(ctx, blobstore.BlobRef{
|
||||||
Namespace: planet.Satellites[0].ID().Bytes(),
|
Namespace: planet.Satellites[0].ID().Bytes(),
|
||||||
Key: pieceID.Bytes(),
|
Key: pieceID.Bytes(),
|
||||||
})
|
})
|
||||||
@ -1392,7 +1392,7 @@ func TestEndpoint_Object_With_StorageNodes(t *testing.T) {
|
|||||||
node := planet.FindNode(piece.StorageNode)
|
node := planet.FindNode(piece.StorageNode)
|
||||||
pieceID := segments[0].RootPieceID.Derive(piece.StorageNode, int32(piece.Number))
|
pieceID := segments[0].RootPieceID.Derive(piece.StorageNode, int32(piece.Number))
|
||||||
|
|
||||||
piece, err := node.DB.Pieces().Stat(ctx, storage.BlobRef{
|
piece, err := node.DB.Pieces().Stat(ctx, blobstore.BlobRef{
|
||||||
Namespace: planet.Satellites[0].ID().Bytes(),
|
Namespace: planet.Satellites[0].ID().Bytes(),
|
||||||
Key: pieceID.Bytes(),
|
Key: pieceID.Bytes(),
|
||||||
})
|
})
|
||||||
|
@ -18,12 +18,12 @@ import (
|
|||||||
"storj.io/common/storj"
|
"storj.io/common/storj"
|
||||||
"storj.io/common/testcontext"
|
"storj.io/common/testcontext"
|
||||||
"storj.io/common/testrand"
|
"storj.io/common/testrand"
|
||||||
"storj.io/storj/private/testblobs"
|
|
||||||
"storj.io/storj/private/testplanet"
|
"storj.io/storj/private/testplanet"
|
||||||
"storj.io/storj/satellite"
|
"storj.io/storj/satellite"
|
||||||
"storj.io/storj/satellite/metainfo/piecedeletion"
|
"storj.io/storj/satellite/metainfo/piecedeletion"
|
||||||
"storj.io/storj/satellite/overlay"
|
"storj.io/storj/satellite/overlay"
|
||||||
"storj.io/storj/storagenode"
|
"storj.io/storj/storagenode"
|
||||||
|
"storj.io/storj/storagenode/blobstore/testblobs"
|
||||||
"storj.io/storj/storagenode/pieces"
|
"storj.io/storj/storagenode/pieces"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -26,7 +26,6 @@ import (
|
|||||||
"storj.io/common/testcontext"
|
"storj.io/common/testcontext"
|
||||||
"storj.io/common/testrand"
|
"storj.io/common/testrand"
|
||||||
"storj.io/common/uuid"
|
"storj.io/common/uuid"
|
||||||
"storj.io/storj/private/testblobs"
|
|
||||||
"storj.io/storj/private/testplanet"
|
"storj.io/storj/private/testplanet"
|
||||||
"storj.io/storj/satellite"
|
"storj.io/storj/satellite"
|
||||||
"storj.io/storj/satellite/accounting"
|
"storj.io/storj/satellite/accounting"
|
||||||
@ -35,8 +34,9 @@ import (
|
|||||||
"storj.io/storj/satellite/repair/checker"
|
"storj.io/storj/satellite/repair/checker"
|
||||||
"storj.io/storj/satellite/repair/repairer"
|
"storj.io/storj/satellite/repair/repairer"
|
||||||
"storj.io/storj/satellite/reputation"
|
"storj.io/storj/satellite/reputation"
|
||||||
"storj.io/storj/storage"
|
|
||||||
"storj.io/storj/storagenode"
|
"storj.io/storj/storagenode"
|
||||||
|
"storj.io/storj/storagenode/blobstore"
|
||||||
|
"storj.io/storj/storagenode/blobstore/testblobs"
|
||||||
"storj.io/uplink/private/eestream"
|
"storj.io/uplink/private/eestream"
|
||||||
"storj.io/uplink/private/piecestore"
|
"storj.io/uplink/private/piecestore"
|
||||||
)
|
)
|
||||||
@ -2378,7 +2378,7 @@ func getRemoteSegment(
|
|||||||
func corruptPieceData(ctx context.Context, t *testing.T, planet *testplanet.Planet, corruptedNode *testplanet.StorageNode, corruptedPieceID storj.PieceID) {
|
func corruptPieceData(ctx context.Context, t *testing.T, planet *testplanet.Planet, corruptedNode *testplanet.StorageNode, corruptedPieceID storj.PieceID) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
blobRef := storage.BlobRef{
|
blobRef := blobstore.BlobRef{
|
||||||
Namespace: planet.Satellites[0].ID().Bytes(),
|
Namespace: planet.Satellites[0].ID().Bytes(),
|
||||||
Key: corruptedPieceID.Bytes(),
|
Key: corruptedPieceID.Bytes(),
|
||||||
}
|
}
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
// Copyright (C) 2019 Storj Labs, Inc.
|
// Copyright (C) 2019 Storj Labs, Inc.
|
||||||
// See LICENSE for copying information.
|
// See LICENSE for copying information.
|
||||||
|
|
||||||
package storage
|
package blobstore
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
@ -11,15 +11,15 @@ import (
|
|||||||
|
|
||||||
"github.com/zeebo/errs"
|
"github.com/zeebo/errs"
|
||||||
|
|
||||||
"storj.io/storj/storage"
|
"storj.io/storj/storagenode/blobstore"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// FormatV0 is the identifier for storage format v0, which also corresponds to an absence of
|
// FormatV0 is the identifier for storage format v0, which also corresponds to an absence of
|
||||||
// format version information.
|
// format version information.
|
||||||
FormatV0 storage.FormatVersion = 0
|
FormatV0 blobstore.FormatVersion = 0
|
||||||
// FormatV1 is the identifier for storage format v1.
|
// FormatV1 is the identifier for storage format v1.
|
||||||
FormatV1 storage.FormatVersion = 1
|
FormatV1 blobstore.FormatVersion = 1
|
||||||
|
|
||||||
// Note: New FormatVersion values should be consecutive, as certain parts of this blob store
|
// Note: New FormatVersion values should be consecutive, as certain parts of this blob store
|
||||||
// iterate over them numerically and check for blobs stored with each version.
|
// iterate over them numerically and check for blobs stored with each version.
|
||||||
@ -42,10 +42,10 @@ const (
|
|||||||
// blobReader implements reading blobs.
|
// blobReader implements reading blobs.
|
||||||
type blobReader struct {
|
type blobReader struct {
|
||||||
*os.File
|
*os.File
|
||||||
formatVersion storage.FormatVersion
|
formatVersion blobstore.FormatVersion
|
||||||
}
|
}
|
||||||
|
|
||||||
func newBlobReader(file *os.File, formatVersion storage.FormatVersion) *blobReader {
|
func newBlobReader(file *os.File, formatVersion blobstore.FormatVersion) *blobReader {
|
||||||
return &blobReader{file, formatVersion}
|
return &blobReader{file, formatVersion}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -59,21 +59,21 @@ func (blob *blobReader) Size() (int64, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// StorageFormatVersion gets the storage format version being used by the blob.
|
// StorageFormatVersion gets the storage format version being used by the blob.
|
||||||
func (blob *blobReader) StorageFormatVersion() storage.FormatVersion {
|
func (blob *blobReader) StorageFormatVersion() blobstore.FormatVersion {
|
||||||
return blob.formatVersion
|
return blob.formatVersion
|
||||||
}
|
}
|
||||||
|
|
||||||
// blobWriter implements writing blobs.
|
// blobWriter implements writing blobs.
|
||||||
type blobWriter struct {
|
type blobWriter struct {
|
||||||
ref storage.BlobRef
|
ref blobstore.BlobRef
|
||||||
store *blobStore
|
store *blobStore
|
||||||
closed bool
|
closed bool
|
||||||
formatVersion storage.FormatVersion
|
formatVersion blobstore.FormatVersion
|
||||||
buffer *bufio.Writer
|
buffer *bufio.Writer
|
||||||
fh *os.File
|
fh *os.File
|
||||||
}
|
}
|
||||||
|
|
||||||
func newBlobWriter(ref storage.BlobRef, store *blobStore, formatVersion storage.FormatVersion, file *os.File, bufferSize int) *blobWriter {
|
func newBlobWriter(ref blobstore.BlobRef, store *blobStore, formatVersion blobstore.FormatVersion, file *os.File, bufferSize int) *blobWriter {
|
||||||
return &blobWriter{
|
return &blobWriter{
|
||||||
ref: ref,
|
ref: ref,
|
||||||
store: store,
|
store: store,
|
||||||
@ -140,6 +140,6 @@ func (blob *blobWriter) Size() (int64, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// StorageFormatVersion indicates what storage format version the blob is using.
|
// StorageFormatVersion indicates what storage format version the blob is using.
|
||||||
func (blob *blobWriter) StorageFormatVersion() storage.FormatVersion {
|
func (blob *blobWriter) StorageFormatVersion() blobstore.FormatVersion {
|
||||||
return blob.formatVersion
|
return blob.formatVersion
|
||||||
}
|
}
|
@ -22,7 +22,7 @@ import (
|
|||||||
|
|
||||||
"storj.io/common/experiment"
|
"storj.io/common/experiment"
|
||||||
"storj.io/common/storj"
|
"storj.io/common/storj"
|
||||||
"storj.io/storj/storage"
|
"storj.io/storj/storagenode/blobstore"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -162,14 +162,14 @@ func (dir *Dir) DeleteTemporary(ctx context.Context, file *os.File) (err error)
|
|||||||
// entire path; blobPathForFormatVersion() must also be used. This is a separate call because this
|
// entire path; blobPathForFormatVersion() must also be used. This is a separate call because this
|
||||||
// part of the filepath is constant, and blobPathForFormatVersion may need to be called multiple
|
// part of the filepath is constant, and blobPathForFormatVersion may need to be called multiple
|
||||||
// times with different storage.FormatVersion values.
|
// times with different storage.FormatVersion values.
|
||||||
func (dir *Dir) blobToBasePath(ref storage.BlobRef) (string, error) {
|
func (dir *Dir) blobToBasePath(ref blobstore.BlobRef) (string, error) {
|
||||||
return dir.refToDirPath(ref, dir.blobsdir())
|
return dir.refToDirPath(ref, dir.blobsdir())
|
||||||
}
|
}
|
||||||
|
|
||||||
// refToDirPath converts a blob reference to a filepath in the specified sub-directory.
|
// refToDirPath converts a blob reference to a filepath in the specified sub-directory.
|
||||||
func (dir *Dir) refToDirPath(ref storage.BlobRef, subDir string) (string, error) {
|
func (dir *Dir) refToDirPath(ref blobstore.BlobRef, subDir string) (string, error) {
|
||||||
if !ref.IsValid() {
|
if !ref.IsValid() {
|
||||||
return "", storage.ErrInvalidBlobRef.New("")
|
return "", blobstore.ErrInvalidBlobRef.New("")
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace := pathEncoding.EncodeToString(ref.Namespace)
|
namespace := pathEncoding.EncodeToString(ref.Namespace)
|
||||||
@ -183,7 +183,7 @@ func (dir *Dir) refToDirPath(ref storage.BlobRef, subDir string) (string, error)
|
|||||||
|
|
||||||
// fileConfirmedInTrash returns true if it is able to confirm the file is in
|
// fileConfirmedInTrash returns true if it is able to confirm the file is in
|
||||||
// the trash. On errors, or if the file is not in the trash, it returns false.
|
// the trash. On errors, or if the file is not in the trash, it returns false.
|
||||||
func (dir *Dir) fileConfirmedInTrash(ctx context.Context, ref storage.BlobRef, formatVer storage.FormatVersion) bool {
|
func (dir *Dir) fileConfirmedInTrash(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) bool {
|
||||||
trashBasePath, err := dir.refToDirPath(ref, dir.trashdir())
|
trashBasePath, err := dir.refToDirPath(ref, dir.trashdir())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false
|
return false
|
||||||
@ -195,7 +195,7 @@ func (dir *Dir) fileConfirmedInTrash(ctx context.Context, ref storage.BlobRef, f
|
|||||||
|
|
||||||
// blobPathForFormatVersion adjusts a bare blob path (as might have been generated by a call to
|
// blobPathForFormatVersion adjusts a bare blob path (as might have been generated by a call to
|
||||||
// blobToBasePath()) to what it should be for the given storage format version.
|
// blobToBasePath()) to what it should be for the given storage format version.
|
||||||
func blobPathForFormatVersion(path string, formatVersion storage.FormatVersion) string {
|
func blobPathForFormatVersion(path string, formatVersion blobstore.FormatVersion) string {
|
||||||
switch formatVersion {
|
switch formatVersion {
|
||||||
case FormatV0:
|
case FormatV0:
|
||||||
return path + v0PieceFileSuffix
|
return path + v0PieceFileSuffix
|
||||||
@ -208,7 +208,7 @@ func blobPathForFormatVersion(path string, formatVersion storage.FormatVersion)
|
|||||||
// blobToGarbagePath converts a blob reference to a filepath in transient
|
// blobToGarbagePath converts a blob reference to a filepath in transient
|
||||||
// storage. The files in garbage are deleted on an interval (in case the
|
// storage. The files in garbage are deleted on an interval (in case the
|
||||||
// initial deletion didn't work for some reason).
|
// initial deletion didn't work for some reason).
|
||||||
func (dir *Dir) blobToGarbagePath(ref storage.BlobRef) string {
|
func (dir *Dir) blobToGarbagePath(ref blobstore.BlobRef) string {
|
||||||
var name []byte
|
var name []byte
|
||||||
name = append(name, ref.Namespace...)
|
name = append(name, ref.Namespace...)
|
||||||
name = append(name, ref.Key...)
|
name = append(name, ref.Key...)
|
||||||
@ -216,7 +216,7 @@ func (dir *Dir) blobToGarbagePath(ref storage.BlobRef) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Commit commits the temporary file to permanent storage.
|
// Commit commits the temporary file to permanent storage.
|
||||||
func (dir *Dir) Commit(ctx context.Context, file *os.File, ref storage.BlobRef, formatVersion storage.FormatVersion) (err error) {
|
func (dir *Dir) Commit(ctx context.Context, file *os.File, ref blobstore.BlobRef, formatVersion blobstore.FormatVersion) (err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
position, seekErr := file.Seek(0, io.SeekCurrent)
|
position, seekErr := file.Seek(0, io.SeekCurrent)
|
||||||
truncErr := file.Truncate(position)
|
truncErr := file.Truncate(position)
|
||||||
@ -264,7 +264,7 @@ func (dir *Dir) Commit(ctx context.Context, file *os.File, ref storage.BlobRef,
|
|||||||
// order to find the blob, if it was stored with an older version of the storage node software.
|
// order to find the blob, if it was stored with an older version of the storage node software.
|
||||||
// In cases where the storage format version of a blob is already known, OpenWithStorageFormat()
|
// In cases where the storage format version of a blob is already known, OpenWithStorageFormat()
|
||||||
// will generally be a better choice.
|
// will generally be a better choice.
|
||||||
func (dir *Dir) Open(ctx context.Context, ref storage.BlobRef) (_ *os.File, _ storage.FormatVersion, err error) {
|
func (dir *Dir) Open(ctx context.Context, ref blobstore.BlobRef) (_ *os.File, _ blobstore.FormatVersion, err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
path, err := dir.blobToBasePath(ref)
|
path, err := dir.blobToBasePath(ref)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -290,7 +290,7 @@ func (dir *Dir) Open(ctx context.Context, ref storage.BlobRef) (_ *os.File, _ st
|
|||||||
|
|
||||||
// OpenWithStorageFormat opens an already-located blob file with a known storage format version,
|
// OpenWithStorageFormat opens an already-located blob file with a known storage format version,
|
||||||
// which avoids the potential need to search through multiple storage formats to find the blob.
|
// which avoids the potential need to search through multiple storage formats to find the blob.
|
||||||
func (dir *Dir) OpenWithStorageFormat(ctx context.Context, ref storage.BlobRef, formatVer storage.FormatVersion) (_ *os.File, err error) {
|
func (dir *Dir) OpenWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (_ *os.File, err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
path, err := dir.blobToBasePath(ref)
|
path, err := dir.blobToBasePath(ref)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -315,7 +315,7 @@ func (dir *Dir) OpenWithStorageFormat(ctx context.Context, ref storage.BlobRef,
|
|||||||
// in order to find the blob, if it was stored with an older version of the storage node software.
|
// in order to find the blob, if it was stored with an older version of the storage node software.
|
||||||
// In cases where the storage format version of a blob is already known, StatWithStorageFormat()
|
// In cases where the storage format version of a blob is already known, StatWithStorageFormat()
|
||||||
// will generally be a better choice.
|
// will generally be a better choice.
|
||||||
func (dir *Dir) Stat(ctx context.Context, ref storage.BlobRef) (_ storage.BlobInfo, err error) {
|
func (dir *Dir) Stat(ctx context.Context, ref blobstore.BlobRef) (_ blobstore.BlobInfo, err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
path, err := dir.blobToBasePath(ref)
|
path, err := dir.blobToBasePath(ref)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -337,7 +337,7 @@ func (dir *Dir) Stat(ctx context.Context, ref storage.BlobRef) (_ storage.BlobIn
|
|||||||
// StatWithStorageFormat looks up disk metadata on the blob file with the given storage format
|
// StatWithStorageFormat looks up disk metadata on the blob file with the given storage format
|
||||||
// version. This avoids the need for checking for the file in multiple different storage format
|
// version. This avoids the need for checking for the file in multiple different storage format
|
||||||
// types.
|
// types.
|
||||||
func (dir *Dir) StatWithStorageFormat(ctx context.Context, ref storage.BlobRef, formatVer storage.FormatVersion) (_ storage.BlobInfo, err error) {
|
func (dir *Dir) StatWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (_ blobstore.BlobInfo, err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
path, err := dir.blobToBasePath(ref)
|
path, err := dir.blobToBasePath(ref)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -355,13 +355,13 @@ func (dir *Dir) StatWithStorageFormat(ctx context.Context, ref storage.BlobRef,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Trash moves the piece specified by ref to the trashdir for every format version.
|
// Trash moves the piece specified by ref to the trashdir for every format version.
|
||||||
func (dir *Dir) Trash(ctx context.Context, ref storage.BlobRef) (err error) {
|
func (dir *Dir) Trash(ctx context.Context, ref blobstore.BlobRef) (err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
return dir.iterateStorageFormatVersions(ctx, ref, dir.TrashWithStorageFormat)
|
return dir.iterateStorageFormatVersions(ctx, ref, dir.TrashWithStorageFormat)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TrashWithStorageFormat moves the piece specified by ref to the trashdir for the specified format version.
|
// TrashWithStorageFormat moves the piece specified by ref to the trashdir for the specified format version.
|
||||||
func (dir *Dir) TrashWithStorageFormat(ctx context.Context, ref storage.BlobRef, formatVer storage.FormatVersion) (err error) {
|
func (dir *Dir) TrashWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (err error) {
|
||||||
// Ensure trashdir exists so that we know any os.IsNotExist errors below
|
// Ensure trashdir exists so that we know any os.IsNotExist errors below
|
||||||
// are not from a missing trash dir
|
// are not from a missing trash dir
|
||||||
_, err = os.Stat(dir.trashdir())
|
_, err = os.Stat(dir.trashdir())
|
||||||
@ -426,7 +426,7 @@ func (dir *Dir) ReplaceTrashnow(trashnow func() time.Time) {
|
|||||||
// RestoreTrash moves every piece in the trash folder back into blobsdir.
|
// RestoreTrash moves every piece in the trash folder back into blobsdir.
|
||||||
func (dir *Dir) RestoreTrash(ctx context.Context, namespace []byte) (keysRestored [][]byte, err error) {
|
func (dir *Dir) RestoreTrash(ctx context.Context, namespace []byte) (keysRestored [][]byte, err error) {
|
||||||
var errorsEncountered errs.Group
|
var errorsEncountered errs.Group
|
||||||
err = dir.walkNamespaceInPath(ctx, namespace, dir.trashdir(), func(info storage.BlobInfo) error {
|
err = dir.walkNamespaceInPath(ctx, namespace, dir.trashdir(), func(info blobstore.BlobInfo) error {
|
||||||
blobsBasePath, err := dir.blobToBasePath(info.BlobRef())
|
blobsBasePath, err := dir.blobToBasePath(info.BlobRef())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errorsEncountered.Add(err)
|
errorsEncountered.Add(err)
|
||||||
@ -476,7 +476,7 @@ func (dir *Dir) RestoreTrash(ctx context.Context, namespace []byte) (keysRestore
|
|||||||
func (dir *Dir) EmptyTrash(ctx context.Context, namespace []byte, trashedBefore time.Time) (bytesEmptied int64, deletedKeys [][]byte, err error) {
|
func (dir *Dir) EmptyTrash(ctx context.Context, namespace []byte, trashedBefore time.Time) (bytesEmptied int64, deletedKeys [][]byte, err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
var errorsEncountered errs.Group
|
var errorsEncountered errs.Group
|
||||||
err = dir.walkNamespaceInPath(ctx, namespace, dir.trashdir(), func(info storage.BlobInfo) error {
|
err = dir.walkNamespaceInPath(ctx, namespace, dir.trashdir(), func(info blobstore.BlobInfo) error {
|
||||||
fileInfo, err := info.Stat(ctx)
|
fileInfo, err := info.Stat(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
@ -523,7 +523,7 @@ func (dir *Dir) EmptyTrash(ctx context.Context, namespace []byte, trashedBefore
|
|||||||
//
|
//
|
||||||
// f will be executed for every storage formate version regardless of the
|
// f will be executed for every storage formate version regardless of the
|
||||||
// result, and will aggregate errors into a single returned error.
|
// result, and will aggregate errors into a single returned error.
|
||||||
func (dir *Dir) iterateStorageFormatVersions(ctx context.Context, ref storage.BlobRef, f func(ctx context.Context, ref storage.BlobRef, i storage.FormatVersion) error) (err error) {
|
func (dir *Dir) iterateStorageFormatVersions(ctx context.Context, ref blobstore.BlobRef, f func(ctx context.Context, ref blobstore.BlobRef, i blobstore.FormatVersion) error) (err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
var combinedErrors errs.Group
|
var combinedErrors errs.Group
|
||||||
for i := MinFormatVersionSupported; i <= MaxFormatVersionSupported; i++ {
|
for i := MinFormatVersionSupported; i <= MaxFormatVersionSupported; i++ {
|
||||||
@ -536,7 +536,7 @@ func (dir *Dir) iterateStorageFormatVersions(ctx context.Context, ref storage.Bl
|
|||||||
//
|
//
|
||||||
// It doesn't return an error if the blob is not found for any reason or it
|
// It doesn't return an error if the blob is not found for any reason or it
|
||||||
// cannot be deleted at this moment and it's delayed.
|
// cannot be deleted at this moment and it's delayed.
|
||||||
func (dir *Dir) Delete(ctx context.Context, ref storage.BlobRef) (err error) {
|
func (dir *Dir) Delete(ctx context.Context, ref blobstore.BlobRef) (err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
return dir.iterateStorageFormatVersions(ctx, ref, dir.DeleteWithStorageFormat)
|
return dir.iterateStorageFormatVersions(ctx, ref, dir.DeleteWithStorageFormat)
|
||||||
}
|
}
|
||||||
@ -550,7 +550,7 @@ func (dir *Dir) Delete(ctx context.Context, ref storage.BlobRef) (err error) {
|
|||||||
// * push the blobs to queue for retrying later.
|
// * push the blobs to queue for retrying later.
|
||||||
//
|
//
|
||||||
// It doesn't return an error if the piece isn't found for any reason.
|
// It doesn't return an error if the piece isn't found for any reason.
|
||||||
func (dir *Dir) DeleteWithStorageFormat(ctx context.Context, ref storage.BlobRef, formatVer storage.FormatVersion) (err error) {
|
func (dir *Dir) DeleteWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
return dir.deleteWithStorageFormatInPath(ctx, dir.blobsdir(), ref, formatVer)
|
return dir.deleteWithStorageFormatInPath(ctx, dir.blobsdir(), ref, formatVer)
|
||||||
}
|
}
|
||||||
@ -561,7 +561,7 @@ func (dir *Dir) DeleteNamespace(ctx context.Context, ref []byte) (err error) {
|
|||||||
return dir.deleteNamespace(ctx, dir.blobsdir(), ref)
|
return dir.deleteNamespace(ctx, dir.blobsdir(), ref)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (dir *Dir) deleteWithStorageFormatInPath(ctx context.Context, path string, ref storage.BlobRef, formatVer storage.FormatVersion) (err error) {
|
func (dir *Dir) deleteWithStorageFormatInPath(ctx context.Context, path string, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
|
|
||||||
// Ensure garbage dir exists so that we know any os.IsNotExist errors below
|
// Ensure garbage dir exists so that we know any os.IsNotExist errors below
|
||||||
@ -705,12 +705,12 @@ func (dir *Dir) listNamespacesInPath(ctx context.Context, path string) (ids [][]
|
|||||||
// greater, in the given namespace. If walkFunc returns a non-nil error, WalkNamespace will stop
|
// greater, in the given namespace. If walkFunc returns a non-nil error, WalkNamespace will stop
|
||||||
// iterating and return the error immediately. The ctx parameter is intended specifically to allow
|
// iterating and return the error immediately. The ctx parameter is intended specifically to allow
|
||||||
// canceling iteration early.
|
// canceling iteration early.
|
||||||
func (dir *Dir) WalkNamespace(ctx context.Context, namespace []byte, walkFunc func(storage.BlobInfo) error) (err error) {
|
func (dir *Dir) WalkNamespace(ctx context.Context, namespace []byte, walkFunc func(blobstore.BlobInfo) error) (err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
return dir.walkNamespaceInPath(ctx, namespace, dir.blobsdir(), walkFunc)
|
return dir.walkNamespaceInPath(ctx, namespace, dir.blobsdir(), walkFunc)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (dir *Dir) walkNamespaceInPath(ctx context.Context, namespace []byte, path string, walkFunc func(storage.BlobInfo) error) (err error) {
|
func (dir *Dir) walkNamespaceInPath(ctx context.Context, namespace []byte, path string, walkFunc func(blobstore.BlobInfo) error) (err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
namespaceDir := pathEncoding.EncodeToString(namespace)
|
namespaceDir := pathEncoding.EncodeToString(namespace)
|
||||||
nsDir := filepath.Join(path, namespaceDir)
|
nsDir := filepath.Join(path, namespaceDir)
|
||||||
@ -755,7 +755,7 @@ func (dir *Dir) walkNamespaceInPath(ctx context.Context, namespace []byte, path
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func decodeBlobInfo(namespace []byte, keyPrefix, keyDir, name string) (info storage.BlobInfo, ok bool) {
|
func decodeBlobInfo(namespace []byte, keyPrefix, keyDir, name string) (info blobstore.BlobInfo, ok bool) {
|
||||||
blobFileName := name
|
blobFileName := name
|
||||||
encodedKey := keyPrefix + blobFileName
|
encodedKey := keyPrefix + blobFileName
|
||||||
formatVer := FormatV0
|
formatVer := FormatV0
|
||||||
@ -767,14 +767,14 @@ func decodeBlobInfo(namespace []byte, keyPrefix, keyDir, name string) (info stor
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, false
|
return nil, false
|
||||||
}
|
}
|
||||||
ref := storage.BlobRef{
|
ref := blobstore.BlobRef{
|
||||||
Namespace: namespace,
|
Namespace: namespace,
|
||||||
Key: key,
|
Key: key,
|
||||||
}
|
}
|
||||||
return newBlobInfo(ref, filepath.Join(keyDir, blobFileName), nil, formatVer), true
|
return newBlobInfo(ref, filepath.Join(keyDir, blobFileName), nil, formatVer), true
|
||||||
}
|
}
|
||||||
|
|
||||||
func walkNamespaceWithPrefix(ctx context.Context, log *zap.Logger, namespace []byte, nsDir, keyPrefix string, walkFunc func(storage.BlobInfo) error) (err error) {
|
func walkNamespaceWithPrefix(ctx context.Context, log *zap.Logger, namespace []byte, nsDir, keyPrefix string, walkFunc func(blobstore.BlobInfo) error) (err error) {
|
||||||
keyDir := filepath.Join(nsDir, keyPrefix)
|
keyDir := filepath.Join(nsDir, keyPrefix)
|
||||||
openDir, err := os.Open(keyDir)
|
openDir, err := os.Open(keyDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -852,13 +852,13 @@ func (dir *Dir) Info(ctx context.Context) (DiskInfo, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type blobInfo struct {
|
type blobInfo struct {
|
||||||
ref storage.BlobRef
|
ref blobstore.BlobRef
|
||||||
path string
|
path string
|
||||||
fileInfo os.FileInfo
|
fileInfo os.FileInfo
|
||||||
formatVersion storage.FormatVersion
|
formatVersion blobstore.FormatVersion
|
||||||
}
|
}
|
||||||
|
|
||||||
func newBlobInfo(ref storage.BlobRef, path string, fileInfo os.FileInfo, formatVer storage.FormatVersion) storage.BlobInfo {
|
func newBlobInfo(ref blobstore.BlobRef, path string, fileInfo os.FileInfo, formatVer blobstore.FormatVersion) blobstore.BlobInfo {
|
||||||
return &blobInfo{
|
return &blobInfo{
|
||||||
ref: ref,
|
ref: ref,
|
||||||
path: path,
|
path: path,
|
||||||
@ -867,11 +867,11 @@ func newBlobInfo(ref storage.BlobRef, path string, fileInfo os.FileInfo, formatV
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (info *blobInfo) BlobRef() storage.BlobRef {
|
func (info *blobInfo) BlobRef() blobstore.BlobRef {
|
||||||
return info.ref
|
return info.ref
|
||||||
}
|
}
|
||||||
|
|
||||||
func (info *blobInfo) StorageFormatVersion() storage.FormatVersion {
|
func (info *blobInfo) StorageFormatVersion() blobstore.FormatVersion {
|
||||||
return info.formatVersion
|
return info.formatVersion
|
||||||
}
|
}
|
||||||
|
|
@ -18,7 +18,7 @@ import (
|
|||||||
|
|
||||||
"storj.io/common/memory"
|
"storj.io/common/memory"
|
||||||
"storj.io/common/storj"
|
"storj.io/common/storj"
|
||||||
"storj.io/storj/storage"
|
"storj.io/storj/storagenode/blobstore"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@ -29,12 +29,14 @@ var (
|
|||||||
ErrIsDir = Error.New("file is a directory")
|
ErrIsDir = Error.New("file is a directory")
|
||||||
|
|
||||||
mon = monkit.Package()
|
mon = monkit.Package()
|
||||||
|
// for backwards compatibility.
|
||||||
|
monStorage = monkit.ScopeNamed("storj.io/storj/storage/filestore")
|
||||||
|
|
||||||
_ storage.Blobs = (*blobStore)(nil)
|
_ blobstore.Blobs = (*blobStore)(nil)
|
||||||
)
|
)
|
||||||
|
|
||||||
func monFileInTrash(namespace []byte) *monkit.Meter {
|
func monFileInTrash(namespace []byte) *monkit.Meter {
|
||||||
return mon.Meter("open_file_in_trash", monkit.NewSeriesTag("namespace", hex.EncodeToString(namespace))) //mon:locked
|
return monStorage.Meter("open_file_in_trash", monkit.NewSeriesTag("namespace", hex.EncodeToString(namespace))) //mon:locked
|
||||||
}
|
}
|
||||||
|
|
||||||
// Config is configuration for the blob store.
|
// Config is configuration for the blob store.
|
||||||
@ -55,12 +57,12 @@ type blobStore struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new disk blob store in the specified directory.
|
// New creates a new disk blob store in the specified directory.
|
||||||
func New(log *zap.Logger, dir *Dir, config Config) storage.Blobs {
|
func New(log *zap.Logger, dir *Dir, config Config) blobstore.Blobs {
|
||||||
return &blobStore{dir: dir, log: log, config: config}
|
return &blobStore{dir: dir, log: log, config: config}
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewAt creates a new disk blob store in the specified directory.
|
// NewAt creates a new disk blob store in the specified directory.
|
||||||
func NewAt(log *zap.Logger, path string, config Config) (storage.Blobs, error) {
|
func NewAt(log *zap.Logger, path string, config Config) (blobstore.Blobs, error) {
|
||||||
dir, err := NewDir(log, path)
|
dir, err := NewDir(log, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, Error.Wrap(err)
|
return nil, Error.Wrap(err)
|
||||||
@ -72,7 +74,7 @@ func NewAt(log *zap.Logger, path string, config Config) (storage.Blobs, error) {
|
|||||||
func (store *blobStore) Close() error { return nil }
|
func (store *blobStore) Close() error { return nil }
|
||||||
|
|
||||||
// Open loads blob with the specified hash.
|
// Open loads blob with the specified hash.
|
||||||
func (store *blobStore) Open(ctx context.Context, ref storage.BlobRef) (_ storage.BlobReader, err error) {
|
func (store *blobStore) Open(ctx context.Context, ref blobstore.BlobRef) (_ blobstore.BlobReader, err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
file, formatVer, err := store.dir.Open(ctx, ref)
|
file, formatVer, err := store.dir.Open(ctx, ref)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -86,7 +88,7 @@ func (store *blobStore) Open(ctx context.Context, ref storage.BlobRef) (_ storag
|
|||||||
|
|
||||||
// OpenWithStorageFormat loads the already-located blob, avoiding the potential need to check multiple
|
// OpenWithStorageFormat loads the already-located blob, avoiding the potential need to check multiple
|
||||||
// storage formats to find the blob.
|
// storage formats to find the blob.
|
||||||
func (store *blobStore) OpenWithStorageFormat(ctx context.Context, blobRef storage.BlobRef, formatVer storage.FormatVersion) (_ storage.BlobReader, err error) {
|
func (store *blobStore) OpenWithStorageFormat(ctx context.Context, blobRef blobstore.BlobRef, formatVer blobstore.FormatVersion) (_ blobstore.BlobReader, err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
file, err := store.dir.OpenWithStorageFormat(ctx, blobRef, formatVer)
|
file, err := store.dir.OpenWithStorageFormat(ctx, blobRef, formatVer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -99,14 +101,14 @@ func (store *blobStore) OpenWithStorageFormat(ctx context.Context, blobRef stora
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Stat looks up disk metadata on the blob file.
|
// Stat looks up disk metadata on the blob file.
|
||||||
func (store *blobStore) Stat(ctx context.Context, ref storage.BlobRef) (_ storage.BlobInfo, err error) {
|
func (store *blobStore) Stat(ctx context.Context, ref blobstore.BlobRef) (_ blobstore.BlobInfo, err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
info, err := store.dir.Stat(ctx, ref)
|
info, err := store.dir.Stat(ctx, ref)
|
||||||
return info, Error.Wrap(err)
|
return info, Error.Wrap(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// StatWithStorageFormat looks up disk metadata on the blob file with the given storage format version.
|
// StatWithStorageFormat looks up disk metadata on the blob file with the given storage format version.
|
||||||
func (store *blobStore) StatWithStorageFormat(ctx context.Context, ref storage.BlobRef, formatVer storage.FormatVersion) (_ storage.BlobInfo, err error) {
|
func (store *blobStore) StatWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (_ blobstore.BlobInfo, err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
info, err := store.dir.StatWithStorageFormat(ctx, ref, formatVer)
|
info, err := store.dir.StatWithStorageFormat(ctx, ref, formatVer)
|
||||||
return info, Error.Wrap(err)
|
return info, Error.Wrap(err)
|
||||||
@ -116,14 +118,14 @@ func (store *blobStore) StatWithStorageFormat(ctx context.Context, ref storage.B
|
|||||||
//
|
//
|
||||||
// It doesn't return an error if the blob isn't found for any reason or it cannot
|
// It doesn't return an error if the blob isn't found for any reason or it cannot
|
||||||
// be deleted at this moment and it's delayed.
|
// be deleted at this moment and it's delayed.
|
||||||
func (store *blobStore) Delete(ctx context.Context, ref storage.BlobRef) (err error) {
|
func (store *blobStore) Delete(ctx context.Context, ref blobstore.BlobRef) (err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
err = store.dir.Delete(ctx, ref)
|
err = store.dir.Delete(ctx, ref)
|
||||||
return Error.Wrap(err)
|
return Error.Wrap(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteWithStorageFormat deletes blobs with the specified ref and storage format version.
|
// DeleteWithStorageFormat deletes blobs with the specified ref and storage format version.
|
||||||
func (store *blobStore) DeleteWithStorageFormat(ctx context.Context, ref storage.BlobRef, formatVer storage.FormatVersion) (err error) {
|
func (store *blobStore) DeleteWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
err = store.dir.DeleteWithStorageFormat(ctx, ref, formatVer)
|
err = store.dir.DeleteWithStorageFormat(ctx, ref, formatVer)
|
||||||
return Error.Wrap(err)
|
return Error.Wrap(err)
|
||||||
@ -137,7 +139,7 @@ func (store *blobStore) DeleteNamespace(ctx context.Context, ref []byte) (err er
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Trash moves the ref to a trash directory.
|
// Trash moves the ref to a trash directory.
|
||||||
func (store *blobStore) Trash(ctx context.Context, ref storage.BlobRef) (err error) {
|
func (store *blobStore) Trash(ctx context.Context, ref blobstore.BlobRef) (err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
return Error.Wrap(store.dir.Trash(ctx, ref))
|
return Error.Wrap(store.dir.Trash(ctx, ref))
|
||||||
}
|
}
|
||||||
@ -165,7 +167,7 @@ func (store *blobStore) GarbageCollect(ctx context.Context) (err error) {
|
|||||||
|
|
||||||
// Create creates a new blob that can be written.
|
// Create creates a new blob that can be written.
|
||||||
// Optionally takes a size argument for performance improvements, -1 is unknown size.
|
// Optionally takes a size argument for performance improvements, -1 is unknown size.
|
||||||
func (store *blobStore) Create(ctx context.Context, ref storage.BlobRef, size int64) (_ storage.BlobWriter, err error) {
|
func (store *blobStore) Create(ctx context.Context, ref blobstore.BlobRef, size int64) (_ blobstore.BlobWriter, err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
file, err := store.dir.CreateTemporaryFile(ctx, size)
|
file, err := store.dir.CreateTemporaryFile(ctx, size)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -196,7 +198,7 @@ func (store *blobStore) SpaceUsedForBlobs(ctx context.Context) (space int64, err
|
|||||||
// SpaceUsedForBlobsInNamespace adds up how much is used in the given namespace for blob storage.
|
// SpaceUsedForBlobsInNamespace adds up how much is used in the given namespace for blob storage.
|
||||||
func (store *blobStore) SpaceUsedForBlobsInNamespace(ctx context.Context, namespace []byte) (int64, error) {
|
func (store *blobStore) SpaceUsedForBlobsInNamespace(ctx context.Context, namespace []byte) (int64, error) {
|
||||||
var totalUsed int64
|
var totalUsed int64
|
||||||
err := store.WalkNamespace(ctx, namespace, func(info storage.BlobInfo) error {
|
err := store.WalkNamespace(ctx, namespace, func(info blobstore.BlobInfo) error {
|
||||||
statInfo, statErr := info.Stat(ctx)
|
statInfo, statErr := info.Stat(ctx)
|
||||||
if statErr != nil {
|
if statErr != nil {
|
||||||
store.log.Error("failed to stat blob", zap.Binary("namespace", namespace), zap.Binary("key", info.BlobRef().Key), zap.Error(statErr))
|
store.log.Error("failed to stat blob", zap.Binary("namespace", namespace), zap.Binary("key", info.BlobRef().Key), zap.Error(statErr))
|
||||||
@ -282,12 +284,12 @@ func (store *blobStore) ListNamespaces(ctx context.Context) (ids [][]byte, err e
|
|||||||
// WalkNamespace executes walkFunc for each locally stored blob in the given namespace. If walkFunc
|
// WalkNamespace executes walkFunc for each locally stored blob in the given namespace. If walkFunc
|
||||||
// returns a non-nil error, WalkNamespace will stop iterating and return the error immediately. The
|
// returns a non-nil error, WalkNamespace will stop iterating and return the error immediately. The
|
||||||
// ctx parameter is intended specifically to allow canceling iteration early.
|
// ctx parameter is intended specifically to allow canceling iteration early.
|
||||||
func (store *blobStore) WalkNamespace(ctx context.Context, namespace []byte, walkFunc func(storage.BlobInfo) error) (err error) {
|
func (store *blobStore) WalkNamespace(ctx context.Context, namespace []byte, walkFunc func(blobstore.BlobInfo) error) (err error) {
|
||||||
return store.dir.WalkNamespace(ctx, namespace, walkFunc)
|
return store.dir.WalkNamespace(ctx, namespace, walkFunc)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestCreateV0 creates a new V0 blob that can be written. This is ONLY appropriate in test situations.
|
// TestCreateV0 creates a new V0 blob that can be written. This is ONLY appropriate in test situations.
|
||||||
func (store *blobStore) TestCreateV0(ctx context.Context, ref storage.BlobRef) (_ storage.BlobWriter, err error) {
|
func (store *blobStore) TestCreateV0(ctx context.Context, ref blobstore.BlobRef) (_ blobstore.BlobWriter, err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
|
|
||||||
file, err := store.dir.CreateTemporaryFile(ctx, -1)
|
file, err := store.dir.CreateTemporaryFile(ctx, -1)
|
@ -23,8 +23,8 @@ import (
|
|||||||
"storj.io/common/memory"
|
"storj.io/common/memory"
|
||||||
"storj.io/common/testcontext"
|
"storj.io/common/testcontext"
|
||||||
"storj.io/common/testrand"
|
"storj.io/common/testrand"
|
||||||
"storj.io/storj/storage"
|
"storj.io/storj/storagenode/blobstore"
|
||||||
"storj.io/storj/storage/filestore"
|
"storj.io/storj/storagenode/blobstore/filestore"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -46,13 +46,13 @@ func TestStoreLoad(t *testing.T) {
|
|||||||
data := testrand.Bytes(blobSize)
|
data := testrand.Bytes(blobSize)
|
||||||
temp := make([]byte, len(data))
|
temp := make([]byte, len(data))
|
||||||
|
|
||||||
refs := []storage.BlobRef{}
|
refs := []blobstore.BlobRef{}
|
||||||
|
|
||||||
namespace := testrand.Bytes(32)
|
namespace := testrand.Bytes(32)
|
||||||
|
|
||||||
// store without size
|
// store without size
|
||||||
for i := 0; i < repeatCount; i++ {
|
for i := 0; i < repeatCount; i++ {
|
||||||
ref := storage.BlobRef{
|
ref := blobstore.BlobRef{
|
||||||
Namespace: namespace,
|
Namespace: namespace,
|
||||||
Key: testrand.Bytes(32),
|
Key: testrand.Bytes(32),
|
||||||
}
|
}
|
||||||
@ -75,7 +75,7 @@ func TestStoreLoad(t *testing.T) {
|
|||||||
namespace = testrand.Bytes(32)
|
namespace = testrand.Bytes(32)
|
||||||
// store with size
|
// store with size
|
||||||
for i := 0; i < repeatCount; i++ {
|
for i := 0; i < repeatCount; i++ {
|
||||||
ref := storage.BlobRef{
|
ref := blobstore.BlobRef{
|
||||||
Namespace: namespace,
|
Namespace: namespace,
|
||||||
Key: testrand.Bytes(32),
|
Key: testrand.Bytes(32),
|
||||||
}
|
}
|
||||||
@ -94,7 +94,7 @@ func TestStoreLoad(t *testing.T) {
|
|||||||
namespace = testrand.Bytes(32)
|
namespace = testrand.Bytes(32)
|
||||||
// store with larger size
|
// store with larger size
|
||||||
{
|
{
|
||||||
ref := storage.BlobRef{
|
ref := blobstore.BlobRef{
|
||||||
Namespace: namespace,
|
Namespace: namespace,
|
||||||
Key: testrand.Bytes(32),
|
Key: testrand.Bytes(32),
|
||||||
}
|
}
|
||||||
@ -113,7 +113,7 @@ func TestStoreLoad(t *testing.T) {
|
|||||||
namespace = testrand.Bytes(32)
|
namespace = testrand.Bytes(32)
|
||||||
// store with error
|
// store with error
|
||||||
{
|
{
|
||||||
ref := storage.BlobRef{
|
ref := blobstore.BlobRef{
|
||||||
Namespace: namespace,
|
Namespace: namespace,
|
||||||
Key: testrand.Bytes(32),
|
Key: testrand.Bytes(32),
|
||||||
}
|
}
|
||||||
@ -174,7 +174,7 @@ func TestDeleteWhileReading(t *testing.T) {
|
|||||||
|
|
||||||
data := testrand.Bytes(blobSize)
|
data := testrand.Bytes(blobSize)
|
||||||
|
|
||||||
ref := storage.BlobRef{
|
ref := blobstore.BlobRef{
|
||||||
Namespace: []byte{0},
|
Namespace: []byte{0},
|
||||||
Key: []byte{1},
|
Key: []byte{1},
|
||||||
}
|
}
|
||||||
@ -237,15 +237,15 @@ func TestDeleteWhileReading(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeABlob(ctx context.Context, t testing.TB, store storage.Blobs, blobRef storage.BlobRef, data []byte, formatVersion storage.FormatVersion) {
|
func writeABlob(ctx context.Context, t testing.TB, store blobstore.Blobs, blobRef blobstore.BlobRef, data []byte, formatVersion blobstore.FormatVersion) {
|
||||||
var (
|
var (
|
||||||
blobWriter storage.BlobWriter
|
blobWriter blobstore.BlobWriter
|
||||||
err error
|
err error
|
||||||
)
|
)
|
||||||
switch formatVersion {
|
switch formatVersion {
|
||||||
case filestore.FormatV0:
|
case filestore.FormatV0:
|
||||||
fStore, ok := store.(interface {
|
fStore, ok := store.(interface {
|
||||||
TestCreateV0(ctx context.Context, ref storage.BlobRef) (_ storage.BlobWriter, err error)
|
TestCreateV0(ctx context.Context, ref blobstore.BlobRef) (_ blobstore.BlobWriter, err error)
|
||||||
})
|
})
|
||||||
require.Truef(t, ok, "can't make a WriterForFormatVersion with this blob store (%T)", store)
|
require.Truef(t, ok, "can't make a WriterForFormatVersion with this blob store (%T)", store)
|
||||||
blobWriter, err = fStore.TestCreateV0(ctx, blobRef)
|
blobWriter, err = fStore.TestCreateV0(ctx, blobRef)
|
||||||
@ -265,21 +265,21 @@ func writeABlob(ctx context.Context, t testing.TB, store storage.Blobs, blobRef
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func verifyBlobHandle(t testing.TB, reader storage.BlobReader, expectDataLen int, expectFormat storage.FormatVersion) {
|
func verifyBlobHandle(t testing.TB, reader blobstore.BlobReader, expectDataLen int, expectFormat blobstore.FormatVersion) {
|
||||||
assert.Equal(t, expectFormat, reader.StorageFormatVersion())
|
assert.Equal(t, expectFormat, reader.StorageFormatVersion())
|
||||||
size, err := reader.Size()
|
size, err := reader.Size()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, int64(expectDataLen), size)
|
assert.Equal(t, int64(expectDataLen), size)
|
||||||
}
|
}
|
||||||
|
|
||||||
func verifyBlobInfo(ctx context.Context, t testing.TB, blobInfo storage.BlobInfo, expectDataLen int, expectFormat storage.FormatVersion) {
|
func verifyBlobInfo(ctx context.Context, t testing.TB, blobInfo blobstore.BlobInfo, expectDataLen int, expectFormat blobstore.FormatVersion) {
|
||||||
assert.Equal(t, expectFormat, blobInfo.StorageFormatVersion())
|
assert.Equal(t, expectFormat, blobInfo.StorageFormatVersion())
|
||||||
stat, err := blobInfo.Stat(ctx)
|
stat, err := blobInfo.Stat(ctx)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, int64(expectDataLen), stat.Size())
|
assert.Equal(t, int64(expectDataLen), stat.Size())
|
||||||
}
|
}
|
||||||
|
|
||||||
func tryOpeningABlob(ctx context.Context, t testing.TB, store storage.Blobs, blobRef storage.BlobRef, expectDataLen int, expectFormat storage.FormatVersion) {
|
func tryOpeningABlob(ctx context.Context, t testing.TB, store blobstore.Blobs, blobRef blobstore.BlobRef, expectDataLen int, expectFormat blobstore.FormatVersion) {
|
||||||
reader, err := store.Open(ctx, blobRef)
|
reader, err := store.Open(ctx, blobRef)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
verifyBlobHandle(t, reader, expectDataLen, expectFormat)
|
verifyBlobHandle(t, reader, expectDataLen, expectFormat)
|
||||||
@ -315,8 +315,8 @@ func TestMultipleStorageFormatVersions(t *testing.T) {
|
|||||||
v0BlobKey = testrand.Bytes(keySize)
|
v0BlobKey = testrand.Bytes(keySize)
|
||||||
v1BlobKey = testrand.Bytes(keySize)
|
v1BlobKey = testrand.Bytes(keySize)
|
||||||
|
|
||||||
v0Ref = storage.BlobRef{Namespace: namespace, Key: v0BlobKey}
|
v0Ref = blobstore.BlobRef{Namespace: namespace, Key: v0BlobKey}
|
||||||
v1Ref = storage.BlobRef{Namespace: namespace, Key: v1BlobKey}
|
v1Ref = blobstore.BlobRef{Namespace: namespace, Key: v1BlobKey}
|
||||||
)
|
)
|
||||||
|
|
||||||
// write a V0 blob
|
// write a V0 blob
|
||||||
@ -386,7 +386,7 @@ func TestStoreSpaceUsed(t *testing.T) {
|
|||||||
var totalSoFar memory.Size
|
var totalSoFar memory.Size
|
||||||
for _, size := range sizesToStore {
|
for _, size := range sizesToStore {
|
||||||
contents := testrand.Bytes(size)
|
contents := testrand.Bytes(size)
|
||||||
blobRef := storage.BlobRef{Namespace: namespace, Key: testrand.Bytes(keySize)}
|
blobRef := blobstore.BlobRef{Namespace: namespace, Key: testrand.Bytes(keySize)}
|
||||||
|
|
||||||
blobWriter, err := store.Create(ctx, blobRef, int64(len(contents)))
|
blobWriter, err := store.Create(ctx, blobRef, int64(len(contents)))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@ -420,7 +420,7 @@ func TestStoreTraversals(t *testing.T) {
|
|||||||
// invent some namespaces and store stuff in them
|
// invent some namespaces and store stuff in them
|
||||||
type namespaceWithBlobs struct {
|
type namespaceWithBlobs struct {
|
||||||
namespace []byte
|
namespace []byte
|
||||||
blobs []storage.BlobRef
|
blobs []blobstore.BlobRef
|
||||||
}
|
}
|
||||||
const numNamespaces = 4
|
const numNamespaces = 4
|
||||||
recordsToInsert := make([]namespaceWithBlobs, numNamespaces)
|
recordsToInsert := make([]namespaceWithBlobs, numNamespaces)
|
||||||
@ -433,9 +433,9 @@ func TestStoreTraversals(t *testing.T) {
|
|||||||
recordsToInsert[i].namespace[len(namespaceBase)-1] = byte(i)
|
recordsToInsert[i].namespace[len(namespaceBase)-1] = byte(i)
|
||||||
|
|
||||||
// put varying numbers of blobs in the namespaces
|
// put varying numbers of blobs in the namespaces
|
||||||
recordsToInsert[i].blobs = make([]storage.BlobRef, i+1)
|
recordsToInsert[i].blobs = make([]blobstore.BlobRef, i+1)
|
||||||
for j := range recordsToInsert[i].blobs {
|
for j := range recordsToInsert[i].blobs {
|
||||||
recordsToInsert[i].blobs[j] = storage.BlobRef{
|
recordsToInsert[i].blobs[j] = blobstore.BlobRef{
|
||||||
Namespace: recordsToInsert[i].namespace,
|
Namespace: recordsToInsert[i].namespace,
|
||||||
Key: testrand.Bytes(keySize),
|
Key: testrand.Bytes(keySize),
|
||||||
}
|
}
|
||||||
@ -471,7 +471,7 @@ func TestStoreTraversals(t *testing.T) {
|
|||||||
// keep track of which blobs we visit with WalkNamespace
|
// keep track of which blobs we visit with WalkNamespace
|
||||||
found := make([]bool, len(expected.blobs))
|
found := make([]bool, len(expected.blobs))
|
||||||
|
|
||||||
err = store.WalkNamespace(ctx, expected.namespace, func(info storage.BlobInfo) error {
|
err = store.WalkNamespace(ctx, expected.namespace, func(info blobstore.BlobInfo) error {
|
||||||
gotBlobRef := info.BlobRef()
|
gotBlobRef := info.BlobRef()
|
||||||
assert.Equal(t, expected.namespace, gotBlobRef.Namespace)
|
assert.Equal(t, expected.namespace, gotBlobRef.Namespace)
|
||||||
// find which blob this is in expected.blobs
|
// find which blob this is in expected.blobs
|
||||||
@ -511,7 +511,7 @@ func TestStoreTraversals(t *testing.T) {
|
|||||||
|
|
||||||
// test WalkNamespace on a nonexistent namespace also
|
// test WalkNamespace on a nonexistent namespace also
|
||||||
namespaceBase[len(namespaceBase)-1] = byte(numNamespaces)
|
namespaceBase[len(namespaceBase)-1] = byte(numNamespaces)
|
||||||
err = store.WalkNamespace(ctx, namespaceBase, func(_ storage.BlobInfo) error {
|
err = store.WalkNamespace(ctx, namespaceBase, func(_ blobstore.BlobInfo) error {
|
||||||
t.Fatal("this should not have been called")
|
t.Fatal("this should not have been called")
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
@ -520,7 +520,7 @@ func TestStoreTraversals(t *testing.T) {
|
|||||||
// check that WalkNamespace stops iterating after an error return
|
// check that WalkNamespace stops iterating after an error return
|
||||||
iterations := 0
|
iterations := 0
|
||||||
expectedErr := errs.New("an expected error")
|
expectedErr := errs.New("an expected error")
|
||||||
err = store.WalkNamespace(ctx, recordsToInsert[numNamespaces-1].namespace, func(_ storage.BlobInfo) error {
|
err = store.WalkNamespace(ctx, recordsToInsert[numNamespaces-1].namespace, func(_ blobstore.BlobInfo) error {
|
||||||
iterations++
|
iterations++
|
||||||
if iterations == 2 {
|
if iterations == 2 {
|
||||||
return expectedErr
|
return expectedErr
|
||||||
@ -544,7 +544,7 @@ func TestEmptyTrash(t *testing.T) {
|
|||||||
|
|
||||||
type testfile struct {
|
type testfile struct {
|
||||||
data []byte
|
data []byte
|
||||||
formatVer storage.FormatVersion
|
formatVer blobstore.FormatVersion
|
||||||
}
|
}
|
||||||
type testref struct {
|
type testref struct {
|
||||||
key []byte
|
key []byte
|
||||||
@ -614,16 +614,16 @@ func TestEmptyTrash(t *testing.T) {
|
|||||||
|
|
||||||
for _, namespace := range namespaces {
|
for _, namespace := range namespaces {
|
||||||
for _, ref := range namespace.refs {
|
for _, ref := range namespace.refs {
|
||||||
blobref := storage.BlobRef{
|
blobref := blobstore.BlobRef{
|
||||||
Namespace: namespace.namespace,
|
Namespace: namespace.namespace,
|
||||||
Key: ref.key,
|
Key: ref.key,
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, file := range ref.files {
|
for _, file := range ref.files {
|
||||||
var w storage.BlobWriter
|
var w blobstore.BlobWriter
|
||||||
if file.formatVer == filestore.FormatV0 {
|
if file.formatVer == filestore.FormatV0 {
|
||||||
fStore, ok := store.(interface {
|
fStore, ok := store.(interface {
|
||||||
TestCreateV0(ctx context.Context, ref storage.BlobRef) (_ storage.BlobWriter, err error)
|
TestCreateV0(ctx context.Context, ref blobstore.BlobRef) (_ blobstore.BlobWriter, err error)
|
||||||
})
|
})
|
||||||
require.Truef(t, ok, "can't make TestCreateV0 with this blob store (%T)", store)
|
require.Truef(t, ok, "can't make TestCreateV0 with this blob store (%T)", store)
|
||||||
w, err = fStore.TestCreateV0(ctx, blobref)
|
w, err = fStore.TestCreateV0(ctx, blobref)
|
||||||
@ -669,7 +669,7 @@ func TestTrashAndRestore(t *testing.T) {
|
|||||||
|
|
||||||
type testfile struct {
|
type testfile struct {
|
||||||
data []byte
|
data []byte
|
||||||
formatVer storage.FormatVersion
|
formatVer blobstore.FormatVersion
|
||||||
}
|
}
|
||||||
type testref struct {
|
type testref struct {
|
||||||
key []byte
|
key []byte
|
||||||
@ -739,16 +739,16 @@ func TestTrashAndRestore(t *testing.T) {
|
|||||||
|
|
||||||
for _, namespace := range namespaces {
|
for _, namespace := range namespaces {
|
||||||
for _, ref := range namespace.refs {
|
for _, ref := range namespace.refs {
|
||||||
blobref := storage.BlobRef{
|
blobref := blobstore.BlobRef{
|
||||||
Namespace: namespace.namespace,
|
Namespace: namespace.namespace,
|
||||||
Key: ref.key,
|
Key: ref.key,
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, file := range ref.files {
|
for _, file := range ref.files {
|
||||||
var w storage.BlobWriter
|
var w blobstore.BlobWriter
|
||||||
if file.formatVer == filestore.FormatV0 {
|
if file.formatVer == filestore.FormatV0 {
|
||||||
fStore, ok := store.(interface {
|
fStore, ok := store.(interface {
|
||||||
TestCreateV0(ctx context.Context, ref storage.BlobRef) (_ storage.BlobWriter, err error)
|
TestCreateV0(ctx context.Context, ref blobstore.BlobRef) (_ blobstore.BlobWriter, err error)
|
||||||
})
|
})
|
||||||
require.Truef(t, ok, "can't make TestCreateV0 with this blob store (%T)", store)
|
require.Truef(t, ok, "can't make TestCreateV0 with this blob store (%T)", store)
|
||||||
w, err = fStore.TestCreateV0(ctx, blobref)
|
w, err = fStore.TestCreateV0(ctx, blobref)
|
||||||
@ -791,7 +791,7 @@ func TestTrashAndRestore(t *testing.T) {
|
|||||||
|
|
||||||
// Verify pieces are back and look good for first namespace
|
// Verify pieces are back and look good for first namespace
|
||||||
for _, ref := range namespaces[0].refs {
|
for _, ref := range namespaces[0].refs {
|
||||||
blobref := storage.BlobRef{
|
blobref := blobstore.BlobRef{
|
||||||
Namespace: namespaces[0].namespace,
|
Namespace: namespaces[0].namespace,
|
||||||
Key: ref.key,
|
Key: ref.key,
|
||||||
}
|
}
|
||||||
@ -802,7 +802,7 @@ func TestTrashAndRestore(t *testing.T) {
|
|||||||
|
|
||||||
// Verify pieces in second namespace are still missing (were not restored)
|
// Verify pieces in second namespace are still missing (were not restored)
|
||||||
for _, ref := range namespaces[1].refs {
|
for _, ref := range namespaces[1].refs {
|
||||||
blobref := storage.BlobRef{
|
blobref := blobstore.BlobRef{
|
||||||
Namespace: namespaces[1].namespace,
|
Namespace: namespaces[1].namespace,
|
||||||
Key: ref.key,
|
Key: ref.key,
|
||||||
}
|
}
|
||||||
@ -814,7 +814,7 @@ func TestTrashAndRestore(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func requireFileMatches(ctx context.Context, t *testing.T, store storage.Blobs, data []byte, ref storage.BlobRef, formatVer storage.FormatVersion) {
|
func requireFileMatches(ctx context.Context, t *testing.T, store blobstore.Blobs, data []byte, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) {
|
||||||
r, err := store.OpenWithStorageFormat(ctx, ref, formatVer)
|
r, err := store.OpenWithStorageFormat(ctx, ref, formatVer)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
@ -838,7 +838,7 @@ func TestBlobMemoryBuffer(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
defer ctx.Check(store.Close)
|
defer ctx.Check(store.Close)
|
||||||
|
|
||||||
ref := storage.BlobRef{
|
ref := blobstore.BlobRef{
|
||||||
Namespace: testrand.Bytes(32),
|
Namespace: testrand.Bytes(32),
|
||||||
Key: testrand.Bytes(32),
|
Key: testrand.Bytes(32),
|
||||||
}
|
}
|
@ -11,15 +11,15 @@ import (
|
|||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
|
||||||
"storj.io/common/storj"
|
"storj.io/common/storj"
|
||||||
"storj.io/storj/storage"
|
|
||||||
"storj.io/storj/storagenode"
|
"storj.io/storj/storagenode"
|
||||||
|
"storj.io/storj/storagenode/blobstore"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrorBlobs is the interface of storage.Blobs with the SetError method added.
|
// ErrorBlobs is the interface of blobstore.Blobs with the SetError method added.
|
||||||
// This allows the BadDB{}.Blobs member to be replaced with something that has
|
// This allows the BadDB{}.Blobs member to be replaced with something that has
|
||||||
// specific behavior changes.
|
// specific behavior changes.
|
||||||
type ErrorBlobs interface {
|
type ErrorBlobs interface {
|
||||||
storage.Blobs
|
blobstore.Blobs
|
||||||
SetError(err error)
|
SetError(err error)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -41,7 +41,7 @@ func NewBadDB(log *zap.Logger, db storagenode.DB) *BadDB {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Pieces returns the blob store.
|
// Pieces returns the blob store.
|
||||||
func (bad *BadDB) Pieces() storage.Blobs {
|
func (bad *BadDB) Pieces() blobstore.Blobs {
|
||||||
return bad.Blobs
|
return bad.Blobs
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -53,7 +53,7 @@ func (bad *BadDB) SetError(err error) {
|
|||||||
// BadBlobs implements a bad blob store.
|
// BadBlobs implements a bad blob store.
|
||||||
type BadBlobs struct {
|
type BadBlobs struct {
|
||||||
err lockedErr
|
err lockedErr
|
||||||
blobs storage.Blobs
|
blobs blobstore.Blobs
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -78,7 +78,7 @@ func (m *lockedErr) Set(err error) {
|
|||||||
|
|
||||||
// newBadBlobs creates a new bad blob store wrapping the provided blobs.
|
// newBadBlobs creates a new bad blob store wrapping the provided blobs.
|
||||||
// Use SetError to manually configure the error returned by all operations.
|
// Use SetError to manually configure the error returned by all operations.
|
||||||
func newBadBlobs(log *zap.Logger, blobs storage.Blobs) *BadBlobs {
|
func newBadBlobs(log *zap.Logger, blobs blobstore.Blobs) *BadBlobs {
|
||||||
return &BadBlobs{
|
return &BadBlobs{
|
||||||
log: log,
|
log: log,
|
||||||
blobs: blobs,
|
blobs: blobs,
|
||||||
@ -92,7 +92,7 @@ func (bad *BadBlobs) SetError(err error) {
|
|||||||
|
|
||||||
// Create creates a new blob that can be written optionally takes a size
|
// Create creates a new blob that can be written optionally takes a size
|
||||||
// argument for performance improvements, -1 is unknown size.
|
// argument for performance improvements, -1 is unknown size.
|
||||||
func (bad *BadBlobs) Create(ctx context.Context, ref storage.BlobRef, size int64) (storage.BlobWriter, error) {
|
func (bad *BadBlobs) Create(ctx context.Context, ref blobstore.BlobRef, size int64) (blobstore.BlobWriter, error) {
|
||||||
if err := bad.err.Err(); err != nil {
|
if err := bad.err.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -108,7 +108,7 @@ func (bad *BadBlobs) Close() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Open opens a reader with the specified namespace and key.
|
// Open opens a reader with the specified namespace and key.
|
||||||
func (bad *BadBlobs) Open(ctx context.Context, ref storage.BlobRef) (storage.BlobReader, error) {
|
func (bad *BadBlobs) Open(ctx context.Context, ref blobstore.BlobRef) (blobstore.BlobReader, error) {
|
||||||
if err := bad.err.Err(); err != nil {
|
if err := bad.err.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -117,7 +117,7 @@ func (bad *BadBlobs) Open(ctx context.Context, ref storage.BlobRef) (storage.Blo
|
|||||||
|
|
||||||
// OpenWithStorageFormat opens a reader for the already-located blob, avoiding the potential need
|
// OpenWithStorageFormat opens a reader for the already-located blob, avoiding the potential need
|
||||||
// to check multiple storage formats to find the blob.
|
// to check multiple storage formats to find the blob.
|
||||||
func (bad *BadBlobs) OpenWithStorageFormat(ctx context.Context, ref storage.BlobRef, formatVer storage.FormatVersion) (storage.BlobReader, error) {
|
func (bad *BadBlobs) OpenWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (blobstore.BlobReader, error) {
|
||||||
if err := bad.err.Err(); err != nil {
|
if err := bad.err.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -125,7 +125,7 @@ func (bad *BadBlobs) OpenWithStorageFormat(ctx context.Context, ref storage.Blob
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Trash deletes the blob with the namespace and key.
|
// Trash deletes the blob with the namespace and key.
|
||||||
func (bad *BadBlobs) Trash(ctx context.Context, ref storage.BlobRef) error {
|
func (bad *BadBlobs) Trash(ctx context.Context, ref blobstore.BlobRef) error {
|
||||||
if err := bad.err.Err(); err != nil {
|
if err := bad.err.Err(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -149,7 +149,7 @@ func (bad *BadBlobs) EmptyTrash(ctx context.Context, namespace []byte, trashedBe
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Delete deletes the blob with the namespace and key.
|
// Delete deletes the blob with the namespace and key.
|
||||||
func (bad *BadBlobs) Delete(ctx context.Context, ref storage.BlobRef) error {
|
func (bad *BadBlobs) Delete(ctx context.Context, ref blobstore.BlobRef) error {
|
||||||
if err := bad.err.Err(); err != nil {
|
if err := bad.err.Err(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -157,7 +157,7 @@ func (bad *BadBlobs) Delete(ctx context.Context, ref storage.BlobRef) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// DeleteWithStorageFormat deletes the blob with the namespace, key, and format version.
|
// DeleteWithStorageFormat deletes the blob with the namespace, key, and format version.
|
||||||
func (bad *BadBlobs) DeleteWithStorageFormat(ctx context.Context, ref storage.BlobRef, formatVer storage.FormatVersion) error {
|
func (bad *BadBlobs) DeleteWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) error {
|
||||||
if err := bad.err.Err(); err != nil {
|
if err := bad.err.Err(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -173,7 +173,7 @@ func (bad *BadBlobs) DeleteNamespace(ctx context.Context, ref []byte) (err error
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Stat looks up disk metadata on the blob file.
|
// Stat looks up disk metadata on the blob file.
|
||||||
func (bad *BadBlobs) Stat(ctx context.Context, ref storage.BlobRef) (storage.BlobInfo, error) {
|
func (bad *BadBlobs) Stat(ctx context.Context, ref blobstore.BlobRef) (blobstore.BlobInfo, error) {
|
||||||
if err := bad.err.Err(); err != nil {
|
if err := bad.err.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -183,7 +183,7 @@ func (bad *BadBlobs) Stat(ctx context.Context, ref storage.BlobRef) (storage.Blo
|
|||||||
// StatWithStorageFormat looks up disk metadata for the blob file with the given storage format
|
// StatWithStorageFormat looks up disk metadata for the blob file with the given storage format
|
||||||
// version. This avoids the potential need to check multiple storage formats for the blob
|
// version. This avoids the potential need to check multiple storage formats for the blob
|
||||||
// when the format is already known.
|
// when the format is already known.
|
||||||
func (bad *BadBlobs) StatWithStorageFormat(ctx context.Context, ref storage.BlobRef, formatVer storage.FormatVersion) (storage.BlobInfo, error) {
|
func (bad *BadBlobs) StatWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (blobstore.BlobInfo, error) {
|
||||||
if err := bad.err.Err(); err != nil {
|
if err := bad.err.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -193,7 +193,7 @@ func (bad *BadBlobs) StatWithStorageFormat(ctx context.Context, ref storage.Blob
|
|||||||
// WalkNamespace executes walkFunc for each locally stored blob in the given namespace.
|
// WalkNamespace executes walkFunc for each locally stored blob in the given namespace.
|
||||||
// If walkFunc returns a non-nil error, WalkNamespace will stop iterating and return the
|
// If walkFunc returns a non-nil error, WalkNamespace will stop iterating and return the
|
||||||
// error immediately.
|
// error immediately.
|
||||||
func (bad *BadBlobs) WalkNamespace(ctx context.Context, namespace []byte, walkFunc func(storage.BlobInfo) error) error {
|
func (bad *BadBlobs) WalkNamespace(ctx context.Context, namespace []byte, walkFunc func(blobstore.BlobInfo) error) error {
|
||||||
if err := bad.err.Err(); err != nil {
|
if err := bad.err.Err(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
@ -8,8 +8,8 @@ import (
|
|||||||
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
|
||||||
"storj.io/storj/storage"
|
|
||||||
"storj.io/storj/storagenode"
|
"storj.io/storj/storagenode"
|
||||||
|
"storj.io/storj/storagenode/blobstore"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ensures that limitedSpaceDB implements storagenode.DB.
|
// ensures that limitedSpaceDB implements storagenode.DB.
|
||||||
@ -32,19 +32,19 @@ func NewLimitedSpaceDB(log *zap.Logger, db storagenode.DB, freeSpace int64) stor
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Pieces returns the blob store.
|
// Pieces returns the blob store.
|
||||||
func (lim *limitedSpaceDB) Pieces() storage.Blobs {
|
func (lim *limitedSpaceDB) Pieces() blobstore.Blobs {
|
||||||
return lim.blobs
|
return lim.blobs
|
||||||
}
|
}
|
||||||
|
|
||||||
// LimitedSpaceBlobs implements a limited space blob store.
|
// LimitedSpaceBlobs implements a limited space blob store.
|
||||||
type LimitedSpaceBlobs struct {
|
type LimitedSpaceBlobs struct {
|
||||||
storage.Blobs
|
blobstore.Blobs
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
freeSpace int64
|
freeSpace int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// newLimitedSpaceBlobs creates a new limited space blob store wrapping the provided blobs.
|
// newLimitedSpaceBlobs creates a new limited space blob store wrapping the provided blobs.
|
||||||
func newLimitedSpaceBlobs(log *zap.Logger, blobs storage.Blobs, freeSpace int64) *LimitedSpaceBlobs {
|
func newLimitedSpaceBlobs(log *zap.Logger, blobs blobstore.Blobs, freeSpace int64) *LimitedSpaceBlobs {
|
||||||
return &LimitedSpaceBlobs{
|
return &LimitedSpaceBlobs{
|
||||||
log: log,
|
log: log,
|
||||||
Blobs: blobs,
|
Blobs: blobs,
|
@ -12,8 +12,8 @@ import (
|
|||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
|
||||||
"storj.io/common/storj"
|
"storj.io/common/storj"
|
||||||
"storj.io/storj/storage"
|
|
||||||
"storj.io/storj/storagenode"
|
"storj.io/storj/storagenode"
|
||||||
|
"storj.io/storj/storagenode/blobstore"
|
||||||
)
|
)
|
||||||
|
|
||||||
// SlowDB implements slow storage node DB.
|
// SlowDB implements slow storage node DB.
|
||||||
@ -34,7 +34,7 @@ func NewSlowDB(log *zap.Logger, db storagenode.DB) *SlowDB {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Pieces returns the blob store.
|
// Pieces returns the blob store.
|
||||||
func (slow *SlowDB) Pieces() storage.Blobs {
|
func (slow *SlowDB) Pieces() blobstore.Blobs {
|
||||||
return slow.blobs
|
return slow.blobs
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -47,13 +47,13 @@ func (slow *SlowDB) SetLatency(delay time.Duration) {
|
|||||||
// SlowBlobs implements a slow blob store.
|
// SlowBlobs implements a slow blob store.
|
||||||
type SlowBlobs struct {
|
type SlowBlobs struct {
|
||||||
delay int64 // time.Duration
|
delay int64 // time.Duration
|
||||||
blobs storage.Blobs
|
blobs blobstore.Blobs
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
// newSlowBlobs creates a new slow blob store wrapping the provided blobs.
|
// newSlowBlobs creates a new slow blob store wrapping the provided blobs.
|
||||||
// Use SetLatency to dynamically configure the latency of all operations.
|
// Use SetLatency to dynamically configure the latency of all operations.
|
||||||
func newSlowBlobs(log *zap.Logger, blobs storage.Blobs) *SlowBlobs {
|
func newSlowBlobs(log *zap.Logger, blobs blobstore.Blobs) *SlowBlobs {
|
||||||
return &SlowBlobs{
|
return &SlowBlobs{
|
||||||
log: log,
|
log: log,
|
||||||
blobs: blobs,
|
blobs: blobs,
|
||||||
@ -62,7 +62,7 @@ func newSlowBlobs(log *zap.Logger, blobs storage.Blobs) *SlowBlobs {
|
|||||||
|
|
||||||
// Create creates a new blob that can be written optionally takes a size
|
// Create creates a new blob that can be written optionally takes a size
|
||||||
// argument for performance improvements, -1 is unknown size.
|
// argument for performance improvements, -1 is unknown size.
|
||||||
func (slow *SlowBlobs) Create(ctx context.Context, ref storage.BlobRef, size int64) (storage.BlobWriter, error) {
|
func (slow *SlowBlobs) Create(ctx context.Context, ref blobstore.BlobRef, size int64) (blobstore.BlobWriter, error) {
|
||||||
if err := slow.sleep(ctx); err != nil {
|
if err := slow.sleep(ctx); err != nil {
|
||||||
return nil, errs.Wrap(err)
|
return nil, errs.Wrap(err)
|
||||||
}
|
}
|
||||||
@ -75,7 +75,7 @@ func (slow *SlowBlobs) Close() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Open opens a reader with the specified namespace and key.
|
// Open opens a reader with the specified namespace and key.
|
||||||
func (slow *SlowBlobs) Open(ctx context.Context, ref storage.BlobRef) (storage.BlobReader, error) {
|
func (slow *SlowBlobs) Open(ctx context.Context, ref blobstore.BlobRef) (blobstore.BlobReader, error) {
|
||||||
if err := slow.sleep(ctx); err != nil {
|
if err := slow.sleep(ctx); err != nil {
|
||||||
return nil, errs.Wrap(err)
|
return nil, errs.Wrap(err)
|
||||||
}
|
}
|
||||||
@ -84,7 +84,7 @@ func (slow *SlowBlobs) Open(ctx context.Context, ref storage.BlobRef) (storage.B
|
|||||||
|
|
||||||
// OpenWithStorageFormat opens a reader for the already-located blob, avoiding the potential need
|
// OpenWithStorageFormat opens a reader for the already-located blob, avoiding the potential need
|
||||||
// to check multiple storage formats to find the blob.
|
// to check multiple storage formats to find the blob.
|
||||||
func (slow *SlowBlobs) OpenWithStorageFormat(ctx context.Context, ref storage.BlobRef, formatVer storage.FormatVersion) (storage.BlobReader, error) {
|
func (slow *SlowBlobs) OpenWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (blobstore.BlobReader, error) {
|
||||||
if err := slow.sleep(ctx); err != nil {
|
if err := slow.sleep(ctx); err != nil {
|
||||||
return nil, errs.Wrap(err)
|
return nil, errs.Wrap(err)
|
||||||
}
|
}
|
||||||
@ -92,7 +92,7 @@ func (slow *SlowBlobs) OpenWithStorageFormat(ctx context.Context, ref storage.Bl
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Trash deletes the blob with the namespace and key.
|
// Trash deletes the blob with the namespace and key.
|
||||||
func (slow *SlowBlobs) Trash(ctx context.Context, ref storage.BlobRef) error {
|
func (slow *SlowBlobs) Trash(ctx context.Context, ref blobstore.BlobRef) error {
|
||||||
if err := slow.sleep(ctx); err != nil {
|
if err := slow.sleep(ctx); err != nil {
|
||||||
return errs.Wrap(err)
|
return errs.Wrap(err)
|
||||||
}
|
}
|
||||||
@ -116,7 +116,7 @@ func (slow *SlowBlobs) EmptyTrash(ctx context.Context, namespace []byte, trashed
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Delete deletes the blob with the namespace and key.
|
// Delete deletes the blob with the namespace and key.
|
||||||
func (slow *SlowBlobs) Delete(ctx context.Context, ref storage.BlobRef) error {
|
func (slow *SlowBlobs) Delete(ctx context.Context, ref blobstore.BlobRef) error {
|
||||||
if err := slow.sleep(ctx); err != nil {
|
if err := slow.sleep(ctx); err != nil {
|
||||||
return errs.Wrap(err)
|
return errs.Wrap(err)
|
||||||
}
|
}
|
||||||
@ -124,7 +124,7 @@ func (slow *SlowBlobs) Delete(ctx context.Context, ref storage.BlobRef) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// DeleteWithStorageFormat deletes the blob with the namespace, key, and format version.
|
// DeleteWithStorageFormat deletes the blob with the namespace, key, and format version.
|
||||||
func (slow *SlowBlobs) DeleteWithStorageFormat(ctx context.Context, ref storage.BlobRef, formatVer storage.FormatVersion) error {
|
func (slow *SlowBlobs) DeleteWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) error {
|
||||||
if err := slow.sleep(ctx); err != nil {
|
if err := slow.sleep(ctx); err != nil {
|
||||||
return errs.Wrap(err)
|
return errs.Wrap(err)
|
||||||
}
|
}
|
||||||
@ -140,7 +140,7 @@ func (slow *SlowBlobs) DeleteNamespace(ctx context.Context, ref []byte) (err err
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Stat looks up disk metadata on the blob file.
|
// Stat looks up disk metadata on the blob file.
|
||||||
func (slow *SlowBlobs) Stat(ctx context.Context, ref storage.BlobRef) (storage.BlobInfo, error) {
|
func (slow *SlowBlobs) Stat(ctx context.Context, ref blobstore.BlobRef) (blobstore.BlobInfo, error) {
|
||||||
if err := slow.sleep(ctx); err != nil {
|
if err := slow.sleep(ctx); err != nil {
|
||||||
return nil, errs.Wrap(err)
|
return nil, errs.Wrap(err)
|
||||||
}
|
}
|
||||||
@ -150,7 +150,7 @@ func (slow *SlowBlobs) Stat(ctx context.Context, ref storage.BlobRef) (storage.B
|
|||||||
// StatWithStorageFormat looks up disk metadata for the blob file with the given storage format
|
// StatWithStorageFormat looks up disk metadata for the blob file with the given storage format
|
||||||
// version. This avoids the potential need to check multiple storage formats for the blob
|
// version. This avoids the potential need to check multiple storage formats for the blob
|
||||||
// when the format is already known.
|
// when the format is already known.
|
||||||
func (slow *SlowBlobs) StatWithStorageFormat(ctx context.Context, ref storage.BlobRef, formatVer storage.FormatVersion) (storage.BlobInfo, error) {
|
func (slow *SlowBlobs) StatWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (blobstore.BlobInfo, error) {
|
||||||
if err := slow.sleep(ctx); err != nil {
|
if err := slow.sleep(ctx); err != nil {
|
||||||
return nil, errs.Wrap(err)
|
return nil, errs.Wrap(err)
|
||||||
}
|
}
|
||||||
@ -160,7 +160,7 @@ func (slow *SlowBlobs) StatWithStorageFormat(ctx context.Context, ref storage.Bl
|
|||||||
// WalkNamespace executes walkFunc for each locally stored blob in the given namespace.
|
// WalkNamespace executes walkFunc for each locally stored blob in the given namespace.
|
||||||
// If walkFunc returns a non-nil error, WalkNamespace will stop iterating and return the
|
// If walkFunc returns a non-nil error, WalkNamespace will stop iterating and return the
|
||||||
// error immediately.
|
// error immediately.
|
||||||
func (slow *SlowBlobs) WalkNamespace(ctx context.Context, namespace []byte, walkFunc func(storage.BlobInfo) error) error {
|
func (slow *SlowBlobs) WalkNamespace(ctx context.Context, namespace []byte, walkFunc func(blobstore.BlobInfo) error) error {
|
||||||
if err := slow.sleep(ctx); err != nil {
|
if err := slow.sleep(ctx); err != nil {
|
||||||
return errs.Wrap(err)
|
return errs.Wrap(err)
|
||||||
}
|
}
|
@ -17,7 +17,7 @@ import (
|
|||||||
"storj.io/common/testrand"
|
"storj.io/common/testrand"
|
||||||
"storj.io/storj/private/testplanet"
|
"storj.io/storj/private/testplanet"
|
||||||
"storj.io/storj/satellite/overlay"
|
"storj.io/storj/satellite/overlay"
|
||||||
"storj.io/storj/storage"
|
"storj.io/storj/storagenode/blobstore"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestChore(t *testing.T) {
|
func TestChore(t *testing.T) {
|
||||||
@ -129,7 +129,7 @@ func exitSatellite(ctx context.Context, t *testing.T, planet *testplanet.Planet,
|
|||||||
namespaces, err := exitingNode.DB.Pieces().ListNamespaces(ctx)
|
namespaces, err := exitingNode.DB.Pieces().ListNamespaces(ctx)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
for _, ns := range namespaces {
|
for _, ns := range namespaces {
|
||||||
err = exitingNode.DB.Pieces().WalkNamespace(ctx, ns, func(blobInfo storage.BlobInfo) error {
|
err = exitingNode.DB.Pieces().WalkNamespace(ctx, ns, func(blobInfo blobstore.BlobInfo) error {
|
||||||
return errs.New("found a piece on the node. this shouldn't happen.")
|
return errs.New("found a piece on the node. this shouldn't happen.")
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@ -146,7 +146,7 @@ func getNodePieceCounts(ctx context.Context, planet *testplanet.Planet) (_ map[s
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
for _, ns := range namespaces {
|
for _, ns := range namespaces {
|
||||||
err = node.DB.Pieces().WalkNamespace(ctx, ns, func(blobInfo storage.BlobInfo) error {
|
err = node.DB.Pieces().WalkNamespace(ctx, ns, func(blobInfo blobstore.BlobInfo) error {
|
||||||
nodePieceCounts[node.ID()]++
|
nodePieceCounts[node.ID()]++
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
@ -16,11 +16,11 @@ import (
|
|||||||
"storj.io/common/rpc/rpcstatus"
|
"storj.io/common/rpc/rpcstatus"
|
||||||
"storj.io/common/testcontext"
|
"storj.io/common/testcontext"
|
||||||
"storj.io/common/testrand"
|
"storj.io/common/testrand"
|
||||||
"storj.io/storj/private/testblobs"
|
|
||||||
"storj.io/storj/private/testplanet"
|
"storj.io/storj/private/testplanet"
|
||||||
"storj.io/storj/satellite"
|
"storj.io/storj/satellite"
|
||||||
"storj.io/storj/satellite/overlay"
|
"storj.io/storj/satellite/overlay"
|
||||||
"storj.io/storj/storagenode"
|
"storj.io/storj/storagenode"
|
||||||
|
"storj.io/storj/storagenode/blobstore/testblobs"
|
||||||
"storj.io/storj/storagenode/gracefulexit"
|
"storj.io/storj/storagenode/gracefulexit"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -30,10 +30,10 @@ import (
|
|||||||
"storj.io/storj/private/multinodepb"
|
"storj.io/storj/private/multinodepb"
|
||||||
"storj.io/storj/private/server"
|
"storj.io/storj/private/server"
|
||||||
"storj.io/storj/private/version/checker"
|
"storj.io/storj/private/version/checker"
|
||||||
"storj.io/storj/storage"
|
|
||||||
"storj.io/storj/storage/filestore"
|
|
||||||
"storj.io/storj/storagenode/apikeys"
|
"storj.io/storj/storagenode/apikeys"
|
||||||
"storj.io/storj/storagenode/bandwidth"
|
"storj.io/storj/storagenode/bandwidth"
|
||||||
|
"storj.io/storj/storagenode/blobstore"
|
||||||
|
"storj.io/storj/storagenode/blobstore/filestore"
|
||||||
"storj.io/storj/storagenode/collector"
|
"storj.io/storj/storagenode/collector"
|
||||||
"storj.io/storj/storagenode/console"
|
"storj.io/storj/storagenode/console"
|
||||||
"storj.io/storj/storagenode/console/consoleserver"
|
"storj.io/storj/storagenode/console/consoleserver"
|
||||||
@ -80,7 +80,7 @@ type DB interface {
|
|||||||
// Close closes the database
|
// Close closes the database
|
||||||
Close() error
|
Close() error
|
||||||
|
|
||||||
Pieces() storage.Blobs
|
Pieces() blobstore.Blobs
|
||||||
|
|
||||||
Orders() orders.DB
|
Orders() orders.DB
|
||||||
V0PieceInfo() pieces.V0PieceInfoDB
|
V0PieceInfo() pieces.V0PieceInfoDB
|
||||||
|
@ -14,7 +14,7 @@ import (
|
|||||||
|
|
||||||
"storj.io/common/storj"
|
"storj.io/common/storj"
|
||||||
"storj.io/common/sync2"
|
"storj.io/common/sync2"
|
||||||
"storj.io/storj/storage"
|
"storj.io/storj/storagenode/blobstore"
|
||||||
)
|
)
|
||||||
|
|
||||||
// CacheService updates the space used cache.
|
// CacheService updates the space used cache.
|
||||||
@ -157,7 +157,7 @@ func (service *CacheService) Close() (err error) {
|
|||||||
//
|
//
|
||||||
// architecture: Database
|
// architecture: Database
|
||||||
type BlobsUsageCache struct {
|
type BlobsUsageCache struct {
|
||||||
storage.Blobs
|
blobstore.Blobs
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
@ -168,7 +168,7 @@ type BlobsUsageCache struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewBlobsUsageCache creates a new disk blob store with a space used cache.
|
// NewBlobsUsageCache creates a new disk blob store with a space used cache.
|
||||||
func NewBlobsUsageCache(log *zap.Logger, blob storage.Blobs) *BlobsUsageCache {
|
func NewBlobsUsageCache(log *zap.Logger, blob blobstore.Blobs) *BlobsUsageCache {
|
||||||
return &BlobsUsageCache{
|
return &BlobsUsageCache{
|
||||||
log: log,
|
log: log,
|
||||||
Blobs: blob,
|
Blobs: blob,
|
||||||
@ -177,7 +177,7 @@ func NewBlobsUsageCache(log *zap.Logger, blob storage.Blobs) *BlobsUsageCache {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewBlobsUsageCacheTest creates a new disk blob store with a space used cache.
|
// NewBlobsUsageCacheTest creates a new disk blob store with a space used cache.
|
||||||
func NewBlobsUsageCacheTest(log *zap.Logger, blob storage.Blobs, piecesTotal, piecesContentSize, trashTotal int64, spaceUsedBySatellite map[storj.NodeID]SatelliteUsage) *BlobsUsageCache {
|
func NewBlobsUsageCacheTest(log *zap.Logger, blob blobstore.Blobs, piecesTotal, piecesContentSize, trashTotal int64, spaceUsedBySatellite map[storj.NodeID]SatelliteUsage) *BlobsUsageCache {
|
||||||
return &BlobsUsageCache{
|
return &BlobsUsageCache{
|
||||||
log: log,
|
log: log,
|
||||||
Blobs: blob,
|
Blobs: blob,
|
||||||
@ -222,7 +222,7 @@ func (blobs *BlobsUsageCache) SpaceUsedForTrash(ctx context.Context) (int64, err
|
|||||||
|
|
||||||
// Delete gets the size of the piece that is going to be deleted then deletes it and
|
// Delete gets the size of the piece that is going to be deleted then deletes it and
|
||||||
// updates the space used cache accordingly.
|
// updates the space used cache accordingly.
|
||||||
func (blobs *BlobsUsageCache) Delete(ctx context.Context, blobRef storage.BlobRef) error {
|
func (blobs *BlobsUsageCache) Delete(ctx context.Context, blobRef blobstore.BlobRef) error {
|
||||||
pieceTotal, pieceContentSize, err := blobs.pieceSizes(ctx, blobRef)
|
pieceTotal, pieceContentSize, err := blobs.pieceSizes(ctx, blobRef)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Error.Wrap(err)
|
return Error.Wrap(err)
|
||||||
@ -241,7 +241,7 @@ func (blobs *BlobsUsageCache) Delete(ctx context.Context, blobRef storage.BlobRe
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (blobs *BlobsUsageCache) pieceSizes(ctx context.Context, blobRef storage.BlobRef) (pieceTotal int64, pieceContentSize int64, err error) {
|
func (blobs *BlobsUsageCache) pieceSizes(ctx context.Context, blobRef blobstore.BlobRef) (pieceTotal int64, pieceContentSize int64, err error) {
|
||||||
blobInfo, err := blobs.Stat(ctx, blobRef)
|
blobInfo, err := blobs.Stat(ctx, blobRef)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, err
|
return 0, 0, err
|
||||||
@ -286,7 +286,7 @@ func (blobs *BlobsUsageCache) ensurePositiveCacheValue(value *int64, name string
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Trash moves the ref to the trash and updates the cache.
|
// Trash moves the ref to the trash and updates the cache.
|
||||||
func (blobs *BlobsUsageCache) Trash(ctx context.Context, blobRef storage.BlobRef) error {
|
func (blobs *BlobsUsageCache) Trash(ctx context.Context, blobRef blobstore.BlobRef) error {
|
||||||
pieceTotal, pieceContentSize, err := blobs.pieceSizes(ctx, blobRef)
|
pieceTotal, pieceContentSize, err := blobs.pieceSizes(ctx, blobRef)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Error.Wrap(err)
|
return Error.Wrap(err)
|
||||||
@ -336,7 +336,7 @@ func (blobs *BlobsUsageCache) RestoreTrash(ctx context.Context, namespace []byte
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, key := range keysRestored {
|
for _, key := range keysRestored {
|
||||||
pieceTotal, pieceContentSize, sizeErr := blobs.pieceSizes(ctx, storage.BlobRef{
|
pieceTotal, pieceContentSize, sizeErr := blobs.pieceSizes(ctx, blobstore.BlobRef{
|
||||||
Key: key,
|
Key: key,
|
||||||
Namespace: namespace,
|
Namespace: namespace,
|
||||||
})
|
})
|
||||||
@ -492,9 +492,9 @@ func (blobs *BlobsUsageCache) Close() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestCreateV0 creates a new V0 blob that can be written. This is only appropriate in test situations.
|
// TestCreateV0 creates a new V0 blob that can be written. This is only appropriate in test situations.
|
||||||
func (blobs *BlobsUsageCache) TestCreateV0(ctx context.Context, ref storage.BlobRef) (_ storage.BlobWriter, err error) {
|
func (blobs *BlobsUsageCache) TestCreateV0(ctx context.Context, ref blobstore.BlobRef) (_ blobstore.BlobWriter, err error) {
|
||||||
fStore := blobs.Blobs.(interface {
|
fStore := blobs.Blobs.(interface {
|
||||||
TestCreateV0(ctx context.Context, ref storage.BlobRef) (_ storage.BlobWriter, err error)
|
TestCreateV0(ctx context.Context, ref blobstore.BlobRef) (_ blobstore.BlobWriter, err error)
|
||||||
})
|
})
|
||||||
return fStore.TestCreateV0(ctx, ref)
|
return fStore.TestCreateV0(ctx, ref)
|
||||||
}
|
}
|
||||||
|
@ -18,9 +18,9 @@ import (
|
|||||||
"storj.io/common/testcontext"
|
"storj.io/common/testcontext"
|
||||||
"storj.io/common/testrand"
|
"storj.io/common/testrand"
|
||||||
"storj.io/storj/private/testplanet"
|
"storj.io/storj/private/testplanet"
|
||||||
"storj.io/storj/storage"
|
|
||||||
"storj.io/storj/storage/filestore"
|
|
||||||
"storj.io/storj/storagenode"
|
"storj.io/storj/storagenode"
|
||||||
|
"storj.io/storj/storagenode/blobstore"
|
||||||
|
"storj.io/storj/storagenode/blobstore/filestore"
|
||||||
"storj.io/storj/storagenode/pieces"
|
"storj.io/storj/storagenode/pieces"
|
||||||
"storj.io/storj/storagenode/storagenodedb/storagenodedbtest"
|
"storj.io/storj/storagenode/storagenodedb/storagenodedbtest"
|
||||||
)
|
)
|
||||||
@ -195,13 +195,13 @@ func TestCachServiceRun(t *testing.T) {
|
|||||||
storagenodedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db storagenode.DB) {
|
storagenodedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db storagenode.DB) {
|
||||||
spaceUsedDB := db.PieceSpaceUsedDB()
|
spaceUsedDB := db.PieceSpaceUsedDB()
|
||||||
|
|
||||||
blobstore, err := filestore.NewAt(log, ctx.Dir(), filestore.DefaultConfig)
|
store, err := filestore.NewAt(log, ctx.Dir(), filestore.DefaultConfig)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Prior to initializing the cache service (which should walk the files),
|
// Prior to initializing the cache service (which should walk the files),
|
||||||
// write a single file so something exists to be counted
|
// write a single file so something exists to be counted
|
||||||
expBlobSize := memory.KB
|
expBlobSize := memory.KB
|
||||||
w, err := blobstore.Create(ctx, storage.BlobRef{
|
w, err := store.Create(ctx, blobstore.BlobRef{
|
||||||
Namespace: testrand.NodeID().Bytes(),
|
Namespace: testrand.NodeID().Bytes(),
|
||||||
Key: testrand.PieceID().Bytes(),
|
Key: testrand.PieceID().Bytes(),
|
||||||
}, -1)
|
}, -1)
|
||||||
@ -212,19 +212,19 @@ func TestCachServiceRun(t *testing.T) {
|
|||||||
|
|
||||||
// Now write a piece that we are going to trash
|
// Now write a piece that we are going to trash
|
||||||
expTrashSize := 2 * memory.KB
|
expTrashSize := 2 * memory.KB
|
||||||
trashRef := storage.BlobRef{
|
trashRef := blobstore.BlobRef{
|
||||||
Namespace: testrand.NodeID().Bytes(),
|
Namespace: testrand.NodeID().Bytes(),
|
||||||
Key: testrand.PieceID().Bytes(),
|
Key: testrand.PieceID().Bytes(),
|
||||||
}
|
}
|
||||||
w, err = blobstore.Create(ctx, trashRef, -1)
|
w, err = store.Create(ctx, trashRef, -1)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
_, err = w.Write(testrand.Bytes(expTrashSize))
|
_, err = w.Write(testrand.Bytes(expTrashSize))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NoError(t, w.Commit(ctx))
|
require.NoError(t, w.Commit(ctx))
|
||||||
require.NoError(t, blobstore.Trash(ctx, trashRef)) // trash it
|
require.NoError(t, store.Trash(ctx, trashRef)) // trash it
|
||||||
|
|
||||||
// Now instantiate the cache
|
// Now instantiate the cache
|
||||||
cache := pieces.NewBlobsUsageCache(log, blobstore)
|
cache := pieces.NewBlobsUsageCache(log, store)
|
||||||
cacheService := pieces.NewService(log,
|
cacheService := pieces.NewService(log,
|
||||||
cache,
|
cache,
|
||||||
pieces.NewStore(log, pieces.NewFileWalker(log, cache, nil), cache, nil, nil, spaceUsedDB, pieces.DefaultConfig),
|
pieces.NewStore(log, pieces.NewFileWalker(log, cache, nil), cache, nil, nil, spaceUsedDB, pieces.DefaultConfig),
|
||||||
@ -591,7 +591,7 @@ func TestCacheCreateDeleteAndTrash(t *testing.T) {
|
|||||||
cache := pieces.NewBlobsUsageCache(zaptest.NewLogger(t), db.Pieces())
|
cache := pieces.NewBlobsUsageCache(zaptest.NewLogger(t), db.Pieces())
|
||||||
pieceContent := []byte("stuff")
|
pieceContent := []byte("stuff")
|
||||||
satelliteID := testrand.NodeID()
|
satelliteID := testrand.NodeID()
|
||||||
refs := []storage.BlobRef{
|
refs := []blobstore.BlobRef{
|
||||||
{
|
{
|
||||||
Namespace: satelliteID.Bytes(),
|
Namespace: satelliteID.Bytes(),
|
||||||
Key: testrand.Bytes(32),
|
Key: testrand.Bytes(32),
|
||||||
|
@ -15,8 +15,8 @@ import (
|
|||||||
"storj.io/common/storj"
|
"storj.io/common/storj"
|
||||||
"storj.io/common/testcontext"
|
"storj.io/common/testcontext"
|
||||||
"storj.io/common/testrand"
|
"storj.io/common/testrand"
|
||||||
"storj.io/storj/storage/filestore"
|
|
||||||
"storj.io/storj/storagenode"
|
"storj.io/storj/storagenode"
|
||||||
|
"storj.io/storj/storagenode/blobstore/filestore"
|
||||||
"storj.io/storj/storagenode/pieces"
|
"storj.io/storj/storagenode/pieces"
|
||||||
"storj.io/storj/storagenode/storagenodedb/storagenodedbtest"
|
"storj.io/storj/storagenode/storagenodedb/storagenodedbtest"
|
||||||
)
|
)
|
||||||
|
@ -9,20 +9,20 @@ import (
|
|||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
|
||||||
"storj.io/common/storj"
|
"storj.io/common/storj"
|
||||||
"storj.io/storj/storage"
|
"storj.io/storj/storagenode/blobstore"
|
||||||
"storj.io/storj/storage/filestore"
|
"storj.io/storj/storagenode/blobstore/filestore"
|
||||||
)
|
)
|
||||||
|
|
||||||
// FileWalker implements methods to walk over pieces in a storage directory.
|
// FileWalker implements methods to walk over pieces in a storage directory.
|
||||||
type FileWalker struct {
|
type FileWalker struct {
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
|
|
||||||
blobs storage.Blobs
|
blobs blobstore.Blobs
|
||||||
v0PieceInfo V0PieceInfoDB
|
v0PieceInfo V0PieceInfoDB
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewFileWalker creates a new FileWalker.
|
// NewFileWalker creates a new FileWalker.
|
||||||
func NewFileWalker(log *zap.Logger, blobs storage.Blobs, db V0PieceInfoDB) *FileWalker {
|
func NewFileWalker(log *zap.Logger, blobs blobstore.Blobs, db V0PieceInfoDB) *FileWalker {
|
||||||
return &FileWalker{
|
return &FileWalker{
|
||||||
log: log,
|
log: log,
|
||||||
blobs: blobs,
|
blobs: blobs,
|
||||||
@ -39,7 +39,7 @@ func NewFileWalker(log *zap.Logger, blobs storage.Blobs, db V0PieceInfoDB) *File
|
|||||||
func (fw *FileWalker) WalkSatellitePieces(ctx context.Context, satellite storj.NodeID, fn func(StoredPieceAccess) error) (err error) {
|
func (fw *FileWalker) WalkSatellitePieces(ctx context.Context, satellite storj.NodeID, fn func(StoredPieceAccess) error) (err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
// iterate over all in V1 storage, skipping v0 pieces
|
// iterate over all in V1 storage, skipping v0 pieces
|
||||||
err = fw.blobs.WalkNamespace(ctx, satellite.Bytes(), func(blobInfo storage.BlobInfo) error {
|
err = fw.blobs.WalkNamespace(ctx, satellite.Bytes(), func(blobInfo blobstore.BlobInfo) error {
|
||||||
if blobInfo.StorageFormatVersion() < filestore.FormatV1 {
|
if blobInfo.StorageFormatVersion() < filestore.FormatV1 {
|
||||||
// skip v0 pieces, which are handled separately
|
// skip v0 pieces, which are handled separately
|
||||||
return nil
|
return nil
|
||||||
|
@ -15,8 +15,8 @@ import (
|
|||||||
|
|
||||||
"storj.io/common/pb"
|
"storj.io/common/pb"
|
||||||
"storj.io/common/storj"
|
"storj.io/common/storj"
|
||||||
"storj.io/storj/storage"
|
"storj.io/storj/storagenode/blobstore"
|
||||||
"storj.io/storj/storage/filestore"
|
"storj.io/storj/storagenode/blobstore/filestore"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -63,16 +63,16 @@ var BadFormatVersion = errs.Class("Incompatible storage format version")
|
|||||||
type Writer struct {
|
type Writer struct {
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
hash hash.Hash
|
hash hash.Hash
|
||||||
blob storage.BlobWriter
|
blob blobstore.BlobWriter
|
||||||
pieceSize int64 // piece size only; i.e., not including piece header
|
pieceSize int64 // piece size only; i.e., not including piece header
|
||||||
|
|
||||||
blobs storage.Blobs
|
blobs blobstore.Blobs
|
||||||
satellite storj.NodeID
|
satellite storj.NodeID
|
||||||
closed bool
|
closed bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewWriter creates a new writer for storage.BlobWriter.
|
// NewWriter creates a new writer for blobstore.BlobWriter.
|
||||||
func NewWriter(log *zap.Logger, blobWriter storage.BlobWriter, blobs storage.Blobs, satellite storj.NodeID, hashAlgorithm pb.PieceHashAlgorithm) (*Writer, error) {
|
func NewWriter(log *zap.Logger, blobWriter blobstore.BlobWriter, blobs blobstore.Blobs, satellite storj.NodeID, hashAlgorithm pb.PieceHashAlgorithm) (*Writer, error) {
|
||||||
w := &Writer{log: log}
|
w := &Writer{log: log}
|
||||||
if blobWriter.StorageFormatVersion() >= filestore.FormatV1 {
|
if blobWriter.StorageFormatVersion() >= filestore.FormatV1 {
|
||||||
// We skip past the reserved header area for now- we want the header to be at the
|
// We skip past the reserved header area for now- we want the header to be at the
|
||||||
@ -211,15 +211,15 @@ func (w *Writer) Cancel(ctx context.Context) (err error) {
|
|||||||
|
|
||||||
// Reader implements a piece reader that reads content from blob store.
|
// Reader implements a piece reader that reads content from blob store.
|
||||||
type Reader struct {
|
type Reader struct {
|
||||||
formatVersion storage.FormatVersion
|
formatVersion blobstore.FormatVersion
|
||||||
|
|
||||||
blob storage.BlobReader
|
blob blobstore.BlobReader
|
||||||
pos int64 // relative to file start; i.e., it includes piece header
|
pos int64 // relative to file start; i.e., it includes piece header
|
||||||
pieceSize int64 // piece size only; i.e., not including piece header
|
pieceSize int64 // piece size only; i.e., not including piece header
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewReader creates a new reader for storage.BlobReader.
|
// NewReader creates a new reader for blobstore.BlobReader.
|
||||||
func NewReader(blob storage.BlobReader) (*Reader, error) {
|
func NewReader(blob blobstore.BlobReader) (*Reader, error) {
|
||||||
size, err := blob.Size()
|
size, err := blob.Size()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, Error.Wrap(err)
|
return nil, Error.Wrap(err)
|
||||||
@ -241,7 +241,7 @@ func NewReader(blob storage.BlobReader) (*Reader, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// StorageFormatVersion returns the storage format version of the piece being read.
|
// StorageFormatVersion returns the storage format version of the piece being read.
|
||||||
func (r *Reader) StorageFormatVersion() storage.FormatVersion {
|
func (r *Reader) StorageFormatVersion() blobstore.FormatVersion {
|
||||||
return r.formatVersion
|
return r.formatVersion
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -19,8 +19,8 @@ import (
|
|||||||
"storj.io/common/storj"
|
"storj.io/common/storj"
|
||||||
"storj.io/common/testcontext"
|
"storj.io/common/testcontext"
|
||||||
"storj.io/common/testrand"
|
"storj.io/common/testrand"
|
||||||
"storj.io/storj/storage"
|
"storj.io/storj/storagenode/blobstore"
|
||||||
"storj.io/storj/storage/filestore"
|
"storj.io/storj/storagenode/blobstore/filestore"
|
||||||
"storj.io/storj/storagenode/pieces"
|
"storj.io/storj/storagenode/pieces"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -158,7 +158,7 @@ func readAndWritePiece(t *testing.T, content []byte) {
|
|||||||
assert.Truef(t, header.OrderLimit.PieceExpiration.Equal(expirationTime),
|
assert.Truef(t, header.OrderLimit.PieceExpiration.Equal(expirationTime),
|
||||||
"*header.ExpirationTime = %s, but expected expirationTime = %s", header.OrderLimit.PieceExpiration, expirationTime)
|
"*header.ExpirationTime = %s, but expected expirationTime = %s", header.OrderLimit.PieceExpiration, expirationTime)
|
||||||
assert.Equal(t, pb.OrderLimit{PieceExpiration: expirationTime.UTC()}, header.OrderLimit)
|
assert.Equal(t, pb.OrderLimit{PieceExpiration: expirationTime.UTC()}, header.OrderLimit)
|
||||||
assert.Equal(t, filestore.FormatV1, storage.FormatVersion(header.FormatVersion))
|
assert.Equal(t, filestore.FormatV1, blobstore.FormatVersion(header.FormatVersion))
|
||||||
|
|
||||||
// make sure seek-nowhere works as expected after piece header is read too
|
// make sure seek-nowhere works as expected after piece header is read too
|
||||||
// (from the point of view of the piece store, the file position has not moved)
|
// (from the point of view of the piece store, the file position has not moved)
|
||||||
|
@ -17,8 +17,8 @@ import (
|
|||||||
"storj.io/common/memory"
|
"storj.io/common/memory"
|
||||||
"storj.io/common/pb"
|
"storj.io/common/pb"
|
||||||
"storj.io/common/storj"
|
"storj.io/common/storj"
|
||||||
"storj.io/storj/storage"
|
"storj.io/storj/storagenode/blobstore"
|
||||||
"storj.io/storj/storage/filestore"
|
"storj.io/storj/storagenode/blobstore/filestore"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@ -72,7 +72,7 @@ type PieceExpirationDB interface {
|
|||||||
|
|
||||||
// V0PieceInfoDB stores meta information about pieces stored with storage format V0 (where
|
// V0PieceInfoDB stores meta information about pieces stored with storage format V0 (where
|
||||||
// metadata goes in the "pieceinfo" table in the storagenodedb). The actual pieces are stored
|
// metadata goes in the "pieceinfo" table in the storagenodedb). The actual pieces are stored
|
||||||
// behind something providing the storage.Blobs interface.
|
// behind something providing the blobstore.Blobs interface.
|
||||||
//
|
//
|
||||||
// architecture: Database
|
// architecture: Database
|
||||||
type V0PieceInfoDB interface {
|
type V0PieceInfoDB interface {
|
||||||
@ -90,7 +90,7 @@ type V0PieceInfoDB interface {
|
|||||||
// non-nil error, WalkSatelliteV0Pieces will stop iterating and return the error
|
// non-nil error, WalkSatelliteV0Pieces will stop iterating and return the error
|
||||||
// immediately. The ctx parameter is intended specifically to allow canceling iteration
|
// immediately. The ctx parameter is intended specifically to allow canceling iteration
|
||||||
// early.
|
// early.
|
||||||
WalkSatelliteV0Pieces(ctx context.Context, blobStore storage.Blobs, satellite storj.NodeID, walkFunc func(StoredPieceAccess) error) error
|
WalkSatelliteV0Pieces(ctx context.Context, blobStore blobstore.Blobs, satellite storj.NodeID, walkFunc func(StoredPieceAccess) error) error
|
||||||
}
|
}
|
||||||
|
|
||||||
// V0PieceInfoDBForTest is like V0PieceInfoDB, but adds on the Add() method so
|
// V0PieceInfoDBForTest is like V0PieceInfoDB, but adds on the Add() method so
|
||||||
@ -127,7 +127,7 @@ type PieceSpaceUsedDB interface {
|
|||||||
// StoredPieceAccess allows inspection and manipulation of a piece during iteration with
|
// StoredPieceAccess allows inspection and manipulation of a piece during iteration with
|
||||||
// WalkSatellitePieces-type methods.
|
// WalkSatellitePieces-type methods.
|
||||||
type StoredPieceAccess interface {
|
type StoredPieceAccess interface {
|
||||||
storage.BlobInfo
|
blobstore.BlobInfo
|
||||||
|
|
||||||
// PieceID gives the pieceID of the piece
|
// PieceID gives the pieceID of the piece
|
||||||
PieceID() storj.PieceID
|
PieceID() storj.PieceID
|
||||||
@ -171,7 +171,7 @@ type Store struct {
|
|||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
config Config
|
config Config
|
||||||
|
|
||||||
blobs storage.Blobs
|
blobs blobstore.Blobs
|
||||||
expirationInfo PieceExpirationDB
|
expirationInfo PieceExpirationDB
|
||||||
spaceUsedDB PieceSpaceUsedDB
|
spaceUsedDB PieceSpaceUsedDB
|
||||||
v0PieceInfo V0PieceInfoDB
|
v0PieceInfo V0PieceInfoDB
|
||||||
@ -186,7 +186,7 @@ type StoreForTest struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewStore creates a new piece store.
|
// NewStore creates a new piece store.
|
||||||
func NewStore(log *zap.Logger, fw *FileWalker, blobs storage.Blobs, v0PieceInfo V0PieceInfoDB, expirationInfo PieceExpirationDB, spaceUsedDB PieceSpaceUsedDB, config Config) *Store {
|
func NewStore(log *zap.Logger, fw *FileWalker, blobs blobstore.Blobs, v0PieceInfo V0PieceInfoDB, expirationInfo PieceExpirationDB, spaceUsedDB PieceSpaceUsedDB, config Config) *Store {
|
||||||
return &Store{
|
return &Store{
|
||||||
log: log,
|
log: log,
|
||||||
config: config,
|
config: config,
|
||||||
@ -231,7 +231,7 @@ func (store *Store) VerifyStorageDirWithTimeout(ctx context.Context, id storj.No
|
|||||||
// Writer returns a new piece writer.
|
// Writer returns a new piece writer.
|
||||||
func (store *Store) Writer(ctx context.Context, satellite storj.NodeID, pieceID storj.PieceID, hashAlgorithm pb.PieceHashAlgorithm) (_ *Writer, err error) {
|
func (store *Store) Writer(ctx context.Context, satellite storj.NodeID, pieceID storj.PieceID, hashAlgorithm pb.PieceHashAlgorithm) (_ *Writer, err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
blobWriter, err := store.blobs.Create(ctx, storage.BlobRef{
|
blobWriter, err := store.blobs.Create(ctx, blobstore.BlobRef{
|
||||||
Namespace: satellite.Bytes(),
|
Namespace: satellite.Bytes(),
|
||||||
Key: pieceID.Bytes(),
|
Key: pieceID.Bytes(),
|
||||||
}, store.config.WritePreallocSize.Int64())
|
}, store.config.WritePreallocSize.Int64())
|
||||||
@ -247,19 +247,19 @@ func (store *Store) Writer(ctx context.Context, satellite storj.NodeID, pieceID
|
|||||||
// This is meant to be used externally only in test situations (thus the StoreForTest receiver
|
// This is meant to be used externally only in test situations (thus the StoreForTest receiver
|
||||||
// type).
|
// type).
|
||||||
func (store StoreForTest) WriterForFormatVersion(ctx context.Context, satellite storj.NodeID,
|
func (store StoreForTest) WriterForFormatVersion(ctx context.Context, satellite storj.NodeID,
|
||||||
pieceID storj.PieceID, formatVersion storage.FormatVersion, hashAlgorithm pb.PieceHashAlgorithm) (_ *Writer, err error) {
|
pieceID storj.PieceID, formatVersion blobstore.FormatVersion, hashAlgorithm pb.PieceHashAlgorithm) (_ *Writer, err error) {
|
||||||
|
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
|
|
||||||
blobRef := storage.BlobRef{
|
blobRef := blobstore.BlobRef{
|
||||||
Namespace: satellite.Bytes(),
|
Namespace: satellite.Bytes(),
|
||||||
Key: pieceID.Bytes(),
|
Key: pieceID.Bytes(),
|
||||||
}
|
}
|
||||||
var blobWriter storage.BlobWriter
|
var blobWriter blobstore.BlobWriter
|
||||||
switch formatVersion {
|
switch formatVersion {
|
||||||
case filestore.FormatV0:
|
case filestore.FormatV0:
|
||||||
fStore, ok := store.blobs.(interface {
|
fStore, ok := store.blobs.(interface {
|
||||||
TestCreateV0(ctx context.Context, ref storage.BlobRef) (_ storage.BlobWriter, err error)
|
TestCreateV0(ctx context.Context, ref blobstore.BlobRef) (_ blobstore.BlobWriter, err error)
|
||||||
})
|
})
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, Error.New("can't make a WriterForFormatVersion with this blob store (%T)", store.blobs)
|
return nil, Error.New("can't make a WriterForFormatVersion with this blob store (%T)", store.blobs)
|
||||||
@ -280,10 +280,10 @@ func (store StoreForTest) WriterForFormatVersion(ctx context.Context, satellite
|
|||||||
// ReaderWithStorageFormat returns a new piece reader for a located piece, which avoids the
|
// ReaderWithStorageFormat returns a new piece reader for a located piece, which avoids the
|
||||||
// potential need to check multiple storage formats to find the right blob.
|
// potential need to check multiple storage formats to find the right blob.
|
||||||
func (store *StoreForTest) ReaderWithStorageFormat(ctx context.Context, satellite storj.NodeID,
|
func (store *StoreForTest) ReaderWithStorageFormat(ctx context.Context, satellite storj.NodeID,
|
||||||
pieceID storj.PieceID, formatVersion storage.FormatVersion) (_ *Reader, err error) {
|
pieceID storj.PieceID, formatVersion blobstore.FormatVersion) (_ *Reader, err error) {
|
||||||
|
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
ref := storage.BlobRef{Namespace: satellite.Bytes(), Key: pieceID.Bytes()}
|
ref := blobstore.BlobRef{Namespace: satellite.Bytes(), Key: pieceID.Bytes()}
|
||||||
blob, err := store.blobs.OpenWithStorageFormat(ctx, ref, formatVersion)
|
blob, err := store.blobs.OpenWithStorageFormat(ctx, ref, formatVersion)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
@ -299,7 +299,7 @@ func (store *StoreForTest) ReaderWithStorageFormat(ctx context.Context, satellit
|
|||||||
// Reader returns a new piece reader.
|
// Reader returns a new piece reader.
|
||||||
func (store *Store) Reader(ctx context.Context, satellite storj.NodeID, pieceID storj.PieceID) (_ *Reader, err error) {
|
func (store *Store) Reader(ctx context.Context, satellite storj.NodeID, pieceID storj.PieceID) (_ *Reader, err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
blob, err := store.blobs.Open(ctx, storage.BlobRef{
|
blob, err := store.blobs.Open(ctx, blobstore.BlobRef{
|
||||||
Namespace: satellite.Bytes(),
|
Namespace: satellite.Bytes(),
|
||||||
Key: pieceID.Bytes(),
|
Key: pieceID.Bytes(),
|
||||||
})
|
})
|
||||||
@ -317,7 +317,7 @@ func (store *Store) Reader(ctx context.Context, satellite storj.NodeID, pieceID
|
|||||||
// Delete deletes the specified piece.
|
// Delete deletes the specified piece.
|
||||||
func (store *Store) Delete(ctx context.Context, satellite storj.NodeID, pieceID storj.PieceID) (err error) {
|
func (store *Store) Delete(ctx context.Context, satellite storj.NodeID, pieceID storj.PieceID) (err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
err = store.blobs.Delete(ctx, storage.BlobRef{
|
err = store.blobs.Delete(ctx, blobstore.BlobRef{
|
||||||
Namespace: satellite.Bytes(),
|
Namespace: satellite.Bytes(),
|
||||||
Key: pieceID.Bytes(),
|
Key: pieceID.Bytes(),
|
||||||
})
|
})
|
||||||
@ -366,7 +366,7 @@ func (store *Store) Trash(ctx context.Context, satellite storj.NodeID, pieceID s
|
|||||||
|
|
||||||
// Check if the MaxFormatVersionSupported piece exists. If not, we assume
|
// Check if the MaxFormatVersionSupported piece exists. If not, we assume
|
||||||
// this is an old piece version and attempt to migrate it.
|
// this is an old piece version and attempt to migrate it.
|
||||||
_, err = store.blobs.StatWithStorageFormat(ctx, storage.BlobRef{
|
_, err = store.blobs.StatWithStorageFormat(ctx, blobstore.BlobRef{
|
||||||
Namespace: satellite.Bytes(),
|
Namespace: satellite.Bytes(),
|
||||||
Key: pieceID.Bytes(),
|
Key: pieceID.Bytes(),
|
||||||
}, filestore.MaxFormatVersionSupported)
|
}, filestore.MaxFormatVersionSupported)
|
||||||
@ -386,7 +386,7 @@ func (store *Store) Trash(ctx context.Context, satellite storj.NodeID, pieceID s
|
|||||||
}
|
}
|
||||||
|
|
||||||
err = store.expirationInfo.Trash(ctx, satellite, pieceID)
|
err = store.expirationInfo.Trash(ctx, satellite, pieceID)
|
||||||
err = errs.Combine(err, store.blobs.Trash(ctx, storage.BlobRef{
|
err = errs.Combine(err, store.blobs.Trash(ctx, blobstore.BlobRef{
|
||||||
Namespace: satellite.Bytes(),
|
Namespace: satellite.Bytes(),
|
||||||
Key: pieceID.Bytes(),
|
Key: pieceID.Bytes(),
|
||||||
}))
|
}))
|
||||||
@ -471,7 +471,7 @@ func (store *Store) MigrateV0ToV1(ctx context.Context, satelliteID storj.NodeID,
|
|||||||
return Error.Wrap(err)
|
return Error.Wrap(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = store.blobs.DeleteWithStorageFormat(ctx, storage.BlobRef{
|
err = store.blobs.DeleteWithStorageFormat(ctx, blobstore.BlobRef{
|
||||||
Namespace: satelliteID.Bytes(),
|
Namespace: satelliteID.Bytes(),
|
||||||
Key: pieceID.Bytes(),
|
Key: pieceID.Bytes(),
|
||||||
}, filestore.FormatV0)
|
}, filestore.FormatV0)
|
||||||
@ -751,20 +751,20 @@ func (store *Store) CheckWritabilityWithTimeout(ctx context.Context, timeout tim
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Stat looks up disk metadata on the blob file.
|
// Stat looks up disk metadata on the blob file.
|
||||||
func (store *Store) Stat(ctx context.Context, satellite storj.NodeID, pieceID storj.PieceID) (storage.BlobInfo, error) {
|
func (store *Store) Stat(ctx context.Context, satellite storj.NodeID, pieceID storj.PieceID) (blobstore.BlobInfo, error) {
|
||||||
return store.blobs.Stat(ctx, storage.BlobRef{
|
return store.blobs.Stat(ctx, blobstore.BlobRef{
|
||||||
Namespace: satellite.Bytes(),
|
Namespace: satellite.Bytes(),
|
||||||
Key: pieceID.Bytes(),
|
Key: pieceID.Bytes(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
type storedPieceAccess struct {
|
type storedPieceAccess struct {
|
||||||
storage.BlobInfo
|
blobstore.BlobInfo
|
||||||
pieceID storj.PieceID
|
pieceID storj.PieceID
|
||||||
blobs storage.Blobs
|
blobs blobstore.Blobs
|
||||||
}
|
}
|
||||||
|
|
||||||
func newStoredPieceAccess(blobs storage.Blobs, blobInfo storage.BlobInfo) (storedPieceAccess, error) {
|
func newStoredPieceAccess(blobs blobstore.Blobs, blobInfo blobstore.BlobInfo) (storedPieceAccess, error) {
|
||||||
ref := blobInfo.BlobRef()
|
ref := blobInfo.BlobRef()
|
||||||
pieceID, err := storj.PieceIDFromBytes(ref.Key)
|
pieceID, err := storj.PieceIDFromBytes(ref.Key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -26,9 +26,9 @@ import (
|
|||||||
"storj.io/common/storj"
|
"storj.io/common/storj"
|
||||||
"storj.io/common/testcontext"
|
"storj.io/common/testcontext"
|
||||||
"storj.io/common/testrand"
|
"storj.io/common/testrand"
|
||||||
"storj.io/storj/storage"
|
|
||||||
"storj.io/storj/storage/filestore"
|
|
||||||
"storj.io/storj/storagenode"
|
"storj.io/storj/storagenode"
|
||||||
|
"storj.io/storj/storagenode/blobstore"
|
||||||
|
"storj.io/storj/storagenode/blobstore/filestore"
|
||||||
"storj.io/storj/storagenode/pieces"
|
"storj.io/storj/storagenode/pieces"
|
||||||
"storj.io/storj/storagenode/storagenodedb/storagenodedbtest"
|
"storj.io/storj/storagenode/storagenodedb/storagenodedbtest"
|
||||||
"storj.io/storj/storagenode/trust"
|
"storj.io/storj/storagenode/trust"
|
||||||
@ -144,7 +144,7 @@ func TestPieces(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeAPiece(ctx context.Context, t testing.TB, store *pieces.Store, satelliteID storj.NodeID, pieceID storj.PieceID, data []byte, atTime time.Time, expireTime *time.Time, formatVersion storage.FormatVersion) {
|
func writeAPiece(ctx context.Context, t testing.TB, store *pieces.Store, satelliteID storj.NodeID, pieceID storj.PieceID, data []byte, atTime time.Time, expireTime *time.Time, formatVersion blobstore.FormatVersion) {
|
||||||
tStore := &pieces.StoreForTest{store}
|
tStore := &pieces.StoreForTest{store}
|
||||||
writer, err := tStore.WriterForFormatVersion(ctx, satelliteID, pieceID, formatVersion, pb.PieceHashAlgorithm_SHA256)
|
writer, err := tStore.WriterForFormatVersion(ctx, satelliteID, pieceID, formatVersion, pb.PieceHashAlgorithm_SHA256)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@ -165,18 +165,18 @@ func writeAPiece(ctx context.Context, t testing.TB, store *pieces.Store, satelli
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func verifyPieceHandle(t testing.TB, reader *pieces.Reader, expectDataLen int, expectCreateTime time.Time, expectFormat storage.FormatVersion) {
|
func verifyPieceHandle(t testing.TB, reader *pieces.Reader, expectDataLen int, expectCreateTime time.Time, expectFormat blobstore.FormatVersion) {
|
||||||
assert.Equal(t, expectFormat, reader.StorageFormatVersion())
|
assert.Equal(t, expectFormat, reader.StorageFormatVersion())
|
||||||
assert.Equal(t, int64(expectDataLen), reader.Size())
|
assert.Equal(t, int64(expectDataLen), reader.Size())
|
||||||
if expectFormat != filestore.FormatV0 {
|
if expectFormat != filestore.FormatV0 {
|
||||||
pieceHeader, err := reader.GetPieceHeader()
|
pieceHeader, err := reader.GetPieceHeader()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, expectFormat, storage.FormatVersion(pieceHeader.FormatVersion))
|
assert.Equal(t, expectFormat, blobstore.FormatVersion(pieceHeader.FormatVersion))
|
||||||
assert.True(t, expectCreateTime.Equal(pieceHeader.CreationTime))
|
assert.True(t, expectCreateTime.Equal(pieceHeader.CreationTime))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func tryOpeningAPiece(ctx context.Context, t testing.TB, store *pieces.StoreForTest, satelliteID storj.NodeID, pieceID storj.PieceID, expectDataLen int, expectTime time.Time, expectFormat storage.FormatVersion) {
|
func tryOpeningAPiece(ctx context.Context, t testing.TB, store *pieces.StoreForTest, satelliteID storj.NodeID, pieceID storj.PieceID, expectDataLen int, expectTime time.Time, expectFormat blobstore.FormatVersion) {
|
||||||
reader, err := store.Reader(ctx, satelliteID, pieceID)
|
reader, err := store.Reader(ctx, satelliteID, pieceID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
verifyPieceHandle(t, reader, expectDataLen, expectTime, expectFormat)
|
verifyPieceHandle(t, reader, expectDataLen, expectTime, expectFormat)
|
||||||
@ -191,7 +191,7 @@ func tryOpeningAPiece(ctx context.Context, t testing.TB, store *pieces.StoreForT
|
|||||||
func TestTrashAndRestore(t *testing.T) {
|
func TestTrashAndRestore(t *testing.T) {
|
||||||
type testfile struct {
|
type testfile struct {
|
||||||
data []byte
|
data []byte
|
||||||
formatVer storage.FormatVersion
|
formatVer blobstore.FormatVersion
|
||||||
}
|
}
|
||||||
type testpiece struct {
|
type testpiece struct {
|
||||||
pieceID storj.PieceID
|
pieceID storj.PieceID
|
||||||
@ -476,7 +476,7 @@ func TestTrashAndRestore(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func verifyPieceData(ctx context.Context, t testing.TB, store *pieces.StoreForTest, satelliteID storj.NodeID, pieceID storj.PieceID, formatVer storage.FormatVersion, expected []byte, expiration time.Time, publicKey storj.PiecePublicKey) {
|
func verifyPieceData(ctx context.Context, t testing.TB, store *pieces.StoreForTest, satelliteID storj.NodeID, pieceID storj.PieceID, formatVer blobstore.FormatVersion, expected []byte, expiration time.Time, publicKey storj.PiecePublicKey) {
|
||||||
r, err := store.ReaderWithStorageFormat(ctx, satelliteID, pieceID, formatVer)
|
r, err := store.ReaderWithStorageFormat(ctx, satelliteID, pieceID, formatVer)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
@ -26,10 +26,10 @@ import (
|
|||||||
"storj.io/common/storj"
|
"storj.io/common/storj"
|
||||||
"storj.io/common/testcontext"
|
"storj.io/common/testcontext"
|
||||||
"storj.io/common/testrand"
|
"storj.io/common/testrand"
|
||||||
"storj.io/storj/private/testblobs"
|
|
||||||
"storj.io/storj/private/testplanet"
|
"storj.io/storj/private/testplanet"
|
||||||
"storj.io/storj/storagenode"
|
"storj.io/storj/storagenode"
|
||||||
"storj.io/storj/storagenode/bandwidth"
|
"storj.io/storj/storagenode/bandwidth"
|
||||||
|
"storj.io/storj/storagenode/blobstore/testblobs"
|
||||||
"storj.io/uplink/private/piecestore"
|
"storj.io/uplink/private/piecestore"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -21,9 +21,9 @@ import (
|
|||||||
"storj.io/common/storj"
|
"storj.io/common/storj"
|
||||||
"storj.io/common/testcontext"
|
"storj.io/common/testcontext"
|
||||||
"storj.io/common/testrand"
|
"storj.io/common/testrand"
|
||||||
"storj.io/storj/storage"
|
|
||||||
"storj.io/storj/storage/filestore"
|
|
||||||
"storj.io/storj/storagenode"
|
"storj.io/storj/storagenode"
|
||||||
|
"storj.io/storj/storagenode/blobstore"
|
||||||
|
"storj.io/storj/storagenode/blobstore/filestore"
|
||||||
"storj.io/storj/storagenode/pieces"
|
"storj.io/storj/storagenode/pieces"
|
||||||
"storj.io/storj/storagenode/retain"
|
"storj.io/storj/storagenode/retain"
|
||||||
"storj.io/storj/storagenode/storagenodedb/storagenodedbtest"
|
"storj.io/storj/storagenode/storagenodedb/storagenodedbtest"
|
||||||
@ -59,7 +59,7 @@ func TestRetainPieces(t *testing.T) {
|
|||||||
// keep pieceIDs[numPiecesToKeep+numOldPieces : numPieces] (recent + not in filter)
|
// keep pieceIDs[numPiecesToKeep+numOldPieces : numPieces] (recent + not in filter)
|
||||||
// add all pieces to the node pieces info DB - but only count piece ids in filter
|
// add all pieces to the node pieces info DB - but only count piece ids in filter
|
||||||
for index, id := range pieceIDs {
|
for index, id := range pieceIDs {
|
||||||
var formatVer storage.FormatVersion
|
var formatVer blobstore.FormatVersion
|
||||||
if index%2 == 0 {
|
if index%2 == 0 {
|
||||||
formatVer = filestore.FormatV0
|
formatVer = filestore.FormatV0
|
||||||
} else {
|
} else {
|
||||||
|
@ -23,10 +23,10 @@ import (
|
|||||||
"storj.io/private/dbutil/sqliteutil"
|
"storj.io/private/dbutil/sqliteutil"
|
||||||
"storj.io/private/tagsql"
|
"storj.io/private/tagsql"
|
||||||
"storj.io/storj/private/migrate"
|
"storj.io/storj/private/migrate"
|
||||||
"storj.io/storj/storage"
|
|
||||||
"storj.io/storj/storage/filestore"
|
|
||||||
"storj.io/storj/storagenode/apikeys"
|
"storj.io/storj/storagenode/apikeys"
|
||||||
"storj.io/storj/storagenode/bandwidth"
|
"storj.io/storj/storagenode/bandwidth"
|
||||||
|
"storj.io/storj/storagenode/blobstore"
|
||||||
|
"storj.io/storj/storagenode/blobstore/filestore"
|
||||||
"storj.io/storj/storagenode/notifications"
|
"storj.io/storj/storagenode/notifications"
|
||||||
"storj.io/storj/storagenode/orders"
|
"storj.io/storj/storagenode/orders"
|
||||||
"storj.io/storj/storagenode/payouts"
|
"storj.io/storj/storagenode/payouts"
|
||||||
@ -92,7 +92,7 @@ type DB struct {
|
|||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
config Config
|
config Config
|
||||||
|
|
||||||
pieces storage.Blobs
|
pieces blobstore.Blobs
|
||||||
|
|
||||||
dbDirectory string
|
dbDirectory string
|
||||||
|
|
||||||
@ -512,7 +512,7 @@ func (db *DB) Orders() orders.DB {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Pieces returns blob storage for pieces.
|
// Pieces returns blob storage for pieces.
|
||||||
func (db *DB) Pieces() storage.Blobs {
|
func (db *DB) Pieces() blobstore.Blobs {
|
||||||
return db.pieces
|
return db.pieces
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -16,7 +16,7 @@ import (
|
|||||||
"storj.io/common/testcontext"
|
"storj.io/common/testcontext"
|
||||||
"storj.io/private/dbutil/dbschema"
|
"storj.io/private/dbutil/dbschema"
|
||||||
"storj.io/private/dbutil/sqliteutil"
|
"storj.io/private/dbutil/sqliteutil"
|
||||||
"storj.io/storj/storage/filestore"
|
"storj.io/storj/storagenode/blobstore/filestore"
|
||||||
"storj.io/storj/storagenode/storagenodedb"
|
"storj.io/storj/storagenode/storagenodedb"
|
||||||
"storj.io/storj/storagenode/storagenodedb/testdata"
|
"storj.io/storj/storagenode/storagenodedb/testdata"
|
||||||
)
|
)
|
||||||
|
@ -12,8 +12,8 @@ import (
|
|||||||
|
|
||||||
"storj.io/common/pb"
|
"storj.io/common/pb"
|
||||||
"storj.io/common/storj"
|
"storj.io/common/storj"
|
||||||
"storj.io/storj/storage"
|
"storj.io/storj/storagenode/blobstore"
|
||||||
"storj.io/storj/storage/filestore"
|
"storj.io/storj/storagenode/blobstore/filestore"
|
||||||
"storj.io/storj/storagenode/pieces"
|
"storj.io/storj/storagenode/pieces"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -57,7 +57,7 @@ func (db *v0PieceInfoDB) Add(ctx context.Context, info *pieces.Info) (err error)
|
|||||||
return ErrPieceInfo.Wrap(err)
|
return ErrPieceInfo.Wrap(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *v0PieceInfoDB) getAllPiecesOwnedBy(ctx context.Context, blobStore storage.Blobs, satelliteID storj.NodeID) ([]v0StoredPieceAccess, error) {
|
func (db *v0PieceInfoDB) getAllPiecesOwnedBy(ctx context.Context, blobStore blobstore.Blobs, satelliteID storj.NodeID) ([]v0StoredPieceAccess, error) {
|
||||||
rows, err := db.QueryContext(ctx, `
|
rows, err := db.QueryContext(ctx, `
|
||||||
SELECT piece_id, piece_size, piece_creation, piece_expiration
|
SELECT piece_id, piece_size, piece_creation, piece_expiration
|
||||||
FROM pieceinfo_
|
FROM pieceinfo_
|
||||||
@ -90,7 +90,7 @@ func (db *v0PieceInfoDB) getAllPiecesOwnedBy(ctx context.Context, blobStore stor
|
|||||||
//
|
//
|
||||||
// If blobStore is nil, the .Stat() and .FullPath() methods of the provided StoredPieceAccess
|
// If blobStore is nil, the .Stat() and .FullPath() methods of the provided StoredPieceAccess
|
||||||
// instances will not work, but otherwise everything should be ok.
|
// instances will not work, but otherwise everything should be ok.
|
||||||
func (db *v0PieceInfoDB) WalkSatelliteV0Pieces(ctx context.Context, blobStore storage.Blobs, satelliteID storj.NodeID, walkFunc func(pieces.StoredPieceAccess) error) (err error) {
|
func (db *v0PieceInfoDB) WalkSatelliteV0Pieces(ctx context.Context, blobStore blobstore.Blobs, satelliteID storj.NodeID, walkFunc func(pieces.StoredPieceAccess) error) (err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
|
|
||||||
// TODO: is it worth paging this query? we hope that SNs will not yet have too many V0 pieces.
|
// TODO: is it worth paging this query? we hope that SNs will not yet have too many V0 pieces.
|
||||||
@ -206,13 +206,13 @@ func (db *v0PieceInfoDB) GetExpired(ctx context.Context, expiredAt time.Time, li
|
|||||||
}
|
}
|
||||||
|
|
||||||
type v0StoredPieceAccess struct {
|
type v0StoredPieceAccess struct {
|
||||||
blobStore storage.Blobs
|
blobStore blobstore.Blobs
|
||||||
satellite storj.NodeID
|
satellite storj.NodeID
|
||||||
pieceID storj.PieceID
|
pieceID storj.PieceID
|
||||||
pieceSize int64
|
pieceSize int64
|
||||||
creationTime time.Time
|
creationTime time.Time
|
||||||
expirationTime *time.Time
|
expirationTime *time.Time
|
||||||
blobInfo storage.BlobInfo
|
blobInfo blobstore.BlobInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
// PieceID returns the piece ID for the piece.
|
// PieceID returns the piece ID for the piece.
|
||||||
@ -225,9 +225,9 @@ func (v0Access *v0StoredPieceAccess) Satellite() (storj.NodeID, error) {
|
|||||||
return v0Access.satellite, nil
|
return v0Access.satellite, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// BlobRef returns the relevant storage.BlobRef locator for the piece.
|
// BlobRef returns the relevant blobstore.BlobRef locator for the piece.
|
||||||
func (v0Access *v0StoredPieceAccess) BlobRef() storage.BlobRef {
|
func (v0Access *v0StoredPieceAccess) BlobRef() blobstore.BlobRef {
|
||||||
return storage.BlobRef{
|
return blobstore.BlobRef{
|
||||||
Namespace: v0Access.satellite.Bytes(),
|
Namespace: v0Access.satellite.Bytes(),
|
||||||
Key: v0Access.pieceID.Bytes(),
|
Key: v0Access.pieceID.Bytes(),
|
||||||
}
|
}
|
||||||
@ -274,7 +274,7 @@ func (v0Access *v0StoredPieceAccess) FullPath(ctx context.Context) (string, erro
|
|||||||
}
|
}
|
||||||
|
|
||||||
// StorageFormatVersion indicates the storage format version used to store the piece.
|
// StorageFormatVersion indicates the storage format version used to store the piece.
|
||||||
func (v0Access *v0StoredPieceAccess) StorageFormatVersion() storage.FormatVersion {
|
func (v0Access *v0StoredPieceAccess) StorageFormatVersion() blobstore.FormatVersion {
|
||||||
return filestore.FormatV0
|
return filestore.FormatV0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -22,8 +22,8 @@ import (
|
|||||||
"storj.io/common/testcontext"
|
"storj.io/common/testcontext"
|
||||||
"storj.io/common/testrand"
|
"storj.io/common/testrand"
|
||||||
"storj.io/common/uuid"
|
"storj.io/common/uuid"
|
||||||
"storj.io/storj/storage/filestore"
|
|
||||||
"storj.io/storj/storagenode"
|
"storj.io/storj/storagenode"
|
||||||
|
"storj.io/storj/storagenode/blobstore/filestore"
|
||||||
"storj.io/storj/storagenode/orders/ordersfile"
|
"storj.io/storj/storagenode/orders/ordersfile"
|
||||||
"storj.io/storj/storagenode/storagenodedb"
|
"storj.io/storj/storagenode/storagenodedb"
|
||||||
"storj.io/storj/storagenode/storagenodedb/storagenodedbtest"
|
"storj.io/storj/storagenode/storagenodedb/storagenodedbtest"
|
||||||
|
@ -16,7 +16,7 @@ import (
|
|||||||
|
|
||||||
"storj.io/common/testcontext"
|
"storj.io/common/testcontext"
|
||||||
"storj.io/private/tagsql"
|
"storj.io/private/tagsql"
|
||||||
"storj.io/storj/storage/filestore"
|
"storj.io/storj/storagenode/blobstore/filestore"
|
||||||
"storj.io/storj/storagenode/storagenodedb"
|
"storj.io/storj/storagenode/storagenodedb"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user