JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/metabase: expose ConvertNodesToAliases and ConvertAliasesToNodes

Browse Source 
They are needed for segment-verify tool.

Also rename some of the conversion methods to make clear,
which of them have side-effects.

Change-Id: Ie9a0952548e9ed5068c7a30c2fd2134b07139bca
This commit is contained in:
Egon Elbre 2022-09-15 16:50:13 +03:00
parent cd81c5bd58
commit 9b520b2114
7 changed files with 65 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/tools/segment-verify/service.go
View File

@ -30,7 +30,9 @@ const ConcurrentRequests = 10000
// Metabase defines implementation dependencies we need from metabase.
type Metabase interface {
ConvertNodesToAliases(ctx context.Context, nodeID []storj.NodeID) ([]metabase.NodeAlias, error)
ConvertAliasesToNodes(ctx context.Context, aliases []metabase.NodeAlias) ([]storj.NodeID, error)
GetSegmentByPosition(ctx context.Context, opts metabase.GetSegmentByPosition) (segment metabase.Segment, err error)
ListVerifySegments(ctx context.Context, opts metabase.ListVerifySegments) (result metabase.ListVerifySegmentsResult, err error)
}

14
satellite/metabase/alias.go
View File

@ -73,3 +73,17 @@ func (db *DB) ListNodeAliases(ctx context.Context) (_ []NodeAliasEntry, err erro
return aliases, nil
}
// ConvertNodesToAliases converts nodeIDs to node aliases.
// Returns an error when an alias is missing.
func (db *DB) ConvertNodesToAliases(ctx context.Context, nodeIDs []storj.NodeID) (_ []NodeAlias, err error) {
defer mon.Task()(&ctx)(&err)
return db.aliasCache.Aliases(ctx, nodeIDs)
}
// ConvertAliasesToNodes converts aliases to node ID-s.
// Returns an error when a node alias is missing.
func (db *DB) ConvertAliasesToNodes(ctx context.Context, aliases []NodeAlias) (_ []storj.NodeID, err error) {
defer mon.Task()(&ctx)(&err)
return db.aliasCache.Nodes(ctx, aliases)
}

40
satellite/metabase/aliascache.go
View File

@ -48,13 +48,11 @@ func (cache *NodeAliasCache) Nodes(ctx context.Context, aliases []NodeAlias) ([]
return nodes, nil
}
if len(missing) > 0 {
var err error
latest, err = cache.refresh(ctx, nil, missing)
if err != nil {
return nil, Error.New("failed to refresh node alias db: %w", err)
}
}
nodes, missing = latest.Nodes(aliases)
if len(missing) == 0 {
@ -64,9 +62,9 @@ func (cache *NodeAliasCache) Nodes(ctx context.Context, aliases []NodeAlias) ([]
return nil, Error.New("aliases missing in database: %v", missing)
}
// Aliases returns node aliases corresponding to the node ID-s,
// EnsureAliases returns node aliases corresponding to the node ID-s,
// adding missing node ID-s to the database when needed.
func (cache *NodeAliasCache) Aliases(ctx context.Context, nodes []storj.NodeID) ([]NodeAlias, error) {
func (cache *NodeAliasCache) EnsureAliases(ctx context.Context, nodes []storj.NodeID) ([]NodeAlias, error) {
latest := cache.getLatest()
aliases, missing := latest.Aliases(nodes)
@ -74,13 +72,11 @@ func (cache *NodeAliasCache) Aliases(ctx context.Context, nodes []storj.NodeID)
return aliases, nil
}
if len(missing) > 0 {
var err error
latest, err = cache.ensure(ctx, missing...)
if err != nil {
return nil, Error.Wrap(err)
}
}
aliases, missing = latest.Aliases(nodes)
if len(missing) == 0 {
@ -90,6 +86,29 @@ func (cache *NodeAliasCache) Aliases(ctx context.Context, nodes []storj.NodeID)
return nil, Error.New("nodes still missing after ensuring: %v", missing)
}
// Aliases returns node aliases corresponding to the node ID-s and returns an error when node is missing.
func (cache *NodeAliasCache) Aliases(ctx context.Context, nodes []storj.NodeID) ([]NodeAlias, error) {
latest := cache.getLatest()
aliases, missing := latest.Aliases(nodes)
if len(missing) == 0 {
return aliases, nil
}
var err error
latest, err = cache.refresh(ctx, missing, nil)
if err != nil {
return nil, Error.New("failed to refresh node alias db: %w", err)
}
aliases, missing = latest.Aliases(nodes)
if len(missing) > 0 {
return aliases, Error.New("aliases missing for %v", missing)
}
return aliases, nil
}
// ensure tries to ensure that the specified missing node ID-s are assigned a alias.
func (cache *NodeAliasCache) ensure(ctx context.Context, missing ...storj.NodeID) (_ *NodeAliasMap, err error) {
defer mon.Task()(&ctx)(&err)
@ -134,8 +153,9 @@ func (cache *NodeAliasCache) refresh(ctx context.Context, missingNodes []storj.N
return xs, nil
}
// ConvertPiecesToAliases converts pieces to alias pieces.
func (cache *NodeAliasCache) ConvertPiecesToAliases(ctx context.Context, pieces Pieces) (_ AliasPieces, err error) {
// EnsurePiecesToAliases converts pieces to alias pieces and automatically adds storage node
// to alias table when necessary.
func (cache *NodeAliasCache) EnsurePiecesToAliases(ctx context.Context, pieces Pieces) (_ AliasPieces, err error) {
defer mon.Task()(&ctx)(&err)
if len(pieces) == 0 {
@ -147,7 +167,7 @@ func (cache *NodeAliasCache) ConvertPiecesToAliases(ctx context.Context, pieces
nodes[i] = p.StorageNode
}
aliases, err := cache.Aliases(ctx, nodes)
aliases, err := cache.EnsureAliases(ctx, nodes)
if err != nil {
return nil, Error.Wrap(err)
}
@ -267,7 +287,7 @@ func (m *NodeAliasMap) Nodes(aliases []NodeAlias) (xs []storj.NodeID, missing []
return xs, missing
}
// Aliases returns alises-s for the given node ID-s and node ID-s that are not in this map.
// Aliases returns aliases-s for the given node ID-s and node ID-s that are not in this map.
func (m *NodeAliasMap) Aliases(nodes []storj.NodeID) (xs []NodeAlias, missing []storj.NodeID) {
xs = make([]NodeAlias, 0, len(nodes))
for _, n := range nodes {

14
satellite/metabase/aliascache_test.go
View File

@ -37,12 +37,12 @@ func TestNodeAliasCache(t *testing.T) {
n1, n2 := testrand.NodeID(), testrand.NodeID()
aliases, err := cache.Aliases(ctx, []storj.NodeID{n1, n2})
aliases, err := cache.EnsureAliases(ctx, []storj.NodeID{n1, n2})
require.NoError(t, err)
require.Equal(t, []metabase.NodeAlias{1, 2}, aliases)
nx1 := testrand.NodeID()
aliases, err = cache.Aliases(ctx, []storj.NodeID{nx1, n1, n2})
aliases, err = cache.EnsureAliases(ctx, []storj.NodeID{nx1, n1, n2})
require.NoError(t, err)
require.Equal(t, []metabase.NodeAlias{3, 1, 2}, aliases)
@ -62,7 +62,7 @@ func TestNodeAliasCache(t *testing.T) {
n1, n2 := testrand.NodeID(), testrand.NodeID()
aliases, err := cache.Aliases(ctx, []storj.NodeID{n1, n2})
aliases, err := cache.EnsureAliases(ctx, []storj.NodeID{n1, n2})
require.EqualError(t, err, "metabase: failed to update node alias db: io.EOF")
require.Empty(t, aliases)
@ -71,7 +71,7 @@ func TestNodeAliasCache(t *testing.T) {
require.Empty(t, nodes)
})
t.Run("Aliases refresh once", func(t *testing.T) {
t.Run("EnsureAliases refresh once", func(t *testing.T) {
for repeat := 0; repeat < 3; repeat++ {
database := &NodeAliasDB{}
cache := metabase.NewNodeAliasCache(database)
@ -88,7 +88,7 @@ func TestNodeAliasCache(t *testing.T) {
waiting.Done()
<-start
_, err := cache.Aliases(ctx, []storj.NodeID{n1, n2})
_, err := cache.EnsureAliases(ctx, []storj.NodeID{n1, n2})
return err
})
}
@ -156,7 +156,7 @@ func TestNodeAliasCache_DB(t *testing.T) {
n1, n2 := testrand.NodeID(), testrand.NodeID()
aliases, err := cache.Aliases(ctx, []storj.NodeID{n1, n2})
aliases, err := cache.EnsureAliases(ctx, []storj.NodeID{n1, n2})
require.NoError(t, err)
require.Equal(t, []metabase.NodeAlias{1, 2}, aliases)
@ -302,7 +302,7 @@ func BenchmarkNodeAliasCache_ConvertAliasesToPieces(b *testing.B) {
for i := range nodeIDs {
nodeIDs[i] = testrand.NodeID()
}
aliases, err := cache.Aliases(ctx, nodeIDs)
aliases, err := cache.EnsureAliases(ctx, nodeIDs)
if err != nil {
b.Fatal(err)
}

2
satellite/metabase/commit.go
View File

@ -324,7 +324,7 @@ func (db *DB) CommitSegment(ctx context.Context, opts CommitSegment) (err error)
return ErrInvalidRequest.New("number of pieces is less than redundancy optimal shares value")
}
aliasPieces, err := db.aliasCache.ConvertPiecesToAliases(ctx, opts.Pieces)
aliasPieces, err := db.aliasCache.EnsurePiecesToAliases(ctx, opts.Pieces)
if err != nil {
return Error.New("unable to convert pieces to aliases: %w", err)
}

2
satellite/metabase/delete.go
View File

@ -448,7 +448,7 @@ func (db *DB) promoteNewAncestors(ctx context.Context, tx tagsql.Tx, objects []d
for i, segment := range object.Segments {
positions[i] = int64(segment.Position.Encode())
aliases, err := db.aliasCache.ConvertPiecesToAliases(ctx, segment.Pieces)
aliases, err := db.aliasCache.EnsurePiecesToAliases(ctx, segment.Pieces)
if err != nil {
return err
}

4
satellite/metabase/update.go
View File

@ -64,12 +64,12 @@ func (db *DB) UpdateSegmentPieces(ctx context.Context, opts UpdateSegmentPieces)
updateRepairAt := !opts.NewRepairedAt.IsZero()
oldPieces, err := db.aliasCache.ConvertPiecesToAliases(ctx, opts.OldPieces)
oldPieces, err := db.aliasCache.EnsurePiecesToAliases(ctx, opts.OldPieces)
if err != nil {
return Error.New("unable to convert pieces to aliases: %w", err)
}
newPieces, err := db.aliasCache.ConvertPiecesToAliases(ctx, opts.NewPieces)
newPieces, err := db.aliasCache.EnsurePiecesToAliases(ctx, opts.NewPieces)
if err != nil {
return Error.New("unable to convert pieces to aliases: %w", err)
}