JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/metainfo: refactor SegmentLocation.Index to SegmentPosition

Browse Source 
Change-Id: Ic9403c8126712693326dd83d6ba4f3b84be3e0c7
This commit is contained in:
Kaloyan Raev 2020-12-14 12:11:28 +02:00
parent 7d8f19e94d
commit fc85179a19
9 changed files with 26 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
satellite/accounting/projectusage_test.go
View File

@ -579,7 +579,7 @@ func TestProjectUsage_FreeUsedStorageSpace(t *testing.T) {
require.NoError(t, err)
// check if usage is equal to first uploaded file
prefix, err := metainfo.CreatePath(ctx, project.ID, -1, []byte("testbucket"), []byte{})
prefix, err := metainfo.CreatePath(ctx, project.ID, metabase.LastSegmentIndex, []byte("testbucket"), []byte{})
require.NoError(t, err)
items, _, err := satMetainfo.Service.List(ctx, prefix.Encode(), "", true, 1, meta.All)
require.NoError(t, err)

6
satellite/accounting/tally/tally_test.go
View File

@ -162,7 +162,7 @@ func TestCalculateBucketAtRestData(t *testing.T) {
var testCases = []struct {
name string
project string
segmentIndex int64
segmentIndex uint32
bucketName string
objectName string
inline bool
@ -195,7 +195,7 @@ func TestCalculateBucketAtRestData(t *testing.T) {
location := metabase.SegmentLocation{
ProjectID: projectID,
BucketName: tt.bucketName,
Index: tt.segmentIndex,
Position: metabase.SegmentPosition{Index: tt.segmentIndex},
ObjectKey: metabase.ObjectKey(tt.objectName),
}
err = metainfo.Put(ctx, location.Encode(), pointer)
@ -240,7 +240,7 @@ func TestTallyIgnoresExpiredPointers(t *testing.T) {
location := metabase.SegmentLocation{
ProjectID: projectID,
BucketName: bucket,
Index: metabase.LastSegmentIndex,
Position: metabase.SegmentPosition{Index: metabase.LastSegmentIndex},
ObjectKey: metabase.ObjectKey("object/name"),
}
err = metainfo.Put(ctx, location.Encode(), pointer)

2
satellite/gc/gc_test.go
View File

@ -139,7 +139,7 @@ func getPointer(ctx *testcontext.Context, t *testing.T, satellite *testplanet.Sa
segmentLocation := metabase.SegmentLocation{
ProjectID: upl.Projects[0].ID,
BucketName: bucket,
Index: metabase.LastSegmentIndex,
Position: metabase.SegmentPosition{Index: metabase.LastSegmentIndex},
ObjectKey: metabase.ObjectKey(encryptedPath.Raw()),
}

9
satellite/inspector/inspector.go
View File

@ -16,6 +16,7 @@ import (
"storj.io/common/uuid"
"storj.io/storj/satellite/internalpb"
"storj.io/storj/satellite/metainfo"
"storj.io/storj/satellite/metainfo/metabase"
"storj.io/storj/satellite/overlay"
)
@ -121,7 +122,7 @@ func (endpoint *Endpoint) SegmentHealth(ctx context.Context, in *internalpb.Segm
return nil, Error.Wrap(err)
}
location, err := metainfo.CreatePath(ctx, projectID, in.GetSegmentIndex(), in.GetBucket(), in.GetEncryptedPath())
location, err := metainfo.CreatePath(ctx, projectID, uint32(in.GetSegmentIndex()), in.GetBucket(), in.GetEncryptedPath())
if err != nil {
return nil, Error.Wrap(err)
}
@ -175,10 +176,10 @@ func (endpoint *Endpoint) SegmentHealth(ctx context.Context, in *internalpb.Segm
health.UnhealthyIds = unhealthyNodes
health.OfflineIds = offlineNodes
if in.GetSegmentIndex() > -1 {
health.Segment = []byte("s" + strconv.FormatInt(in.GetSegmentIndex(), 10))
} else {
if uint32(in.GetSegmentIndex()) == metabase.LastSegmentIndex {
health.Segment = []byte("l")
} else {
health.Segment = []byte("s" + strconv.FormatInt(in.GetSegmentIndex(), 10))
}
return &internalpb.SegmentHealthResponse{

2
satellite/metainfo/loop.go
View File

@ -423,7 +423,7 @@ func iterateSegments(ctx context.Context, streamID uuid.UUID, projectID uuid.UUI
ProjectID: projectID,
BucketName: bucket,
ObjectKey: objectKey,
Index: int64(segment.Position.Index),
Position: segment.Position,
}
keepObserver := handleSegment(ctx, observer, location, segment)
if !keepObserver {

60
satellite/metainfo/metabase/common.go
View File

@ -4,6 +4,7 @@
package metabase
import (
"math"
"sort"
"strconv"
"strings"
@ -19,10 +20,9 @@ var Error = errs.Class("metabase")
// Common constants for segment keys.
const (
Delimiter = '/'
LastSegmentName = "l"
LastSegmentIndex = -1
FirstSegmentIndex = 0
Delimiter = '/'
LastSegmentName = "l"
LastSegmentIndex = uint32(math.MaxUint32)
)
// MaxListLimit is the maximum number of items the client can request for listing.
@ -81,39 +81,6 @@ func (obj ObjectLocation) Bucket() BucketLocation {
}
}
// LastSegment returns the last segment location.
func (obj ObjectLocation) LastSegment() SegmentLocation {
return SegmentLocation{
ProjectID: obj.ProjectID,
BucketName: obj.BucketName,
Index: LastSegmentIndex,
ObjectKey: obj.ObjectKey,
}
}
// FirstSegment returns the first segment location.
func (obj ObjectLocation) FirstSegment() SegmentLocation {
return SegmentLocation{
ProjectID: obj.ProjectID,
BucketName: obj.BucketName,
Index: FirstSegmentIndex,
ObjectKey: obj.ObjectKey,
}
}
// Segment returns segment location for a given index.
func (obj ObjectLocation) Segment(index int64) (SegmentLocation, error) {
if index < LastSegmentIndex {
return SegmentLocation{}, Error.New("invalid index %v", index)
}
return SegmentLocation{
ProjectID: obj.ProjectID,
BucketName: obj.BucketName,
Index: index,
ObjectKey: obj.ObjectKey,
}, nil
}
// Verify object location fields.
func (obj ObjectLocation) Verify() error {
switch {
@ -134,8 +101,8 @@ type SegmentKey []byte
type SegmentLocation struct {
ProjectID uuid.UUID
BucketName string
Index int64 // TODO refactor to SegmentPosition
ObjectKey ObjectKey
Position SegmentPosition
}
// Bucket returns bucket location this segment belongs to.
@ -155,12 +122,6 @@ func (seg SegmentLocation) Object() ObjectLocation {
}
}
// IsLast returns whether this corresponds to last segment.
func (seg SegmentLocation) IsLast() bool { return seg.Index == LastSegmentIndex }
// IsFirst returns whether this corresponds to first segment.
func (seg SegmentLocation) IsFirst() bool { return seg.Index == FirstSegmentIndex }
// ParseSegmentKey parses an segment key into segment location.
func ParseSegmentKey(encoded SegmentKey) (SegmentLocation, error) {
elements := strings.SplitN(string(encoded), "/", 4)
@ -173,22 +134,23 @@ func ParseSegmentKey(encoded SegmentKey) (SegmentLocation, error) {
return SegmentLocation{}, Error.New("invalid key %q", encoded)
}
var index int64
var index uint32
if elements[1] == LastSegmentName {
index = LastSegmentIndex
} else {
numstr := strings.TrimPrefix(elements[1], "s")
// remove prefix `s` from segment index we got
index, err = strconv.ParseInt(numstr, 10, 64)
parsed, err := strconv.ParseUint(numstr, 10, 64)
if err != nil {
return SegmentLocation{}, Error.New("invalid %q, segment number %q", string(encoded), elements[1])
}
index = uint32(parsed)
}
return SegmentLocation{
ProjectID: projectID,
BucketName: elements[2],
Index: index,
Position: SegmentPosition{Index: index},
ObjectKey: ObjectKey(elements[3]),
}, nil
}
@ -196,8 +158,8 @@ func ParseSegmentKey(encoded SegmentKey) (SegmentLocation, error) {
// Encode converts segment location into a segment key.
func (seg SegmentLocation) Encode() SegmentKey {
segment := LastSegmentName
if seg.Index > LastSegmentIndex {
segment = "s" + strconv.FormatInt(seg.Index, 10)
if seg.Position.Index != LastSegmentIndex {
segment = "s" + strconv.FormatUint(seg.Position.Encode(), 10)
}
return SegmentKey(storj.JoinPaths(
seg.ProjectID.String(),

8
satellite/metainfo/metainfo.go
View File

@ -6,7 +6,6 @@ package metainfo
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"time"
@ -2065,16 +2064,13 @@ func (endpoint *Endpoint) RevokeAPIKey(ctx context.Context, req *pb.RevokeAPIKey
}
// CreatePath creates a segment key.
func CreatePath(ctx context.Context, projectID uuid.UUID, segmentIndex int64, bucket, path []byte) (_ metabase.SegmentLocation, err error) {
func CreatePath(ctx context.Context, projectID uuid.UUID, segmentIndex uint32, bucket, path []byte) (_ metabase.SegmentLocation, err error) {
// TODO rename to CreateLocation
defer mon.Task()(&ctx)(&err)
if segmentIndex < metabase.LastSegmentIndex {
return metabase.SegmentLocation{}, errors.New("invalid segment index")
}
return metabase.SegmentLocation{
ProjectID: projectID,
BucketName: string(bucket),
Index: segmentIndex,
Position: metabase.SegmentPosition{Index: segmentIndex},
ObjectKey: metabase.ObjectKey(path),
}, nil
}

2
satellite/metainfo/metainfo_test.go
View File

@ -920,7 +920,7 @@ func TestRemoteSegment(t *testing.T) {
_, limits, err := metainfoClient.DownloadSegment(ctx, metainfo.DownloadSegmentParams{
StreamID: object.StreamID,
Position: storj.SegmentPosition{
Index: metabase.LastSegmentIndex,
Index: -1,
},
})
require.NoError(t, err)

2
satellite/repair/checker/checker_test.go
View File

@ -130,7 +130,7 @@ func TestIdentifyIrreparableSegments(t *testing.T) {
pointerLocation := metabase.SegmentLocation{
ProjectID: projectID,
BucketName: "bucket",
Index: metabase.LastSegmentIndex,
Position: metabase.SegmentPosition{Index: metabase.LastSegmentIndex},
ObjectKey: "piece",
}