Lazy segment ranger (#343)
This commit is contained in:
parent
e90c2140b8
commit
8f4b29abd0
@ -17,6 +17,7 @@ import (
|
|||||||
ranger "storj.io/storj/pkg/ranger"
|
ranger "storj.io/storj/pkg/ranger"
|
||||||
"storj.io/storj/pkg/storage/meta"
|
"storj.io/storj/pkg/storage/meta"
|
||||||
"storj.io/storj/pkg/storage/segments"
|
"storj.io/storj/pkg/storage/segments"
|
||||||
|
"storj.io/storj/pkg/utils"
|
||||||
streamspb "storj.io/storj/protos/streams"
|
streamspb "storj.io/storj/protos/streams"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -147,30 +148,30 @@ func (s *streamStore) Get(ctx context.Context, path paths.Path) (
|
|||||||
msi := streamspb.MetaStreamInfo{}
|
msi := streamspb.MetaStreamInfo{}
|
||||||
err = proto.Unmarshal(lastSegmentMeta.Data, &msi)
|
err = proto.Unmarshal(lastSegmentMeta.Data, &msi)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = lastRangerCloser.Close()
|
utils.LogClose(lastRangerCloser)
|
||||||
return nil, Meta{}, err
|
return nil, Meta{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
newMeta, err := convertMeta(lastSegmentMeta)
|
newMeta, err := convertMeta(lastSegmentMeta)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = lastRangerCloser.Close()
|
utils.LogClose(lastRangerCloser)
|
||||||
return nil, Meta{}, err
|
return nil, Meta{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var rangers []ranger.RangeCloser
|
var rangers []ranger.RangeCloser
|
||||||
|
|
||||||
for i := 0; i < int(msi.NumberOfSegments); i++ {
|
for i := int64(0); i < msi.NumberOfSegments; i++ {
|
||||||
currentPath := fmt.Sprintf("s%d", i)
|
currentPath := fmt.Sprintf("s%d", i)
|
||||||
rangeCloser, _, err := s.segments.Get(ctx, path.Prepend(currentPath))
|
size := msi.SegmentsSize
|
||||||
if err != nil {
|
if i == msi.NumberOfSegments-1 {
|
||||||
for _, ranger := range rangers {
|
size = msi.LastSegmentSize
|
||||||
_ = ranger.Close()
|
|
||||||
}
|
|
||||||
_ = lastRangerCloser.Close()
|
|
||||||
return nil, Meta{}, err
|
|
||||||
}
|
}
|
||||||
|
rr := &lazySegmentRanger{
|
||||||
rangers = append(rangers, rangeCloser)
|
segments: s.segments,
|
||||||
|
path: path.Prepend(currentPath),
|
||||||
|
size: size,
|
||||||
|
}
|
||||||
|
rangers = append(rangers, rr)
|
||||||
}
|
}
|
||||||
|
|
||||||
rangers = append(rangers, lastRangerCloser)
|
rangers = append(rangers, lastRangerCloser)
|
||||||
@ -256,3 +257,35 @@ func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore pa
|
|||||||
|
|
||||||
return items, more, nil
|
return items, more, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type lazySegmentRanger struct {
|
||||||
|
ranger ranger.RangeCloser
|
||||||
|
segments segments.Store
|
||||||
|
path paths.Path
|
||||||
|
size int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Size implements Ranger.Size
|
||||||
|
func (lr *lazySegmentRanger) Size() int64 {
|
||||||
|
return lr.size
|
||||||
|
}
|
||||||
|
|
||||||
|
// Size implements Ranger.Close
|
||||||
|
func (lr *lazySegmentRanger) Close() error {
|
||||||
|
if lr.ranger == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return lr.ranger.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Range implements Ranger.Range to be lazily connected
|
||||||
|
func (lr *lazySegmentRanger) Range(ctx context.Context, offset, length int64) (io.ReadCloser, error) {
|
||||||
|
if lr.ranger == nil {
|
||||||
|
rr, _, err := lr.segments.Get(ctx, lr.path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
lr.ranger = rr
|
||||||
|
}
|
||||||
|
return lr.ranger.Range(ctx, offset, length)
|
||||||
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user