Lazy segment ranger (#343)

This commit is contained in:
Kaloyan Raev 2018-09-12 09:06:44 -04:00 committed by GitHub
parent e90c2140b8
commit 8f4b29abd0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -17,6 +17,7 @@ import (
ranger "storj.io/storj/pkg/ranger" ranger "storj.io/storj/pkg/ranger"
"storj.io/storj/pkg/storage/meta" "storj.io/storj/pkg/storage/meta"
"storj.io/storj/pkg/storage/segments" "storj.io/storj/pkg/storage/segments"
"storj.io/storj/pkg/utils"
streamspb "storj.io/storj/protos/streams" streamspb "storj.io/storj/protos/streams"
) )
@ -147,30 +148,30 @@ func (s *streamStore) Get(ctx context.Context, path paths.Path) (
msi := streamspb.MetaStreamInfo{} msi := streamspb.MetaStreamInfo{}
err = proto.Unmarshal(lastSegmentMeta.Data, &msi) err = proto.Unmarshal(lastSegmentMeta.Data, &msi)
if err != nil { if err != nil {
_ = lastRangerCloser.Close() utils.LogClose(lastRangerCloser)
return nil, Meta{}, err return nil, Meta{}, err
} }
newMeta, err := convertMeta(lastSegmentMeta) newMeta, err := convertMeta(lastSegmentMeta)
if err != nil { if err != nil {
_ = lastRangerCloser.Close() utils.LogClose(lastRangerCloser)
return nil, Meta{}, err return nil, Meta{}, err
} }
var rangers []ranger.RangeCloser var rangers []ranger.RangeCloser
for i := 0; i < int(msi.NumberOfSegments); i++ { for i := int64(0); i < msi.NumberOfSegments; i++ {
currentPath := fmt.Sprintf("s%d", i) currentPath := fmt.Sprintf("s%d", i)
rangeCloser, _, err := s.segments.Get(ctx, path.Prepend(currentPath)) size := msi.SegmentsSize
if err != nil { if i == msi.NumberOfSegments-1 {
for _, ranger := range rangers { size = msi.LastSegmentSize
_ = ranger.Close()
}
_ = lastRangerCloser.Close()
return nil, Meta{}, err
} }
rr := &lazySegmentRanger{
rangers = append(rangers, rangeCloser) segments: s.segments,
path: path.Prepend(currentPath),
size: size,
}
rangers = append(rangers, rr)
} }
rangers = append(rangers, lastRangerCloser) rangers = append(rangers, lastRangerCloser)
@ -256,3 +257,35 @@ func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore pa
return items, more, nil return items, more, nil
} }
type lazySegmentRanger struct {
ranger ranger.RangeCloser
segments segments.Store
path paths.Path
size int64
}
// Size implements Ranger.Size
func (lr *lazySegmentRanger) Size() int64 {
return lr.size
}
// Size implements Ranger.Close
func (lr *lazySegmentRanger) Close() error {
if lr.ranger == nil {
return nil
}
return lr.ranger.Close()
}
// Range implements Ranger.Range to be lazily connected
func (lr *lazySegmentRanger) Range(ctx context.Context, offset, length int64) (io.ReadCloser, error) {
if lr.ranger == nil {
rr, _, err := lr.segments.Get(ctx, lr.path)
if err != nil {
return nil, err
}
lr.ranger = rr
}
return lr.ranger.Range(ctx, offset, length)
}