54ef1c8ca2
the parallelism and parallelism-chunk-size flags which used to control how many parts to split a segment into and many to perform in parallel are now deprecated and replaced by maximum-concurrent-pieces and long-tail-margin. now, for an individual transfer, the total number of piece uploads that transfer will perform is controlled by maximum-concurrent-pieces, and segments within that transfer will automatically be performed in parallel. so if you used to set your parallelism to n, a good value for the pieces might be something approximately like 130*n, and the parallelism-chunk-size is unnecessary. Change-Id: Ibe724ca70b07eba89dad551eb612a1db988b18b9
69 lines
1.3 KiB
Go
69 lines
1.3 KiB
Go
// Copyright (C) 2021 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package ulfs
|
|
|
|
import (
|
|
"github.com/zeebo/errs"
|
|
|
|
"storj.io/storj/cmd/uplink/ulloc"
|
|
)
|
|
|
|
//
|
|
// read handles
|
|
//
|
|
|
|
// osMultiReadHandle implements MultiReadHandle for *os.Files.
|
|
func newOSMultiReadHandle(fh LocalBackendFile) (MultiReadHandle, error) {
|
|
fi, err := fh.Stat()
|
|
if err != nil {
|
|
return nil, errs.Wrap(err)
|
|
}
|
|
return NewGenericMultiReadHandle(fh, ObjectInfo{
|
|
Loc: ulloc.NewLocal(fh.Name()),
|
|
IsPrefix: false,
|
|
Created: fi.ModTime(), // TODO: os specific crtime
|
|
ContentLength: fi.Size(),
|
|
}), nil
|
|
}
|
|
|
|
//
|
|
// write handles
|
|
//
|
|
|
|
type fileGenericWriter struct {
|
|
fs LocalBackend
|
|
raw LocalBackendFile
|
|
done bool
|
|
}
|
|
|
|
func (f *fileGenericWriter) Write(b []byte) (int, error) { return f.raw.Write(b) }
|
|
|
|
func (f *fileGenericWriter) Commit() error {
|
|
if f.done {
|
|
return errs.New("already commit/aborted")
|
|
}
|
|
f.done = true
|
|
|
|
return f.raw.Close()
|
|
}
|
|
|
|
func (f *fileGenericWriter) Abort() error {
|
|
if f.done {
|
|
return errs.New("already commit/aborted")
|
|
}
|
|
f.done = true
|
|
|
|
return errs.Combine(
|
|
f.raw.Close(),
|
|
f.fs.Remove(f.raw.Name()),
|
|
)
|
|
}
|
|
|
|
func newOSWriteHandle(fs LocalBackend, fh LocalBackendFile) WriteHandle {
|
|
return &fileGenericWriter{
|
|
fs: fs,
|
|
raw: fh,
|
|
}
|
|
}
|