54ef1c8ca2
the parallelism and parallelism-chunk-size flags which used to control how many parts to split a segment into and many to perform in parallel are now deprecated and replaced by maximum-concurrent-pieces and long-tail-margin. now, for an individual transfer, the total number of piece uploads that transfer will perform is controlled by maximum-concurrent-pieces, and segments within that transfer will automatically be performed in parallel. so if you used to set your parallelism to n, a good value for the pieces might be something approximately like 130*n, and the parallelism-chunk-size is unnecessary. Change-Id: Ibe724ca70b07eba89dad551eb612a1db988b18b9
58 lines
1.5 KiB
Go
58 lines
1.5 KiB
Go
// Copyright (C) 2021 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
|
|
"storj.io/common/rpc/rpcpool"
|
|
"storj.io/storj/cmd/uplink/ulext"
|
|
"storj.io/storj/cmd/uplink/ulfs"
|
|
"storj.io/uplink"
|
|
privateAccess "storj.io/uplink/private/access"
|
|
"storj.io/uplink/private/testuplink"
|
|
"storj.io/uplink/private/transport"
|
|
)
|
|
|
|
const uplinkCLIUserAgent = "uplink-cli"
|
|
|
|
func (ex *external) OpenFilesystem(ctx context.Context, accessName string, options ...ulext.Option) (ulfs.Filesystem, error) {
|
|
project, err := ex.OpenProject(ctx, accessName, options...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return ulfs.NewMixed(ulfs.NewLocal(ulfs.NewLocalBackendOS()), ulfs.NewRemote(project)), nil
|
|
}
|
|
|
|
func (ex *external) OpenProject(ctx context.Context, accessName string, options ...ulext.Option) (*uplink.Project, error) {
|
|
opts := ulext.LoadOptions(options...)
|
|
|
|
access, err := ex.OpenAccess(accessName)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if opts.EncryptionBypass {
|
|
if err := privateAccess.EnablePathEncryptionBypass(access); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
config := uplink.Config{
|
|
UserAgent: uplinkCLIUserAgent,
|
|
}
|
|
|
|
if opts.ConnectionPoolOptions != (rpcpool.Options{}) {
|
|
if err := transport.SetConnectionPool(ctx, &config, rpcpool.New(opts.ConnectionPoolOptions)); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if opts.ConcurrentSegmentUploadsConfig != (testuplink.ConcurrentSegmentUploadsConfig{}) {
|
|
ctx = testuplink.WithConcurrentSegmentUploadsConfig(ctx, opts.ConcurrentSegmentUploadsConfig)
|
|
}
|
|
|
|
return config.OpenProject(ctx, access)
|
|
}
|