cmd/uplink: add ranged download to uplink cli

Change-Id: Ib274df024a8ffc5db2d5c99f8f363efa3b43723f
This commit is contained in:
Clement Sam 2021-10-20 14:07:03 +00:00
parent a281adddd5
commit 85b49bb27c
3 changed files with 69 additions and 8 deletions

View File

@ -20,16 +20,18 @@ import (
"storj.io/common/fpath"
"storj.io/common/memory"
"storj.io/common/ranger/httpranger"
"storj.io/common/sync2"
"storj.io/uplink"
"storj.io/uplink/private/object"
)
var (
progress *bool
expires *string
metadata *string
parallelism *int
progress *bool
expires *string
metadata *string
parallelism *int
byteRangeStr *string
)
func init() {
@ -44,8 +46,9 @@ func init() {
expires = cpCmd.Flags().String("expires", "", "optional expiration date of an object. Please use format (yyyy-mm-ddThh:mm:ssZhh:mm)")
metadata = cpCmd.Flags().String("metadata", "", "optional metadata for the object. Please use a single level JSON object of string to string only")
parallelism = cpCmd.Flags().Int("parallelism", 1, "controls how many parallel uploads/downloads of a single object will be performed")
byteRangeStr = cpCmd.Flags().String("range", "", "Downloads the specified range bytes of an object. For more information about the HTTP Range header, see https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35")
setBasicFlags(cpCmd.Flags(), "progress", "expires", "metadata", "parallelism")
setBasicFlags(cpCmd.Flags(), "progress", "expires", "metadata", "parallelism", "range")
}
// upload transfers src from local machine to s3 compatible object dst.
@ -280,6 +283,10 @@ func download(ctx context.Context, src fpath.FPath, dst fpath.FPath, showProgres
return fmt.Errorf("parallelism must be at least 1")
}
if *parallelism > 1 && *byteRangeStr != "" {
return fmt.Errorf("--parellelism and --range flags are mutually exclusive")
}
project, err := cfg.getProject(ctx, false)
if err != nil {
return err
@ -306,8 +313,34 @@ func download(ctx context.Context, src fpath.FPath, dst fpath.FPath, showProgres
}
var bar *progressbar.ProgressBar
var contentLength int64
if *parallelism <= 1 {
download, err := project.DownloadObject(ctx, src.Bucket(), src.Path(), nil)
var downloadOpts *uplink.DownloadOptions
if *byteRangeStr != "" {
// TODO: if range option will be frequently used we may think about avoiding this call
statObject, err := project.StatObject(ctx, src.Bucket(), src.Path())
if err != nil {
return err
}
bRange, err := httpranger.ParseRange(*byteRangeStr, statObject.System.ContentLength)
if err != nil && bRange == nil {
return fmt.Errorf("error parsing range: %w", err)
}
if len(bRange) == 0 {
return fmt.Errorf("invalid range")
}
if len(bRange) > 1 {
return fmt.Errorf("retrieval of multiple byte ranges of data not supported: %d provided", len(bRange))
}
downloadOpts = &uplink.DownloadOptions{
Offset: bRange[0].Start,
Length: bRange[0].Length,
}
contentLength = bRange[0].Length
}
download, err := project.DownloadObject(ctx, src.Bucket(), src.Path(), downloadOpts)
if err != nil {
return err
}
@ -315,8 +348,11 @@ func download(ctx context.Context, src fpath.FPath, dst fpath.FPath, showProgres
var reader io.ReadCloser
if showProgress {
info := download.Info()
bar = progressbar.New64(info.System.ContentLength)
if contentLength <= 0 {
info := download.Info()
contentLength = info.System.ContentLength
}
bar = progressbar.New64(contentLength)
reader = bar.NewProxyReader(download)
bar.Start()
} else {

View File

@ -81,6 +81,27 @@ uplink cp "sj://$BUCKET/diff-size-segments_upl_p2" "$DST_DIR/diff-size-segmen
uplink ls "sj://$BUCKET/small-upload-testfile" | grep "small-upload-testfile"
# test ranged download of object
uplink cp "sj://$BUCKET/put-file" "$DST_DIR/put-file-from-cp-range" --range bytes=0-5 --progress=false
EXPECTED_FILE_SIZE="6"
ACTUAL_FILE_SIZE=$(get_file_size "$DST_DIR/put-file-from-cp-range")
if [ "$EXPECTED_FILE_SIZE" != "$ACTUAL_FILE_SIZE" ]
then
echo "expected downloaded file size to be equal to $EXPECTED_FILE_SIZE, got $ACTUAL_FILE_SIZE"
exit 1
fi
# test ranged download with multiple byte range
set +e
EXPECTED_ERROR="retrieval of multiple byte ranges of data not supported: 2 provided"
ERROR=$(uplink cp "sj://$BUCKET/put-file" "$DST_DIR/put-file-from-cp-range" --range bytes=0-5,6-10)
if [ $ERROR != $EXPECTED_ERROR ]
then
echo EXPECTED_ERROR
exit 1
fi
set -e
# test server-side move operation
uplink mv "sj://$BUCKET/big-upload-testfile" "sj://$BUCKET/moved-big-upload-testfile"
uplink ls "sj://$BUCKET/moved-big-upload-testfile" | grep "moved-big-upload-testfile"

View File

@ -144,4 +144,8 @@ require_error_exit_code(){
else
echo "Copy file without permission: PASSED" # Expect unsuccessful exit code
fi
}
get_file_size() {
[ -f "$1" ] && ls -dnL -- "$1" | awk '{print $5;exit}' || { echo 0; return 1; }
}