cmd/uplink: integrate server-side copy with uplink cp command

Resolves https://github.com/storj/storj/issues/4486

Change-Id: I42ac2ad2e1a05df4a83606f1990b639f08791403

cmd/uplink/cmd_cp.go

@@ -211,6 +211,10 @@ func (c *cmdCp) copyFile(ctx clingy.Context, fs ulfs.Filesystem, source, dest ul
 		return nil
 	}
 
+	if dest.Remote() && source.Remote() {
+		return fs.Copy(ctx, source, dest)
+	}
+
 	offset, length, err := parseRange(c.byteRange)
 	if err != nil {
 		return errs.Wrap(err)

cmd/uplink/cmd_cp_test.go

@@ -294,6 +294,20 @@ func TestCpRemoteToRemote(t *testing.T) {
 			ultest.File{Loc: "sj://b2/pre/dot-dot/../foo", Contents: "data3"},
 		)
 	})
+
+	t.Run("ObjectToObject", func(t *testing.T) {
+		state.Succeed(t, "cp", "sj://b1/ends-slash", "sj://b1/ends-slash-copy").RequireFiles(t,
+			ultest.File{Loc: "sj://b1/dot-dot/../../../../../foo", Contents: "data1"},
+			ultest.File{Loc: "sj://b1/dot-dot/../../foo", Contents: "data2"},
+			ultest.File{Loc: "sj://b1/dot-dot/../foo", Contents: "data3"},
+			ultest.File{Loc: "sj://b1//starts-slash", Contents: "data4"},
+			ultest.File{Loc: "sj://b1/ends-slash", Contents: "data5"},
+			ultest.File{Loc: "sj://b1/ends-slash-copy", Contents: "data5"},
+			ultest.File{Loc: "sj://b1/ends-slash/", Contents: "data6"},
+			ultest.File{Loc: "sj://b1/ends-slash//", Contents: "data7"},
+			ultest.File{Loc: "sj://b1/mid-slash//file", Contents: "data8"},
+		)
+	})
 }
 
 func TestCpLocalToLocal(t *testing.T) {

cmd/uplink/ulfs/filesystem.go

@@ -42,6 +42,7 @@ type Filesystem interface {
 	Open(ctx clingy.Context, loc ulloc.Location) (MultiReadHandle, error)
 	Create(ctx clingy.Context, loc ulloc.Location, opts *CreateOptions) (MultiWriteHandle, error)
 	Move(ctx clingy.Context, source, dest ulloc.Location) error
+	Copy(ctx clingy.Context, source, dest ulloc.Location) error
 	Remove(ctx context.Context, loc ulloc.Location, opts *RemoveOptions) error
 	List(ctx context.Context, prefix ulloc.Location, opts *ListOptions) (ObjectIterator, error)
 	IsLocalDir(ctx context.Context, loc ulloc.Location) bool

cmd/uplink/ulfs/local.go

@@ -59,6 +59,11 @@ func (l *Local) Move(ctx context.Context, oldpath, newpath string) error {
 	return os.Rename(oldpath, newpath)
 }
 
+// Copy copies file to provided path.
+func (l *Local) Copy(ctx context.Context, oldpath, newpath string) error {
+	return errs.New("not supported")
+}
+
 // Remove unlinks the file at the path. It is not an error if the file does not exist.
 func (l *Local) Remove(ctx context.Context, path string, opts *RemoveOptions) error {
 	if opts.isPending() {

cmd/uplink/ulfs/mixed.go

@@ -65,6 +65,20 @@ func (m *Mixed) Move(ctx clingy.Context, source, dest ulloc.Location) error {
 	return errs.New("moving objects between local and remote is not supported")
 }
 
+// Copy copies either a local file or remote object.
+func (m *Mixed) Copy(ctx clingy.Context, source, dest ulloc.Location) error {
+	if oldbucket, oldkey, ok := source.RemoteParts(); ok {
+		if newbucket, newkey, ok := dest.RemoteParts(); ok {
+			return m.remote.Copy(ctx, oldbucket, oldkey, newbucket, newkey)
+		}
+	} else if oldpath, ok := source.LocalParts(); ok {
+		if newpath, ok := dest.LocalParts(); ok {
+			return m.local.Copy(ctx, oldpath, newpath)
+		}
+	}
+	return errs.New("copying objects between local and remote is not supported")
+}
+
 // Remove deletes either a local file or remote object.
 func (m *Mixed) Remove(ctx context.Context, loc ulloc.Location, opts *RemoveOptions) error {
 	if bucket, key, ok := loc.RemoteParts(); ok {

cmd/uplink/ulfs/remote.go

@@ -61,6 +61,12 @@ func (r *Remote) Move(ctx context.Context, oldbucket, oldkey, newbucket, newkey
 	return errs.Wrap(r.project.MoveObject(ctx, oldbucket, oldkey, newbucket, newkey, nil))
 }
 
+// Copy copies object to provided key and bucket.
+func (r *Remote) Copy(ctx context.Context, oldbucket, oldkey, newbucket, newkey string) error {
+	_, err := r.project.CopyObject(ctx, oldbucket, oldkey, newbucket, newkey, nil)
+	return errs.Wrap(err)
+}
+
 // Remove deletes the object at the provided key and bucket.
 func (r *Remote) Remove(ctx context.Context, bucket, key string, opts *RemoveOptions) error {
 	if !opts.isPending() {

cmd/uplink/ultest/filesystem.go

@@ -174,6 +174,18 @@ func (tfs *testFilesystem) Move(ctx clingy.Context, source, dest ulloc.Location)
 	return nil
 }
 
+func (tfs *testFilesystem) Copy(ctx clingy.Context, source, dest ulloc.Location) error {
+	tfs.mu.Lock()
+	defer tfs.mu.Unlock()
+
+	mf, ok := tfs.files[source]
+	if !ok {
+		return errs.New("file does not exist %q", source)
+	}
+	tfs.files[dest] = mf
+	return nil
+}
+
 func (tfs *testFilesystem) Remove(ctx context.Context, loc ulloc.Location, opts *ulfs.RemoveOptions) error {
 	tfs.mu.Lock()
 	defer tfs.mu.Unlock()

scripts/test-uplink.sh

@@ -96,10 +96,17 @@ uplink mv "sj://$BUCKET/big-upload-testfile"       "sj://$BUCKET/moved-big-uploa
 uplink ls "sj://$BUCKET/moved-big-upload-testfile" | grep "moved-big-upload-testfile"
 uplink mv "sj://$BUCKET/moved-big-upload-testfile" "sj://$BUCKET/big-upload-testfile"
 
+# test server-side copy operation
+uplink cp "sj://$BUCKET/big-upload-testfile"        "sj://$BUCKET/copied-big-upload-testfile"
+uplink ls "sj://$BUCKET/copied-big-upload-testfile" | grep "copied-big-upload-testfile"
+uplink ls "sj://$BUCKET/big-upload-testfile"        | grep "big-upload-testfile"
+uplink cp "sj://$BUCKET/copied-big-upload-testfile" "$DST_DIR/copied-big-upload-testfile"
+compare_files "$SRC_DIR/big-upload-testfile"        "$DST_DIR/copied-big-upload-testfile"
+
 # move prefix
 uplink mv "sj://$BUCKET/" "sj://$BUCKET/my-prefix/" --recursive
 FILES=$(uplink ls "sj://$BUCKET/my-prefix/" | tee "$TMPDIR/list" | wc -l)
-EXPECTED_FILES="6" # 5 objects + one line more for headers
+EXPECTED_FILES="7" # 6 objects + one line more for headers
 if [ "$FILES" == "$EXPECTED_FILES" ]
 then
     echo "listing after move returns $FILES files"
@@ -115,6 +122,7 @@ uplink rm "sj://$BUCKET/big-upload-testfile"
 uplink rm "sj://$BUCKET/multisegment-upload-testfile"
 uplink rm "sj://$BUCKET/diff-size-segments"
 uplink rm "sj://$BUCKET/diff-size-segments_upl_p2"
+uplink rm "sj://$BUCKET/copied-big-upload-testfile"
 
 uplink ls "sj://$BUCKET"
 uplink ls -x "sj://$BUCKET"