JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cmd/uplink: bring back --metadata for cp command

Browse Source 
At some point uplink cli lost ability to set metadata. This change
brings back this functionality for 'cp' operation.

https://github.com/storj/storj/issues/3848

Change-Id: Ia5f60eb577fcab8a38d94730d8cdc6e0338d3b46
This commit is contained in:
Michał Niewrzał 2022-05-13 12:21:01 +02:00 committed by Jeff Wendling
parent bdcd285685
commit d90ce467fc
8 changed files with 76 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/uplink/cmd_cp.go
View File

@ -36,6 +36,7 @@ type cmdCp struct {
progress bool
byteRange string
expires time.Time
metadata map[string]string
parallelism int
parallelismChunkSize memory.Size
@ -98,6 +99,10 @@ func (c *cmdCp) Setup(params clingy.Parameters) {
"Schedule removal after this time (e.g. '+2h', 'now', '2020-01-02T15:04:05Z0700')",
time.Time{}, clingy.Transform(parseHumanDate), clingy.Type("relative_date")).(time.Time)
c.metadata = params.Flag("metadata",
"optional metadata for the object. Please use a single level JSON object of string to string only",
nil, clingy.Transform(parseJSON), clingy.Type("string")).(map[string]string)
c.source = params.Arg("source", "Source to copy", clingy.Transform(ulloc.Parse)).(ulloc.Location)
c.dest = params.Arg("dest", "Destination to copy", clingy.Transform(ulloc.Parse)).(ulloc.Location)
}
@ -234,7 +239,10 @@ func (c *cmdCp) copyFile(ctx clingy.Context, fs ulfs.Filesystem, source, dest ul
}
defer func() { _ = mrh.Close() }()
mwh, err := fs.Create(ctx, dest, &ulfs.CreateOptions{Expires: c.expires})
mwh, err := fs.Create(ctx, dest, &ulfs.CreateOptions{
Expires: c.expires,
Metadata: c.metadata,
})
if err != nil {
return err
}

9
cmd/uplink/cmd_cp_test.go
View File

@ -200,6 +200,15 @@ func TestCpUpload(t *testing.T) {
state.Succeed(t, "cp", "/home/user/fi", "sj://user/folder", "--recursive").RequireRemoteFiles(t)
})
t.Run("Metadata", func(t *testing.T) {
state.Succeed(t, "cp", "--metadata", "{\"key\":\"value\"}", "/home/user/file1.txt", "sj://user/file_with_metadata.txt").RequireRemoteFiles(t,
ultest.File{Loc: "sj://user/file1.txt", Contents: "remote"},
ultest.File{Loc: "sj://user/file_with_metadata.txt", Contents: "local", Metadata: map[string]string{
"key": "value",
}},
)
})
}
func TestCpRecursiveDifficult(t *testing.T) {

15
cmd/uplink/stdlib_flags.go
View File

@ -4,6 +4,7 @@
package main
import (
"encoding/json"
"flag"
"time"
@ -55,3 +56,17 @@ func parseHumanDate(date string) (time.Time, error) {
return t, errs.Wrap(err)
}
}
// parseJSON parses command-line flags which accept JSON string.
// It can be passed to clingy.Transform to create a clingy.Option.
func parseJSON(jsonString string) (map[string]string, error) {
if len(jsonString) > 0 {
var jsonValue map[string]string
err := json.Unmarshal([]byte(jsonString), &jsonValue)
if err != nil {
return nil, err
}
return jsonValue, nil
}
return nil, nil
}

3
cmd/uplink/ulfs/filesystem.go
View File

@ -16,7 +16,8 @@ import (
// CreateOptions contains extra options to create an object.
type CreateOptions struct {
Expires time.Time
Expires time.Time
Metadata map[string]string
}
// ListOptions describes options to the List command.

20
cmd/uplink/ulfs/handle_uplink.go
View File

@ -167,20 +167,22 @@ func (u *uplinkReadHandle) Info() ObjectInfo { return *u.info }
//
type uplinkMultiWriteHandle struct {
project *uplink.Project
bucket string
info uplink.UploadInfo
project *uplink.Project
bucket string
info uplink.UploadInfo
metadata uplink.CustomMetadata
mu sync.Mutex
tail bool
part uint32
}
func newUplinkMultiWriteHandle(project *uplink.Project, bucket string, info uplink.UploadInfo) *uplinkMultiWriteHandle {
func newUplinkMultiWriteHandle(project *uplink.Project, bucket string, info uplink.UploadInfo, metadata uplink.CustomMetadata) *uplinkMultiWriteHandle {
return &uplinkMultiWriteHandle{
project: project,
bucket: bucket,
info: info,
project: project,
bucket: bucket,
info: info,
metadata: metadata,
}
}
@ -214,7 +216,9 @@ func (u *uplinkMultiWriteHandle) NextPart(ctx context.Context, length int64) (Wr
}
func (u *uplinkMultiWriteHandle) Commit(ctx context.Context) error {
_, err := u.project.CommitUpload(ctx, u.bucket, u.info.Key, u.info.UploadID, nil)
_, err := u.project.CommitUpload(ctx, u.bucket, u.info.Key, u.info.UploadID, &uplink.CommitUploadOptions{
CustomMetadata: u.metadata,
})
return err
}

11
cmd/uplink/ulfs/remote.go
View File

@ -47,13 +47,22 @@ func (r *Remote) Stat(ctx context.Context, bucket, key string) (*ObjectInfo, err
// Create returns a MultiWriteHandle for the object identified by a given bucket and key.
func (r *Remote) Create(ctx context.Context, bucket, key string, opts *CreateOptions) (MultiWriteHandle, error) {
var customMetadata uplink.CustomMetadata
if opts.Metadata != nil {
customMetadata = uplink.CustomMetadata(opts.Metadata)
if err := customMetadata.Verify(); err != nil {
return nil, err
}
}
info, err := r.project.BeginUpload(ctx, bucket, key, &uplink.UploadOptions{
Expires: opts.Expires,
})
if err != nil {
return nil, err
}
return newUplinkMultiWriteHandle(r.project, bucket, info), nil
return newUplinkMultiWriteHandle(r.project, bucket, info, customMetadata), nil
}
// Move moves object to provided key and bucket.

28
cmd/uplink/ultest/filesystem.go
View File

@ -42,6 +42,7 @@ type memFileData struct {
contents string
created int64
expires time.Time
metadata map[string]string
}
func (mf memFileData) expired() bool {
@ -60,6 +61,7 @@ func (rfs *remoteFilesystem) Files() (files []File) {
files = append(files, File{
Loc: loc.String(),
Contents: mf.contents,
Metadata: mf.metadata,
})
}
sort.Slice(files, func(i, j int) bool { return files[i].less(files[j]) })
@ -72,6 +74,7 @@ func (rfs *remoteFilesystem) Pending() (files []File) {
files = append(files, File{
Loc: loc.String(),
Contents: string(h.buf),
Metadata: h.metadata,
})
}
}
@ -119,17 +122,20 @@ func (rfs *remoteFilesystem) Create(ctx context.Context, bucket, key string, opt
return nil, errs.New("bucket %q does not exist", bucket)
}
var metadata map[string]string
expires := time.Time{}
if opts != nil {
expires = opts.Expires
metadata = opts.Metadata
}
rfs.created++
wh := &memWriteHandle{
loc: loc,
rfs: rfs,
cre: rfs.created,
expires: expires,
loc: loc,
rfs: rfs,
cre: rfs.created,
expires: expires,
metadata: metadata,
}
rfs.pending[loc] = append(rfs.pending[loc], wh)
@ -267,12 +273,13 @@ func (rfs *remoteFilesystem) Stat(ctx context.Context, bucket, key string) (*ulf
//
type memWriteHandle struct {
buf []byte
loc ulloc.Location
rfs *remoteFilesystem
cre int64
expires time.Time
done bool
buf []byte
loc ulloc.Location
rfs *remoteFilesystem
cre int64
expires time.Time
metadata map[string]string
done bool
}
func (b *memWriteHandle) WriteAt(p []byte, off int64) (int, error) {
@ -298,6 +305,7 @@ func (b *memWriteHandle) Commit() error {
contents: string(b.buf),
created: b.cre,
expires: b.expires,
metadata: b.metadata,
}
return nil

1
cmd/uplink/ultest/result.go
View File

@ -214,6 +214,7 @@ func globMatchLine(pattern, line string) bool {
type File struct {
Loc string
Contents string
Metadata map[string]string
}
func (f File) less(g File) bool {