cmd/uplinkng: port expanded option to show additional metadata (#4229)

cmd/uplinkng/cmd_ls.go

@@ -12,6 +12,7 @@ import (
 	"storj.io/storj/cmd/uplinkng/ulext"
 	"storj.io/storj/cmd/uplinkng/ulfs"
 	"storj.io/storj/cmd/uplinkng/ulloc"
+	"storj.io/uplink"
 )
 
 type cmdLs struct {
@@ -20,6 +21,7 @@ type cmdLs struct {
 	access    string
 	recursive bool
 	encrypted bool
+	expanded  bool
 	pending   bool
 	utc       bool
 
@@ -42,6 +44,10 @@ func (c *cmdLs) Setup(params clingy.Parameters) {
 	c.pending = params.Flag("pending", "List pending object uploads instead", false,
 		clingy.Transform(strconv.ParseBool),
 	).(bool)
+	c.expanded = params.Flag("expanded", "Use expanded output, showing object expiration times and whether there is custom metadata attached", false,
+		clingy.Short('x'),
+		clingy.Transform(strconv.ParseBool),
+	).(bool)
 	c.utc = params.Flag("utc", "Show all timestamps in UTC instead of local time", false,
 		clingy.Transform(strconv.ParseBool),
 	).(bool)
@@ -87,13 +93,19 @@ func (c *cmdLs) listLocation(ctx clingy.Context, prefix ulloc.Location) error {
 		prefix = prefix.AsDirectoryish()
 	}
 
-	tw := newTabbedWriter(ctx.Stdout(), "KIND", "CREATED", "SIZE", "KEY")
+	headers := []string{"KIND", "CREATED", "SIZE", "KEY"}
+	if c.expanded {
+		headers = append(headers, "EXPIRES", "META")
+	}
+
+	tw := newTabbedWriter(ctx.Stdout(), headers...)
 	defer tw.Done()
 
 	// create the object iterator of either existing objects or pending multipart uploads
 	iter, err := fs.List(ctx, prefix, &ulfs.ListOptions{
 		Recursive: c.recursive,
 		Pending:   c.pending,
+		Expanded:  c.expanded,
 	})
 	if err != nil {
 		return err
@@ -102,16 +114,30 @@ func (c *cmdLs) listLocation(ctx clingy.Context, prefix ulloc.Location) error {
 	// iterate and print the results
 	for iter.Next() {
 		obj := iter.Item()
+
+		var parts []interface{}
 		if obj.IsPrefix {
-			tw.WriteLine("PRE", "", "", obj.Loc.Loc())
+			parts = append(parts, "PRE", "", "", obj.Loc.Loc())
-		} else {
+			if c.expanded {
-			tw.WriteLine("OBJ", formatTime(c.utc, obj.Created), obj.ContentLength, obj.Loc.Loc())
+				parts = append(parts, "", "")
 			}
+		} else {
+			parts = append(parts, "OBJ", formatTime(c.utc, obj.Created), obj.ContentLength, obj.Loc.Loc())
+			if c.expanded {
+				parts = append(parts, formatTime(c.utc, obj.Expires), sumMetadataSize(obj.Metadata))
+			}
+		}
+
+		tw.WriteLine(parts...)
 	}
 	return iter.Err()
 }
 
 func formatTime(utc bool, x time.Time) string {
+	if x.IsZero() {
+		return ""
+	}
+
 	if utc {
 		x = x.UTC()
 	} else {
@@ -119,3 +145,12 @@ func formatTime(utc bool, x time.Time) string {
 	}
 	return x.Format("2006-01-02 15:04:05")
 }
+
+func sumMetadataSize(md uplink.CustomMetadata) int {
+	size := 0
+	for k, v := range md {
+		size += len(k)
+		size += len(v)
+	}
+	return size
+}

cmd/uplinkng/ulfs/filesystem.go

@@ -20,6 +20,7 @@ import (
 type ListOptions struct {
 	Recursive bool
 	Pending   bool
+	Expanded  bool
 }
 
 func (lo *ListOptions) isRecursive() bool { return lo != nil && lo.Recursive }
@@ -53,6 +54,8 @@ type ObjectInfo struct {
 	IsPrefix      bool
 	Created       time.Time
 	ContentLength int64
+	Expires       time.Time
+	Metadata      uplink.CustomMetadata
 }
 
 // uplinkObjectToObjectInfo returns an objectInfo converted from an *uplink.Object.
@@ -62,6 +65,8 @@ func uplinkObjectToObjectInfo(bucket string, obj *uplink.Object) ObjectInfo {
 		IsPrefix:      obj.IsPrefix,
 		Created:       obj.System.Created,
 		ContentLength: obj.System.ContentLength,
+		Expires:       obj.System.Expires,
+		Metadata:      obj.Custom,
 	}
 }
 
@@ -72,6 +77,8 @@ func uplinkUploadInfoToObjectInfo(bucket string, upl *uplink.UploadInfo) ObjectI
 		IsPrefix:      upl.IsPrefix,
 		Created:       upl.System.Created,
 		ContentLength: upl.System.ContentLength,
+		Expires:       upl.System.Expires,
+		Metadata:      upl.Custom,
 	}
 }
 

cmd/uplinkng/ulfs/remote.go

@@ -94,6 +94,7 @@ func (r *Remote) List(ctx context.Context, bucket, prefix string, opts *ListOpti
 				Prefix:    parentPrefix,
 				Recursive: opts.Recursive,
 				System:    true,
+				Custom:    opts.Expanded,
 			}),
 		)
 	} else {
@@ -103,6 +104,7 @@ func (r *Remote) List(ctx context.Context, bucket, prefix string, opts *ListOpti
 				Prefix:    parentPrefix,
 				Recursive: opts.Recursive,
 				System:    true,
+				Custom:    opts.Expanded,
 			}),
 		)
 	}

scripts/test-uplinkng.sh

@@ -78,6 +78,7 @@ uplinkng rm "sj://$BUCKET/multisegment-upload-testfile" --access $STORJ_ACCESS
 uplinkng rm "sj://$BUCKET/diff-size-segments"           --access $STORJ_ACCESS
 
 uplinkng ls "sj://$BUCKET" --access $STORJ_ACCESS
+uplinkng ls -x true "sj://$BUCKET" --access $STORJ_ACCESS
 
 uplinkng rb "sj://$BUCKET" --access $STORJ_ACCESS