JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cmd/uplink: adding output flag for ls command

Browse Source 
With this change users can use the uplink cli in
scripts (ie. bash) more easily, since the output
can be switched to an easier processable json format.
It keeps the default of tabbed output.


Change-Id: I37e2c55f75c2250c3119fd8df8b66a766ff9096b
This commit is contained in:
Stefan Benten 2022-04-28 21:14:18 +02:00
parent 96411ba56a
commit 345ab87b5c
2 changed files with 159 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
cmd/uplink/cmd_ls.go
View File

@ -4,10 +4,12 @@
package main
import (
"encoding/json"
"strconv"
"time"
"github.com/zeebo/clingy"
"github.com/zeebo/errs"
"storj.io/storj/cmd/uplink/ulext"
"storj.io/storj/cmd/uplink/ulfs"
@ -24,6 +26,7 @@ type cmdLs struct {
expanded bool
pending bool
utc bool
output string
prefix *ulloc.Location
}
@ -51,6 +54,9 @@ func (c *cmdLs) Setup(params clingy.Parameters) {
c.utc = params.Flag("utc", "Show all timestamps in UTC instead of local time", false,
clingy.Transform(strconv.ParseBool), clingy.Boolean,
).(bool)
c.output = params.Flag("output", "Output Format (tabbed, json)", "tabbed",
clingy.Short('o'),
).(string)
c.prefix = params.Arg("prefix", "Prefix to list (sj://BUCKET[/KEY])", clingy.Optional,
clingy.Transform(ulloc.Parse),
@ -71,15 +77,16 @@ func (c *cmdLs) listBuckets(ctx clingy.Context) error {
}
defer func() { _ = project.Close() }()
tw := newTabbedWriter(ctx.Stdout(), "CREATED", "NAME")
defer tw.Done()
iter := project.ListBuckets(ctx, nil)
for iter.Next() {
item := iter.Item()
tw.WriteLine(formatTime(c.utc, item.Created), item.Name)
switch c.output {
case "tabbed":
return c.printTabbedBucket(ctx, iter)
case "json":
return c.printJSONBucket(ctx, iter)
default:
return errs.New("unknown output format, got %s", c.output)
}
return iter.Err()
}
func (c *cmdLs) listLocation(ctx clingy.Context, prefix ulloc.Location) error {
@ -93,14 +100,6 @@ func (c *cmdLs) listLocation(ctx clingy.Context, prefix ulloc.Location) error {
prefix = prefix.AsDirectoryish()
}
headers := []string{"KIND", "CREATED", "SIZE", "KEY"}
if c.expanded {
headers = append(headers, "EXPIRES", "META")
}
tw := newTabbedWriter(ctx.Stdout(), headers...)
defer tw.Done()
// create the object iterator of either existing objects or pending multipart uploads
iter, err := fs.List(ctx, prefix, &ulfs.ListOptions{
Recursive: c.recursive,
@ -111,6 +110,50 @@ func (c *cmdLs) listLocation(ctx clingy.Context, prefix ulloc.Location) error {
return err
}
switch c.output {
case "tabbed":
return c.printTabbedLocation(ctx, iter)
case "json":
return c.printJSONLocation(ctx, iter)
default:
return errs.New("unknown output format, got %s", c.output)
}
}
func (c *cmdLs) printTabbedBucket(ctx clingy.Context, iter *uplink.BucketIterator) (err error) {
tw := newTabbedWriter(ctx.Stdout(), "CREATED", "NAME")
defer tw.Done()
for iter.Next() {
item := iter.Item()
tw.WriteLine(formatTime(c.utc, item.Created), item.Name)
}
return iter.Err()
}
func (c *cmdLs) printJSONBucket(ctx clingy.Context, iter *uplink.BucketIterator) (err error) {
jw := json.NewEncoder(ctx.Stdout())
for iter.Next() {
obj := iter.Item()
err = jw.Encode(obj)
if err != nil {
return err
}
}
return iter.Err()
}
func (c *cmdLs) printTabbedLocation(ctx clingy.Context, iter ulfs.ObjectIterator) (err error) {
headers := []string{"KIND", "CREATED", "SIZE", "KEY"}
if c.expanded {
headers = append(headers, "EXPIRES", "META")
}
tw := newTabbedWriter(ctx.Stdout(), headers...)
defer tw.Done()
// iterate and print the results
for iter.Next() {
obj := iter.Item()
@ -133,6 +176,34 @@ func (c *cmdLs) listLocation(ctx clingy.Context, prefix ulloc.Location) error {
return iter.Err()
}
func (c *cmdLs) printJSONLocation(ctx clingy.Context, iter ulfs.ObjectIterator) (err error) {
jw := json.NewEncoder(ctx.Stdout())
for iter.Next() {
obj := iter.Item()
if obj.IsPrefix {
err = jw.Encode(struct {
Kind string `json:"kind"`
Key string `json:"key"`
}{"PRE", obj.Loc.Loc()})
} else {
err = jw.Encode(struct {
Kind string `json:"kind"`
Created string `json:"created"`
Size int64 `json:"size"`
Key string `json:"key"`
Expires string `json:"expires,omitempty"`
Metadata int `json:"meta,omitempty"`
}{"OBJ", formatTime(c.utc, obj.Created), obj.ContentLength, obj.Loc.Loc(), formatTime(c.utc, obj.Expires), sumMetadataSize(obj.Metadata)})
}
if err != nil {
return err
}
}
return iter.Err()
}
func formatTime(utc bool, x time.Time) string {
if x.IsZero() {
return ""

73
cmd/uplink/cmd_ls_test.go
View File

@ -88,6 +88,79 @@ func TestLsRemote(t *testing.T) {
})
}
func TestLsJSON(t *testing.T) {
state := ultest.Setup(commands,
ultest.WithFile("sj://user/deep/aaa/bbb/1"),
ultest.WithFile("sj://user/deep/aaa/bbb/2"),
ultest.WithFile("sj://user/deep/aaa/bbb/3"),
ultest.WithFile("sj://user/foobar"),
ultest.WithFile("sj://user/foobar/"),
ultest.WithFile("sj://user/foobar/1"),
ultest.WithFile("sj://user/foobar/2"),
ultest.WithFile("sj://user/foobar/3"),
ultest.WithFile("sj://user/foobaz/1"),
ultest.WithPendingFile("sj://user/invisible"),
)
t.Run("Recursive", func(t *testing.T) {
state.Succeed(t, "ls", "sj://user", "--recursive", "--utc", "--output", "json").RequireStdout(t, `
{"kind":"OBJ","created":"1970-01-01 00:00:01","size":0,"key":"deep/aaa/bbb/1"}
{"kind":"OBJ","created":"1970-01-01 00:00:02","size":0,"key":"deep/aaa/bbb/2"}
{"kind":"OBJ","created":"1970-01-01 00:00:03","size":0,"key":"deep/aaa/bbb/3"}
{"kind":"OBJ","created":"1970-01-01 00:00:04","size":0,"key":"foobar"}
{"kind":"OBJ","created":"1970-01-01 00:00:05","size":0,"key":"foobar/"}
{"kind":"OBJ","created":"1970-01-01 00:00:06","size":0,"key":"foobar/1"}
{"kind":"OBJ","created":"1970-01-01 00:00:07","size":0,"key":"foobar/2"}
{"kind":"OBJ","created":"1970-01-01 00:00:08","size":0,"key":"foobar/3"}
{"kind":"OBJ","created":"1970-01-01 00:00:09","size":0,"key":"foobaz/1"}
`)
})
t.Run("Basic", func(t *testing.T) {
state.Succeed(t, "ls", "sj://user/fo", "--utc", "--output", "json").RequireStdout(t, ``)
})
t.Run("ExactPrefix", func(t *testing.T) {
state.Succeed(t, "ls", "sj://user/foobar", "--utc", "--output", "json").RequireStdout(t, `
{"kind":"OBJ","created":"1970-01-01 00:00:04","size":0,"key":"foobar"}
{"kind":"PRE","key":"foobar/"}
`)
})
t.Run("ShortFlag", func(t *testing.T) {
state.Succeed(t, "ls", "sj://user/foobar", "--utc", "-o", "json").RequireStdout(t, `
{"kind":"OBJ","created":"1970-01-01 00:00:04","size":0,"key":"foobar"}
{"kind":"PRE","key":"foobar/"}
`)
})
t.Run("ExactPrefixWithSlash", func(t *testing.T) {
state.Succeed(t, "ls", "sj://user/foobar/", "--utc", "--output", "json").RequireStdout(t, `
{"kind":"OBJ","created":"1970-01-01 00:00:05","size":0,"key":""}
{"kind":"OBJ","created":"1970-01-01 00:00:06","size":0,"key":"1"}
{"kind":"OBJ","created":"1970-01-01 00:00:07","size":0,"key":"2"}
{"kind":"OBJ","created":"1970-01-01 00:00:08","size":0,"key":"3"}
`)
})
t.Run("MultipleLayers", func(t *testing.T) {
state.Succeed(t, "ls", "sj://user/deep/", "--output", "json").RequireStdout(t, `
{"kind":"PRE","key":"aaa/"}
`)
state.Succeed(t, "ls", "sj://user/deep/aaa/", "--output", "json").RequireStdout(t, `
{"kind":"PRE","key":"bbb/"}
`)
state.Succeed(t, "ls", "sj://user/deep/aaa/bbb/", "--utc", "--output", "json").RequireStdout(t, `
{"kind":"OBJ","created":"1970-01-01 00:00:01","size":0,"key":"1"}
{"kind":"OBJ","created":"1970-01-01 00:00:02","size":0,"key":"2"}
{"kind":"OBJ","created":"1970-01-01 00:00:03","size":0,"key":"3"}
`)
})
}
func TestLsPending(t *testing.T) {
state := ultest.Setup(commands,
ultest.WithPendingFile("sj://user/deep/aaa/bbb/1"),