JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cmd/uplinkng: implement object listing

Browse Source 
Change-Id: Ib5f6964a0c42718913a680529bb66c6f475aeac9
This commit is contained in:
Jeff Wendling 2021-04-05 23:40:04 -04:00
parent cdcc67207c
commit e460dc51f7
6 changed files with 257 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
cmd/uplinkng/cmd_access_list.go
View File

@ -4,11 +4,9 @@
package main
import (
"fmt"
"sort"
"strconv"
"strings"
"text/tabwriter"
"github.com/zeebo/clingy"
@ -32,14 +30,13 @@ func (c *cmdAccessList) Execute(ctx clingy.Context) error {
return err
}
tw := tabwriter.NewWriter(ctx.Stdout(), 4, 4, 4, ' ', 0)
defer func() { _ = tw.Flush() }()
var tw *tabbedWriter
if c.verbose {
fmt.Fprintln(tw, "CURRENT\tNAME\tSATELLITE\tVALUE")
tw = newTabbedWriter(ctx.Stdout(), "CURRENT", "NAME", "SATELLITE", "VALUE")
} else {
fmt.Fprintln(tw, "CURRENT\tNAME\tSATELLITE")
tw = newTabbedWriter(ctx.Stdout(), "CURRENT", "NAME", "SATELLITE")
}
defer tw.Done()
var names []string
for name := range accesses {
@ -63,9 +60,9 @@ func (c *cmdAccessList) Execute(ctx clingy.Context) error {
}
if c.verbose {
fmt.Fprintf(tw, "%c\t%s\t%s\t%s\n", inUse, name, address, accesses[name])
tw.WriteLine(inUse, name, address, accesses[name])
} else {
fmt.Fprintf(tw, "%c\t%s\t%s\n", inUse, name, address)
tw.WriteLine(inUse, name, address)
}
}

129
cmd/uplinkng/cmd_ls.go
View File

@ -4,11 +4,14 @@
package main
import (
"fmt"
"strconv"
"strings"
"time"
"github.com/zeebo/clingy"
"github.com/zeebo/errs"
"storj.io/uplink"
)
type cmdLs struct {
@ -16,8 +19,9 @@ type cmdLs struct {
recursive bool
encrypted bool
pending bool
path *string
prefix *string
}
func (c *cmdLs) Setup(a clingy.Arguments, f clingy.Flags) {
@ -30,15 +34,18 @@ func (c *cmdLs) Setup(a clingy.Arguments, f clingy.Flags) {
c.encrypted = f.New("encrypted", "Shows paths as base64-encoded encrypted paths", false,
clingy.Transform(strconv.ParseBool),
).(bool)
c.pending = f.New("pending", "List pending multipart object uploads instead", false,
clingy.Transform(strconv.ParseBool),
).(bool)
c.path = a.New("path", "Path to list (sj://BUCKET[/KEY])", clingy.Optional).(*string)
c.prefix = a.New("prefix", "Prefix to list (sj://BUCKET[/KEY])", clingy.Optional).(*string)
}
func (c *cmdLs) Execute(ctx clingy.Context) error {
if c.path == nil {
if c.prefix == nil {
return c.listBuckets(ctx)
}
return c.listPath(ctx, *c.path)
return c.listPath(ctx, *c.prefix)
}
func (c *cmdLs) listBuckets(ctx clingy.Context) error {
@ -48,14 +55,122 @@ func (c *cmdLs) listBuckets(ctx clingy.Context) error {
}
defer func() { _ = project.Close() }()
tw := newTabbedWriter(ctx.Stdout(), "CREATED", "NAME")
defer tw.Done()
iter := project.ListBuckets(ctx, nil)
for iter.Next() {
item := iter.Item()
fmt.Fprintln(ctx, "BKT", item.Created.Local().Format("2006-01-02 15:04:05"), item.Name)
tw.WriteLine(formatTime(item.Created), item.Name)
}
return iter.Err()
}
func (c *cmdLs) listPath(ctx clingy.Context, path string) error {
return errs.New("TODO")
bucket, key, ok, err := parsePath(path)
if err != nil {
return err
} else if !ok {
return errs.New("no bucket specified. use format sj://bucket")
}
project, err := c.OpenProject(ctx, bypassEncryption(c.encrypted))
if err != nil {
return err
}
defer func() { _ = project.Close() }()
tw := newTabbedWriter(ctx.Stdout(), "KIND", "CREATED", "SIZE", "KEY")
defer tw.Done()
// in order to get a correct listing, including non-terminating components, what we
// must do is pop the last component off, ensuring the prefix is either empty or
// ends with a /, list there, then filter the results locally against the popped component.
prefix, filter := "", key
if idx := strings.LastIndexByte(key, '/'); idx >= 0 {
prefix, filter = key[:idx+1], key[idx+1:]
}
// create the object iterator of either existing objects or pending multipart uploads
var iter listObjectIterator
if c.pending {
iter = (*uplinkUploadIterator)(project.ListUploads(ctx, bucket,
&uplink.ListUploadsOptions{
Prefix: prefix,
Recursive: c.recursive,
System: true,
}))
} else {
iter = (*uplinkObjectIterator)(project.ListObjects(ctx, bucket,
&uplink.ListObjectsOptions{
Prefix: prefix,
Recursive: c.recursive,
System: true,
}))
}
// iterate and print the results
for iter.Next() {
obj := iter.Item()
key := obj.Key[len(prefix):]
if !strings.HasPrefix(key, filter) {
continue
}
if obj.IsPrefix {
tw.WriteLine("PRE", "", "", key)
} else {
tw.WriteLine("OBJ", formatTime(obj.Created), obj.ContentLength, key)
}
}
return iter.Err()
}
func formatTime(x time.Time) string {
return x.Local().Format("2006-01-02 15:04:05")
}
// the following code wraps the two list iterator types behind an interface so that
// the list code can be generic against either of them.
type listObjectIterator interface {
Next() bool
Err() error
Item() listObject
}
type listObject struct {
Key string
IsPrefix bool
Created time.Time
ContentLength int64
}
type uplinkObjectIterator uplink.ObjectIterator
func (u *uplinkObjectIterator) Next() bool { return (*uplink.ObjectIterator)(u).Next() }
func (u *uplinkObjectIterator) Err() error { return (*uplink.ObjectIterator)(u).Err() }
func (u *uplinkObjectIterator) Item() listObject {
obj := (*uplink.ObjectIterator)(u).Item()
return listObject{
Key: obj.Key,
IsPrefix: obj.IsPrefix,
Created: obj.System.Created,
ContentLength: obj.System.ContentLength,
}
}
type uplinkUploadIterator uplink.UploadIterator
func (u *uplinkUploadIterator) Next() bool { return (*uplink.UploadIterator)(u).Next() }
func (u *uplinkUploadIterator) Err() error { return (*uplink.UploadIterator)(u).Err() }
func (u *uplinkUploadIterator) Item() listObject {
obj := (*uplink.UploadIterator)(u).Item()
return listObject{
Key: obj.Key,
IsPrefix: obj.IsPrefix,
Created: obj.System.Created,
ContentLength: obj.System.ContentLength,
}
}

10
cmd/uplinkng/cmd_version.go
View File

@ -4,11 +4,9 @@
package main
import (
"fmt"
"runtime/debug"
"strconv"
"strings"
"text/tabwriter"
"github.com/zeebo/clingy"
"github.com/zeebo/errs"
@ -31,13 +29,13 @@ func (c *cmdVersion) Execute(ctx clingy.Context) error {
return errs.New("unable to read build info")
}
tw := tabwriter.NewWriter(ctx.Stdout(), 4, 4, 4, ' ', 0)
defer func() { _ = tw.Flush() }()
tw := newTabbedWriter(ctx.Stdout(), "PATH", "VERSION")
defer tw.Done()
fmt.Fprintf(tw, "%s\t%s\n", bi.Main.Path, bi.Main.Version)
tw.WriteLine(bi.Main.Path, bi.Main.Version)
for _, mod := range bi.Deps {
if c.verbose || strings.HasPrefix(mod.Path, "storj.io/") {
fmt.Fprintf(tw, " %s\t%s\n", mod.Path, mod.Version)
tw.WriteLine(mod.Path, mod.Version)
}
}

44
cmd/uplinkng/path.go Normal file
View File

@ -0,0 +1,44 @@
// Copyright (C) 2021 Storj Labs, Inc.
// See LICENSE for copying information.
package main
import (
"strings"
"github.com/zeebo/errs"
)
func parsePath(path string) (bucket, key string, ok bool, err error) {
// Paths, Chapter 2, Verses 9 to 21.
//
// And the Devs spake, saying,
// First shalt thou find the Special Prefix "sj:".
// Then, shalt thou count two slashes, no more, no less.
// Two shall be the number thou shalt count,
// and the number of the counting shall be two.
// Three shalt thou not count, nor either count thou one,
// excepting that thou then proceed to two.
// Four is right out!
// Once the number two, being the second number, be reached,
// then interpret thou thy Path as a remote path,
// which being made of a bucket and key, shall split it.
if strings.HasPrefix(path, "sj://") || strings.HasPrefix(path, "s3://") {
unschemed := path[5:]
bucketIdx := strings.IndexByte(unschemed, '/')
// handles sj:// or sj:///foo
if len(unschemed) == 0 || bucketIdx == 0 {
return "", "", false, errs.New("invalid path: empty bucket in path: %q", path)
}
// handles sj://foo
if bucketIdx == -1 {
return unschemed, "", true, nil
}
return unschemed[:bucketIdx], unschemed[bucketIdx+1:], true, nil
}
return "", "", false, nil
}

27
cmd/uplinkng/project_provider.go
View File

@ -9,6 +9,7 @@ import (
"github.com/zeebo/clingy"
"storj.io/uplink"
privateAccess "storj.io/uplink/private/access"
)
type projectProvider struct {
@ -20,11 +21,16 @@ func (pp *projectProvider) Setup(a clingy.Arguments, f clingy.Flags) {
pp.access = f.New("access", "Which access to use", "").(string)
}
func (pp *projectProvider) OpenProject(ctx context.Context) (*uplink.Project, error) {
func (pp *projectProvider) OpenProject(ctx context.Context, options ...projectOption) (*uplink.Project, error) {
if pp.openProject != nil {
return pp.openProject(ctx)
}
var opts projectOptions
for _, opt := range options {
opt.apply(&opts)
}
accessDefault, accesses, err := gf.GetAccessInfo()
if err != nil {
return nil, err
@ -42,5 +48,24 @@ func (pp *projectProvider) OpenProject(ctx context.Context) (*uplink.Project, er
if err != nil {
return nil, err
}
if opts.encryptionBypass {
if err := privateAccess.EnablePathEncryptionBypass(access); err != nil {
return nil, err
}
}
return uplink.OpenProject(ctx, access)
}
type projectOptions struct {
encryptionBypass bool
}
type projectOption struct {
apply func(*projectOptions)
}
func bypassEncryption(bypass bool) projectOption {
return projectOption{apply: func(opt *projectOptions) { opt.encryptionBypass = bypass }}
}

55
cmd/uplinkng/tabbed_writer.go Normal file
View File

@ -0,0 +1,55 @@
// Copyright (C) 2021 Storj Labs, Inc.
// See LICENSE for copying information.
package main
import (
"fmt"
"io"
"strings"
"text/tabwriter"
)
type tabbedWriter struct {
tw *tabwriter.Writer
headers []string
wrote bool
}
func newTabbedWriter(w io.Writer, headers ...string) *tabbedWriter {
return &tabbedWriter{
tw: tabwriter.NewWriter(w, 4, 4, 4, ' ', 0),
headers: headers,
}
}
func (t *tabbedWriter) Done() {
if t.wrote {
_ = t.tw.Flush()
}
}
func (t *tabbedWriter) WriteLine(parts ...interface{}) {
if !t.wrote {
fmt.Fprintln(t.tw, strings.Join(t.headers, "\t"))
t.wrote = true
}
for i, part := range parts {
if i > 0 {
fmt.Fprint(t.tw, "\t")
}
fmt.Fprint(t.tw, toString(part))
}
fmt.Fprintln(t.tw)
}
func toString(x interface{}) string {
switch x := x.(type) {
case rune:
return string(x)
case string:
return x
default:
return fmt.Sprint(x)
}
}