cmd/uplinkng: refactor into some focused packages
the directory was starting to get pretty large and it was making it hard to pick concise names for types and variables. this moves the location stuff into a cmd/uplinkng/ulloc package, the filesystem stuff into a cmd/uplinkng/ulfs package, and the testing stuff into a cmd/uplinkng/ultest package. this should make the remaining stuff in cmd/uplinkng only the business logic of how to implement the commands, rather than also including a bunch of helper utilities and scaffolding. Change-Id: Id0901625ebfff9b1cf2dae52366aceb3b6c8f5b6
This commit is contained in:
Jeff Wendling
parent
b24ea2ead5
commit
98be54b9a3
@ -11,6 +11,9 @@ import (
|
||||
progressbar "github.com/cheggaaa/pb/v3"
|
||||
"github.com/zeebo/clingy"
|
||||
"github.com/zeebo/errs"
|
||||
|
||||
"storj.io/storj/cmd/uplinkng/ulfs"
|
||||
"storj.io/storj/cmd/uplinkng/ulloc"
|
||||
)
|
||||
|
||||
type cmdCp struct {
|
||||
@ -19,8 +22,8 @@ type cmdCp struct {
|
||||
recursive bool
|
||||
dryrun bool
|
||||
|
||||
source Location
|
||||
dest Location
|
||||
source ulloc.Location
|
||||
dest ulloc.Location
|
||||
}
|
||||
|
||||
func (c *cmdCp) Setup(a clingy.Arguments, f clingy.Flags) {
|
||||
@ -34,8 +37,8 @@ func (c *cmdCp) Setup(a clingy.Arguments, f clingy.Flags) {
|
||||
clingy.Transform(strconv.ParseBool),
|
||||
).(bool)
|
||||
|
||||
c.source = a.New("source", "Source to copy", clingy.Transform(parseLocation)).(Location)
|
||||
c.dest = a.New("dest", "Desination to copy", clingy.Transform(parseLocation)).(Location)
|
||||
c.source = a.New("source", "Source to copy", clingy.Transform(ulloc.Parse)).(ulloc.Location)
|
||||
c.dest = a.New("dest", "Desination to copy", clingy.Transform(ulloc.Parse)).(ulloc.Location)
|
||||
}
|
||||
|
||||
func (c *cmdCp) Execute(ctx clingy.Context) error {
|
||||
@ -51,7 +54,7 @@ func (c *cmdCp) Execute(ctx clingy.Context) error {
|
||||
return c.copyFile(ctx, fs, c.source, c.dest, true)
|
||||
}
|
||||
|
||||
func (c *cmdCp) copyRecursive(ctx clingy.Context, fs filesystem) error {
|
||||
func (c *cmdCp) copyRecursive(ctx clingy.Context, fs ulfs.Filesystem) error {
|
||||
if c.source.Std() || c.dest.Std() {
|
||||
return errs.New("cannot recursively copy to stdin/stdout")
|
||||
}
|
||||
@ -86,7 +89,7 @@ func (c *cmdCp) copyRecursive(ctx clingy.Context, fs filesystem) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *cmdCp) copyFile(ctx clingy.Context, fs filesystem, source, dest Location, progress bool) error {
|
||||
func (c *cmdCp) copyFile(ctx clingy.Context, fs ulfs.Filesystem, source, dest ulloc.Location, progress bool) error {
|
||||
if isDir := fs.IsLocalDir(ctx, dest); isDir {
|
||||
base, ok := source.Base()
|
||||
if !ok {
|
||||
@ -131,11 +134,11 @@ func (c *cmdCp) copyFile(ctx clingy.Context, fs filesystem, source, dest Locatio
|
||||
return errs.Wrap(wh.Commit())
|
||||
}
|
||||
|
||||
func copyVerb(source, dest Location) string {
|
||||
func copyVerb(source, dest ulloc.Location) string {
|
||||
switch {
|
||||
case dest.remote:
|
||||
case dest.Remote():
|
||||
return "upload"
|
||||
case source.remote:
|
||||
case source.Remote():
|
||||
return "download"
|
||||
default:
|
||||
return "copy"
|
||||
|
||||
@ -8,6 +8,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/zeebo/clingy"
|
||||
|
||||
"storj.io/storj/cmd/uplinkng/ulfs"
|
||||
"storj.io/storj/cmd/uplinkng/ulloc"
|
||||
)
|
||||
|
||||
type cmdLs struct {
|
||||
@ -18,7 +21,7 @@ type cmdLs struct {
|
||||
pending bool
|
||||
utc bool
|
||||
|
||||
prefix *Location
|
||||
prefix *ulloc.Location
|
||||
}
|
||||
|
||||
func (c *cmdLs) Setup(a clingy.Arguments, f clingy.Flags) {
|
||||
@ -39,8 +42,8 @@ func (c *cmdLs) Setup(a clingy.Arguments, f clingy.Flags) {
|
||||
).(bool)
|
||||
|
||||
c.prefix = a.New("prefix", "Prefix to list (sj://BUCKET[/KEY])", clingy.Optional,
|
||||
clingy.Transform(parseLocation),
|
||||
).(*Location)
|
||||
clingy.Transform(ulloc.Parse),
|
||||
).(*ulloc.Location)
|
||||
}
|
||||
|
||||
func (c *cmdLs) Execute(ctx clingy.Context) error {
|
||||
@ -68,7 +71,7 @@ func (c *cmdLs) listBuckets(ctx clingy.Context) error {
|
||||
return iter.Err()
|
||||
}
|
||||
|
||||
func (c *cmdLs) listLocation(ctx clingy.Context, prefix Location) error {
|
||||
func (c *cmdLs) listLocation(ctx clingy.Context, prefix ulloc.Location) error {
|
||||
fs, err := c.OpenFilesystem(ctx, bypassEncryption(c.encrypted))
|
||||
if err != nil {
|
||||
return err
|
||||
@ -79,7 +82,7 @@ func (c *cmdLs) listLocation(ctx clingy.Context, prefix Location) error {
|
||||
defer tw.Done()
|
||||
|
||||
// create the object iterator of either existing objects or pending multipart uploads
|
||||
var iter objectIterator
|
||||
var iter ulfs.ObjectIterator
|
||||
if c.pending {
|
||||
iter, err = fs.ListUploads(ctx, prefix, c.recursive)
|
||||
} else {
|
||||
|
||||
@ -5,32 +5,34 @@ package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"storj.io/storj/cmd/uplinkng/ultest"
|
||||
)
|
||||
|
||||
func TestLsErrors(t *testing.T) {
|
||||
state := Setup(t)
|
||||
state := ultest.Setup(commands)
|
||||
|
||||
// empty bucket name is a parse error
|
||||
state.Fail(t, "ls", "sj:///jeff")
|
||||
state.Fail(t, "ls", "sj:///user")
|
||||
}
|
||||
|
||||
func TestLsRemote(t *testing.T) {
|
||||
state := Setup(t,
|
||||
WithFile("sj://jeff/deep/aaa/bbb/1"),
|
||||
WithFile("sj://jeff/deep/aaa/bbb/2"),
|
||||
WithFile("sj://jeff/deep/aaa/bbb/3"),
|
||||
WithFile("sj://jeff/foobar"),
|
||||
WithFile("sj://jeff/foobar/"),
|
||||
WithFile("sj://jeff/foobar/1"),
|
||||
WithFile("sj://jeff/foobar/2"),
|
||||
WithFile("sj://jeff/foobar/3"),
|
||||
WithFile("sj://jeff/foobaz/1"),
|
||||
state := ultest.Setup(commands,
|
||||
ultest.WithFile("sj://user/deep/aaa/bbb/1"),
|
||||
ultest.WithFile("sj://user/deep/aaa/bbb/2"),
|
||||
ultest.WithFile("sj://user/deep/aaa/bbb/3"),
|
||||
ultest.WithFile("sj://user/foobar"),
|
||||
ultest.WithFile("sj://user/foobar/"),
|
||||
ultest.WithFile("sj://user/foobar/1"),
|
||||
ultest.WithFile("sj://user/foobar/2"),
|
||||
ultest.WithFile("sj://user/foobar/3"),
|
||||
ultest.WithFile("sj://user/foobaz/1"),
|
||||
|
||||
WithPendingFile("sj://jeff/invisible"),
|
||||
ultest.WithPendingFile("sj://user/invisible"),
|
||||
)
|
||||
|
||||
t.Run("Recursive", func(t *testing.T) {
|
||||
state.Succeed(t, "ls", "sj://jeff", "--recursive", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user", "--recursive", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:01 0 deep/aaa/bbb/1
|
||||
OBJ 1970-01-01 00:00:02 0 deep/aaa/bbb/2
|
||||
@ -45,7 +47,7 @@ func TestLsRemote(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("Basic", func(t *testing.T) {
|
||||
state.Succeed(t, "ls", "sj://jeff/fo", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/fo", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:04 0 foobar
|
||||
PRE foobar/
|
||||
@ -54,7 +56,7 @@ func TestLsRemote(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("ExactPrefix", func(t *testing.T) {
|
||||
state.Succeed(t, "ls", "sj://jeff/foobar", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/foobar", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:04 0 foobar
|
||||
PRE foobar/
|
||||
@ -62,7 +64,7 @@ func TestLsRemote(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("ExactPrefixWithSlash", func(t *testing.T) {
|
||||
state.Succeed(t, "ls", "sj://jeff/foobar/", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/foobar/", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:05 0
|
||||
OBJ 1970-01-01 00:00:06 0 1
|
||||
@ -72,17 +74,17 @@ func TestLsRemote(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("MultipleLayers", func(t *testing.T) {
|
||||
state.Succeed(t, "ls", "sj://jeff/deep/").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/deep/").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
PRE aaa/
|
||||
`)
|
||||
|
||||
state.Succeed(t, "ls", "sj://jeff/deep/aaa/").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/deep/aaa/").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
PRE bbb/
|
||||
`)
|
||||
|
||||
state.Succeed(t, "ls", "sj://jeff/deep/aaa/bbb/", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/deep/aaa/bbb/", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:01 0 1
|
||||
OBJ 1970-01-01 00:00:02 0 2
|
||||
@ -92,22 +94,22 @@ func TestLsRemote(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestLsPending(t *testing.T) {
|
||||
state := Setup(t,
|
||||
WithPendingFile("sj://jeff/deep/aaa/bbb/1"),
|
||||
WithPendingFile("sj://jeff/deep/aaa/bbb/2"),
|
||||
WithPendingFile("sj://jeff/deep/aaa/bbb/3"),
|
||||
WithPendingFile("sj://jeff/foobar"),
|
||||
WithPendingFile("sj://jeff/foobar/"),
|
||||
WithPendingFile("sj://jeff/foobar/1"),
|
||||
WithPendingFile("sj://jeff/foobar/2"),
|
||||
WithPendingFile("sj://jeff/foobar/3"),
|
||||
WithPendingFile("sj://jeff/foobaz/1"),
|
||||
state := ultest.Setup(commands,
|
||||
ultest.WithPendingFile("sj://user/deep/aaa/bbb/1"),
|
||||
ultest.WithPendingFile("sj://user/deep/aaa/bbb/2"),
|
||||
ultest.WithPendingFile("sj://user/deep/aaa/bbb/3"),
|
||||
ultest.WithPendingFile("sj://user/foobar"),
|
||||
ultest.WithPendingFile("sj://user/foobar/"),
|
||||
ultest.WithPendingFile("sj://user/foobar/1"),
|
||||
ultest.WithPendingFile("sj://user/foobar/2"),
|
||||
ultest.WithPendingFile("sj://user/foobar/3"),
|
||||
ultest.WithPendingFile("sj://user/foobaz/1"),
|
||||
|
||||
WithFile("sj://jeff/invisible"),
|
||||
ultest.WithFile("sj://user/invisible"),
|
||||
)
|
||||
|
||||
t.Run("Recursive", func(t *testing.T) {
|
||||
state.Succeed(t, "ls", "sj://jeff", "--recursive", "--pending", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user", "--recursive", "--pending", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:01 0 deep/aaa/bbb/1
|
||||
OBJ 1970-01-01 00:00:02 0 deep/aaa/bbb/2
|
||||
@ -122,7 +124,7 @@ func TestLsPending(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("Basic", func(t *testing.T) {
|
||||
state.Succeed(t, "ls", "sj://jeff/fo", "--pending", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/fo", "--pending", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:04 0 foobar
|
||||
PRE foobar/
|
||||
@ -131,7 +133,7 @@ func TestLsPending(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("ExactPrefix", func(t *testing.T) {
|
||||
state.Succeed(t, "ls", "sj://jeff/foobar", "--pending", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/foobar", "--pending", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:04 0 foobar
|
||||
PRE foobar/
|
||||
@ -139,7 +141,7 @@ func TestLsPending(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("ExactPrefixWithSlash", func(t *testing.T) {
|
||||
state.Succeed(t, "ls", "sj://jeff/foobar/", "--pending", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/foobar/", "--pending", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:05 0
|
||||
OBJ 1970-01-01 00:00:06 0 1
|
||||
@ -149,17 +151,17 @@ func TestLsPending(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("MultipleLayers", func(t *testing.T) {
|
||||
state.Succeed(t, "ls", "sj://jeff/deep/", "--pending").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/deep/", "--pending").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
PRE aaa/
|
||||
`)
|
||||
|
||||
state.Succeed(t, "ls", "sj://jeff/deep/aaa/", "--pending").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/deep/aaa/", "--pending").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
PRE bbb/
|
||||
`)
|
||||
|
||||
state.Succeed(t, "ls", "sj://jeff/deep/aaa/bbb/", "--pending", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/deep/aaa/bbb/", "--pending", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:01 0 1
|
||||
OBJ 1970-01-01 00:00:02 0 2
|
||||
@ -169,24 +171,24 @@ func TestLsPending(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestLsDifficult(t *testing.T) {
|
||||
state := Setup(t,
|
||||
WithFile("sj://jeff//"),
|
||||
WithFile("sj://jeff///"),
|
||||
WithFile("sj://jeff////"),
|
||||
state := ultest.Setup(commands,
|
||||
ultest.WithFile("sj://user//"),
|
||||
ultest.WithFile("sj://user///"),
|
||||
ultest.WithFile("sj://user////"),
|
||||
|
||||
WithFile("sj://jeff//starts-slash"),
|
||||
ultest.WithFile("sj://user//starts-slash"),
|
||||
|
||||
WithFile("sj://jeff/ends-slash"),
|
||||
WithFile("sj://jeff/ends-slash/"),
|
||||
WithFile("sj://jeff/ends-slash//"),
|
||||
ultest.WithFile("sj://user/ends-slash"),
|
||||
ultest.WithFile("sj://user/ends-slash/"),
|
||||
ultest.WithFile("sj://user/ends-slash//"),
|
||||
|
||||
WithFile("sj://jeff/mid-slash"),
|
||||
WithFile("sj://jeff/mid-slash//2"),
|
||||
WithFile("sj://jeff/mid-slash/1"),
|
||||
ultest.WithFile("sj://user/mid-slash"),
|
||||
ultest.WithFile("sj://user/mid-slash//2"),
|
||||
ultest.WithFile("sj://user/mid-slash/1"),
|
||||
)
|
||||
|
||||
t.Run("Recursive", func(t *testing.T) {
|
||||
state.Succeed(t, "ls", "sj://jeff", "--recursive", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user", "--recursive", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:01 0 /
|
||||
OBJ 1970-01-01 00:00:02 0 //
|
||||
@ -202,7 +204,7 @@ func TestLsDifficult(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("Basic", func(t *testing.T) {
|
||||
state.Succeed(t, "ls", "sj://jeff", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
PRE /
|
||||
OBJ 1970-01-01 00:00:05 0 ends-slash
|
||||
@ -211,7 +213,7 @@ func TestLsDifficult(t *testing.T) {
|
||||
PRE mid-slash/
|
||||
`)
|
||||
|
||||
state.Succeed(t, "ls", "sj://jeff/", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
PRE /
|
||||
OBJ 1970-01-01 00:00:05 0 ends-slash
|
||||
@ -222,58 +224,58 @@ func TestLsDifficult(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("OnlySlash", func(t *testing.T) {
|
||||
state.Succeed(t, "ls", "sj://jeff//", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user//", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:01 0
|
||||
PRE /
|
||||
OBJ 1970-01-01 00:00:04 0 starts-slash
|
||||
`)
|
||||
|
||||
state.Succeed(t, "ls", "sj://jeff///", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user///", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:02 0
|
||||
PRE /
|
||||
`)
|
||||
|
||||
state.Succeed(t, "ls", "sj://jeff////", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user////", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:03 0
|
||||
`)
|
||||
})
|
||||
|
||||
t.Run("EndsSlash", func(t *testing.T) {
|
||||
state.Succeed(t, "ls", "sj://jeff/ends-slash", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/ends-slash", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:05 0 ends-slash
|
||||
PRE ends-slash/
|
||||
`)
|
||||
|
||||
state.Succeed(t, "ls", "sj://jeff/ends-slash/", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/ends-slash/", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:06 0
|
||||
PRE /
|
||||
`)
|
||||
|
||||
state.Succeed(t, "ls", "sj://jeff/ends-slash//", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/ends-slash//", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:07 0
|
||||
`)
|
||||
})
|
||||
|
||||
t.Run("MidSlash", func(t *testing.T) {
|
||||
state.Succeed(t, "ls", "sj://jeff/mid-slash", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/mid-slash", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:08 0 mid-slash
|
||||
PRE mid-slash/
|
||||
`)
|
||||
|
||||
state.Succeed(t, "ls", "sj://jeff/mid-slash/", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/mid-slash/", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
PRE /
|
||||
OBJ 1970-01-01 00:00:10 0 1
|
||||
`)
|
||||
|
||||
state.Succeed(t, "ls", "sj://jeff/mid-slash//", "--utc").RequireStdout(t, `
|
||||
state.Succeed(t, "ls", "sj://user/mid-slash//", "--utc").RequireStdout(t, `
|
||||
KIND CREATED SIZE KEY
|
||||
OBJ 1970-01-01 00:00:09 0 2
|
||||
`)
|
||||
|
||||
@ -5,12 +5,14 @@ package main
|
||||
|
||||
import (
|
||||
"github.com/zeebo/clingy"
|
||||
|
||||
"storj.io/storj/cmd/uplinkng/ulloc"
|
||||
)
|
||||
|
||||
type cmdMetaGet struct {
|
||||
projectProvider
|
||||
|
||||
location Location
|
||||
location ulloc.Location
|
||||
entry *string
|
||||
}
|
||||
|
||||
@ -18,8 +20,8 @@ func (c *cmdMetaGet) Setup(a clingy.Arguments, f clingy.Flags) {
|
||||
c.projectProvider.Setup(a, f)
|
||||
|
||||
c.location = a.New("location", "Location of object (sj://BUCKET/KEY)",
|
||||
clingy.Transform(parseLocation),
|
||||
).(Location)
|
||||
clingy.Transform(ulloc.Parse),
|
||||
).(ulloc.Location)
|
||||
c.entry = a.New("entry", "Metadata entry to get", clingy.Optional).(*string)
|
||||
}
|
||||
|
||||
|
||||
@ -7,6 +7,8 @@ import (
|
||||
"strconv"
|
||||
|
||||
"github.com/zeebo/clingy"
|
||||
|
||||
"storj.io/storj/cmd/uplinkng/ulloc"
|
||||
)
|
||||
|
||||
type cmdRm struct {
|
||||
@ -15,7 +17,7 @@ type cmdRm struct {
|
||||
recursive bool
|
||||
encrypted bool
|
||||
|
||||
location Location
|
||||
location ulloc.Location
|
||||
}
|
||||
|
||||
func (c *cmdRm) Setup(a clingy.Arguments, f clingy.Flags) {
|
||||
@ -30,8 +32,8 @@ func (c *cmdRm) Setup(a clingy.Arguments, f clingy.Flags) {
|
||||
).(bool)
|
||||
|
||||
c.location = a.New("location", "Location to remove (sj://BUCKET[/KEY])",
|
||||
clingy.Transform(parseLocation),
|
||||
).(Location)
|
||||
clingy.Transform(ulloc.Parse),
|
||||
).(ulloc.Location)
|
||||
}
|
||||
|
||||
func (c *cmdRm) Execute(ctx clingy.Context) error {
|
||||
|
||||
@ -1,62 +0,0 @@
|
||||
// Copyright (C) 2021 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/zeebo/clingy"
|
||||
)
|
||||
|
||||
//
|
||||
// filesystemMixed dispatches to either the local or remote filesystem depending on the path
|
||||
//
|
||||
|
||||
type filesystemMixed struct {
|
||||
local *filesystemLocal
|
||||
remote *filesystemRemote
|
||||
}
|
||||
|
||||
func (m *filesystemMixed) Close() error {
|
||||
return m.remote.Close()
|
||||
}
|
||||
|
||||
func (m *filesystemMixed) Open(ctx clingy.Context, loc Location) (readHandle, error) {
|
||||
if loc.Remote() {
|
||||
return m.remote.Open(ctx, loc.bucket, loc.key)
|
||||
} else if loc.Std() {
|
||||
return newGenericReadHandle(ctx.Stdin()), nil
|
||||
}
|
||||
return m.local.Open(ctx, loc.path)
|
||||
}
|
||||
|
||||
func (m *filesystemMixed) Create(ctx clingy.Context, loc Location) (writeHandle, error) {
|
||||
if loc.Remote() {
|
||||
return m.remote.Create(ctx, loc.bucket, loc.key)
|
||||
} else if loc.Std() {
|
||||
return newGenericWriteHandle(ctx.Stdout()), nil
|
||||
}
|
||||
return m.local.Create(ctx, loc.path)
|
||||
}
|
||||
|
||||
func (m *filesystemMixed) ListObjects(ctx context.Context, prefix Location, recursive bool) (objectIterator, error) {
|
||||
if prefix.Remote() {
|
||||
return m.remote.ListObjects(ctx, prefix.bucket, prefix.key, recursive), nil
|
||||
}
|
||||
return m.local.ListObjects(ctx, prefix.path, recursive)
|
||||
}
|
||||
|
||||
func (m *filesystemMixed) ListUploads(ctx context.Context, prefix Location, recursive bool) (objectIterator, error) {
|
||||
if prefix.Remote() {
|
||||
return m.remote.ListPendingMultiparts(ctx, prefix.bucket, prefix.key, recursive), nil
|
||||
}
|
||||
return emptyObjectIterator{}, nil
|
||||
}
|
||||
|
||||
func (m *filesystemMixed) IsLocalDir(ctx context.Context, loc Location) bool {
|
||||
if !loc.Local() {
|
||||
return false
|
||||
}
|
||||
return m.local.IsLocalDir(ctx, loc.path)
|
||||
}
|
||||
@ -1,476 +0,0 @@
|
||||
// Copyright (C) 2021 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/zeebo/clingy"
|
||||
"github.com/zeebo/errs"
|
||||
)
|
||||
|
||||
//
|
||||
// helpers to execute commands for tests
|
||||
//
|
||||
|
||||
func Setup(t *testing.T, opts ...ExecuteOption) State {
|
||||
return State{
|
||||
opts: opts,
|
||||
}
|
||||
}
|
||||
|
||||
type State struct {
|
||||
opts []ExecuteOption
|
||||
}
|
||||
|
||||
// Succeed is the same as Run followed by result.RequireSuccess.
|
||||
func (st State) Succeed(t *testing.T, args ...string) Result {
|
||||
result := st.Run(t, args...)
|
||||
result.RequireSuccess(t)
|
||||
return result
|
||||
}
|
||||
|
||||
// Fail is the same as Run followed by result.RequireFailure.
|
||||
func (st State) Fail(t *testing.T, args ...string) Result {
|
||||
result := st.Run(t, args...)
|
||||
result.RequireFailure(t)
|
||||
return result
|
||||
}
|
||||
|
||||
func (st State) Run(t *testing.T, args ...string) Result {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
var stdin bytes.Buffer
|
||||
var ops []Operation
|
||||
var ran bool
|
||||
|
||||
ok, err := clingy.Environment{
|
||||
Name: "uplink-test",
|
||||
Args: args,
|
||||
|
||||
Stdin: &stdin,
|
||||
Stdout: &stdout,
|
||||
Stderr: &stderr,
|
||||
|
||||
Wrap: func(ctx clingy.Context, cmd clingy.Cmd) error {
|
||||
tfs := newTestFilesystem()
|
||||
for _, opt := range st.opts {
|
||||
if err := opt(ctx, tfs); err != nil {
|
||||
return errs.Wrap(err)
|
||||
}
|
||||
}
|
||||
tfs.ops = nil
|
||||
|
||||
if len(tfs.stdin) > 0 {
|
||||
_, _ = stdin.WriteString(tfs.stdin)
|
||||
}
|
||||
|
||||
if setter, ok := cmd.(interface {
|
||||
setTestFilesystem(filesystem)
|
||||
}); ok {
|
||||
setter.setTestFilesystem(tfs)
|
||||
}
|
||||
|
||||
ran = true
|
||||
err := cmd.Execute(ctx)
|
||||
ops = tfs.ops
|
||||
return err
|
||||
},
|
||||
}.Run(context.Background(), commands)
|
||||
|
||||
if ok && err == nil {
|
||||
require.True(t, ran, "no command was executed: %q", args)
|
||||
}
|
||||
return Result{
|
||||
Stdout: stdout.String(),
|
||||
Stderr: stderr.String(),
|
||||
Ok: ok,
|
||||
Err: err,
|
||||
Operations: ops,
|
||||
}
|
||||
}
|
||||
|
||||
type ExecuteOption func(ctx clingy.Context, tfs *testFilesystem) error
|
||||
|
||||
func WithFile(location string) ExecuteOption {
|
||||
return func(ctx clingy.Context, tfs *testFilesystem) error {
|
||||
loc, err := parseLocation(location)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if loc.Remote() {
|
||||
tfs.ensureBucket(loc.bucket)
|
||||
}
|
||||
wh, err := tfs.Create(ctx, loc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return wh.Commit()
|
||||
}
|
||||
}
|
||||
|
||||
func WithPendingFile(location string) ExecuteOption {
|
||||
return func(ctx clingy.Context, tfs *testFilesystem) error {
|
||||
loc, err := parseLocation(location)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if loc.Remote() {
|
||||
tfs.ensureBucket(loc.bucket)
|
||||
}
|
||||
_, err = tfs.Create(ctx, loc)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// execution results
|
||||
//
|
||||
|
||||
type Result struct {
|
||||
Stdout string
|
||||
Stderr string
|
||||
Ok bool
|
||||
Err error
|
||||
Operations []Operation
|
||||
}
|
||||
|
||||
func (r Result) RequireSuccess(t *testing.T) {
|
||||
if !r.Ok {
|
||||
errs := parseErrors(r.Stdout)
|
||||
require.True(t, r.Ok, "test did not run successfully. errors:\n%s",
|
||||
strings.Join(errs, "\n"))
|
||||
}
|
||||
require.NoError(t, r.Err)
|
||||
}
|
||||
|
||||
func (r Result) RequireFailure(t *testing.T) {
|
||||
require.False(t, r.Ok && r.Err == nil, "command ran with no error")
|
||||
}
|
||||
|
||||
func (r Result) RequireStdout(t *testing.T, stdout string) {
|
||||
require.Equal(t, trimNewlineSpaces(stdout), trimNewlineSpaces(r.Stdout))
|
||||
}
|
||||
|
||||
func (r Result) RequireStderr(t *testing.T, stderr string) {
|
||||
require.Equal(t, trimNewlineSpaces(stderr), trimNewlineSpaces(r.Stderr))
|
||||
}
|
||||
|
||||
func parseErrors(s string) []string {
|
||||
lines := strings.Split(s, "\n")
|
||||
start := 0
|
||||
for i, line := range lines {
|
||||
if line == "Errors:" {
|
||||
start = i + 1
|
||||
} else if len(line) > 0 && line[0] != ' ' {
|
||||
return lines[start:i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func trimNewlineSpaces(s string) string {
|
||||
lines := strings.Split(s, "\n")
|
||||
|
||||
j := 0
|
||||
for _, line := range lines {
|
||||
if trimmed := strings.TrimSpace(line); len(trimmed) > 0 {
|
||||
lines[j] = trimmed
|
||||
j++
|
||||
}
|
||||
}
|
||||
return strings.Join(lines[:j], "\n")
|
||||
}
|
||||
|
||||
type Operation struct {
|
||||
Kind string
|
||||
Loc string
|
||||
Error bool
|
||||
}
|
||||
|
||||
func newOp(kind string, loc Location, err error) Operation {
|
||||
return Operation{
|
||||
Kind: kind,
|
||||
Loc: loc.String(),
|
||||
Error: err != nil,
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// filesystem
|
||||
//
|
||||
|
||||
type testFilesystem struct {
|
||||
stdin string
|
||||
ops []Operation
|
||||
created int64
|
||||
files map[Location]byteFileData
|
||||
pending map[Location][]*byteWriteHandle
|
||||
buckets map[string]struct{}
|
||||
}
|
||||
|
||||
func newTestFilesystem() *testFilesystem {
|
||||
return &testFilesystem{
|
||||
files: make(map[Location]byteFileData),
|
||||
pending: make(map[Location][]*byteWriteHandle),
|
||||
buckets: make(map[string]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
type byteFileData struct {
|
||||
data []byte
|
||||
created int64
|
||||
}
|
||||
|
||||
func (tfs *testFilesystem) ensureBucket(name string) {
|
||||
tfs.buckets[name] = struct{}{}
|
||||
}
|
||||
|
||||
func (tfs *testFilesystem) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tfs *testFilesystem) Open(ctx clingy.Context, loc Location) (_ readHandle, err error) {
|
||||
defer func() { tfs.ops = append(tfs.ops, newOp("open", loc, err)) }()
|
||||
|
||||
bf, ok := tfs.files[loc]
|
||||
if !ok {
|
||||
return nil, errs.New("file does not exist")
|
||||
}
|
||||
return &byteReadHandle{Buffer: bytes.NewBuffer(bf.data)}, nil
|
||||
}
|
||||
|
||||
func (tfs *testFilesystem) Create(ctx clingy.Context, loc Location) (_ writeHandle, err error) {
|
||||
defer func() { tfs.ops = append(tfs.ops, newOp("create", loc, err)) }()
|
||||
|
||||
if loc.Remote() {
|
||||
if _, ok := tfs.buckets[loc.bucket]; !ok {
|
||||
return nil, errs.New("bucket %q does not exist", loc.bucket)
|
||||
}
|
||||
}
|
||||
|
||||
tfs.created++
|
||||
wh := &byteWriteHandle{
|
||||
buf: bytes.NewBuffer(nil),
|
||||
loc: loc,
|
||||
tfs: tfs,
|
||||
cre: tfs.created,
|
||||
}
|
||||
|
||||
tfs.pending[loc] = append(tfs.pending[loc], wh)
|
||||
|
||||
return wh, nil
|
||||
}
|
||||
|
||||
func (tfs *testFilesystem) ListObjects(ctx context.Context, prefix Location, recursive bool) (objectIterator, error) {
|
||||
var infos []objectInfo
|
||||
for loc, bf := range tfs.files {
|
||||
if loc.HasPrefix(prefix) {
|
||||
infos = append(infos, objectInfo{
|
||||
Loc: loc,
|
||||
Created: time.Unix(bf.created, 0),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
sort.Sort(objectInfos(infos))
|
||||
|
||||
if !recursive {
|
||||
infos = collapseObjectInfos(prefix, infos)
|
||||
}
|
||||
|
||||
return &objectInfoIterator{infos: infos}, nil
|
||||
}
|
||||
|
||||
func (tfs *testFilesystem) ListUploads(ctx context.Context, prefix Location, recursive bool) (objectIterator, error) {
|
||||
var infos []objectInfo
|
||||
for loc, whs := range tfs.pending {
|
||||
if loc.HasPrefix(prefix) {
|
||||
for _, wh := range whs {
|
||||
infos = append(infos, objectInfo{
|
||||
Loc: loc,
|
||||
Created: time.Unix(wh.cre, 0),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort.Sort(objectInfos(infos))
|
||||
|
||||
if !recursive {
|
||||
infos = collapseObjectInfos(prefix, infos)
|
||||
}
|
||||
|
||||
return &objectInfoIterator{infos: infos}, nil
|
||||
}
|
||||
|
||||
func (tfs *testFilesystem) IsLocalDir(ctx context.Context, loc Location) bool {
|
||||
// TODO: implement this
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
//
|
||||
// readHandle
|
||||
//
|
||||
|
||||
type byteReadHandle struct {
|
||||
*bytes.Buffer
|
||||
}
|
||||
|
||||
func (b *byteReadHandle) Close() error { return nil }
|
||||
func (b *byteReadHandle) Info() objectInfo { return objectInfo{} }
|
||||
|
||||
//
|
||||
// writeHandle
|
||||
//
|
||||
|
||||
type byteWriteHandle struct {
|
||||
buf *bytes.Buffer
|
||||
loc Location
|
||||
tfs *testFilesystem
|
||||
cre int64
|
||||
done bool
|
||||
}
|
||||
|
||||
func (b *byteWriteHandle) Write(p []byte) (int, error) {
|
||||
return b.buf.Write(p)
|
||||
}
|
||||
|
||||
func (b *byteWriteHandle) Commit() error {
|
||||
if err := b.close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b.tfs.ops = append(b.tfs.ops, newOp("commit", b.loc, nil))
|
||||
b.tfs.files[b.loc] = byteFileData{
|
||||
data: b.buf.Bytes(),
|
||||
created: b.cre,
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *byteWriteHandle) Abort() error {
|
||||
if err := b.close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b.tfs.ops = append(b.tfs.ops, newOp("append", b.loc, nil))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *byteWriteHandle) close() error {
|
||||
if b.done {
|
||||
return errs.New("already done")
|
||||
}
|
||||
b.done = true
|
||||
|
||||
handles := b.tfs.pending[b.loc]
|
||||
for i, v := range handles {
|
||||
if v == b {
|
||||
handles = append(handles[:i], handles[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if len(handles) > 0 {
|
||||
b.tfs.pending[b.loc] = handles
|
||||
} else {
|
||||
delete(b.tfs.pending, b.loc)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
//
|
||||
// objectIterator
|
||||
//
|
||||
|
||||
type objectInfoIterator struct {
|
||||
infos []objectInfo
|
||||
current objectInfo
|
||||
}
|
||||
|
||||
func (li *objectInfoIterator) Next() bool {
|
||||
if len(li.infos) == 0 {
|
||||
return false
|
||||
}
|
||||
li.current, li.infos = li.infos[0], li.infos[1:]
|
||||
return true
|
||||
}
|
||||
|
||||
func (li *objectInfoIterator) Err() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (li *objectInfoIterator) Item() objectInfo {
|
||||
return li.current
|
||||
}
|
||||
|
||||
type objectInfos []objectInfo
|
||||
|
||||
func (ois objectInfos) Len() int { return len(ois) }
|
||||
func (ois objectInfos) Swap(i int, j int) { ois[i], ois[j] = ois[j], ois[i] }
|
||||
func (ois objectInfos) Less(i int, j int) bool {
|
||||
li, lj := ois[i].Loc, ois[j].Loc
|
||||
|
||||
if !li.remote && lj.remote {
|
||||
return true
|
||||
} else if !lj.remote && li.remote {
|
||||
return false
|
||||
}
|
||||
|
||||
if li.bucket < lj.bucket {
|
||||
return true
|
||||
} else if lj.bucket < li.bucket {
|
||||
return false
|
||||
}
|
||||
|
||||
if li.key < lj.key {
|
||||
return true
|
||||
} else if lj.key < li.key {
|
||||
return false
|
||||
}
|
||||
|
||||
if li.path < lj.path {
|
||||
return true
|
||||
} else if lj.path < li.path {
|
||||
return false
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func collapseObjectInfos(prefix Location, infos []objectInfo) []objectInfo {
|
||||
collapsing := false
|
||||
current := ""
|
||||
j := 0
|
||||
|
||||
for _, oi := range infos {
|
||||
first, ok := oi.Loc.ListKeyName(prefix)
|
||||
if ok {
|
||||
if collapsing && first == current {
|
||||
continue
|
||||
}
|
||||
|
||||
collapsing = true
|
||||
current = first
|
||||
|
||||
oi.IsPrefix = true
|
||||
}
|
||||
|
||||
oi.Loc = oi.Loc.SetKey(first)
|
||||
|
||||
infos[j] = oi
|
||||
j++
|
||||
}
|
||||
|
||||
return infos[:j]
|
||||
}
|
||||
@ -8,6 +8,7 @@ import (
|
||||
|
||||
"github.com/zeebo/clingy"
|
||||
|
||||
"storj.io/storj/cmd/uplinkng/ulfs"
|
||||
"storj.io/uplink"
|
||||
privateAccess "storj.io/uplink/private/access"
|
||||
)
|
||||
@ -16,16 +17,16 @@ type projectProvider struct {
|
||||
access string
|
||||
|
||||
testProject *uplink.Project
|
||||
testFilesystem filesystem
|
||||
testFilesystem ulfs.Filesystem
|
||||
}
|
||||
|
||||
func (pp *projectProvider) Setup(a clingy.Arguments, f clingy.Flags) {
|
||||
pp.access = f.New("access", "Which access to use", "").(string)
|
||||
}
|
||||
|
||||
func (pp *projectProvider) setTestFilesystem(fs filesystem) { pp.testFilesystem = fs }
|
||||
func (pp *projectProvider) SetTestFilesystem(fs ulfs.Filesystem) { pp.testFilesystem = fs }
|
||||
|
||||
func (pp *projectProvider) OpenFilesystem(ctx context.Context, options ...projectOption) (filesystem, error) {
|
||||
func (pp *projectProvider) OpenFilesystem(ctx context.Context, options ...projectOption) (ulfs.Filesystem, error) {
|
||||
if pp.testFilesystem != nil {
|
||||
return pp.testFilesystem, nil
|
||||
}
|
||||
@ -34,12 +35,7 @@ func (pp *projectProvider) OpenFilesystem(ctx context.Context, options ...projec
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &filesystemMixed{
|
||||
local: &filesystemLocal{},
|
||||
remote: &filesystemRemote{
|
||||
project: project,
|
||||
},
|
||||
}, nil
|
||||
return ulfs.NewMixed(ulfs.NewLocal(), ulfs.NewRemote(project)), nil
|
||||
}
|
||||
|
||||
func (pp *projectProvider) OpenProject(ctx context.Context, options ...projectOption) (*uplink.Project, error) {
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
// Copyright (C) 2021 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package main
|
||||
package ulfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
@ -13,40 +13,37 @@ import (
|
||||
"github.com/zeebo/clingy"
|
||||
"github.com/zeebo/errs"
|
||||
|
||||
"storj.io/storj/cmd/uplinkng/ulloc"
|
||||
"storj.io/uplink"
|
||||
)
|
||||
|
||||
// filesystem represents either the local filesystem or the data backed by a project.
|
||||
type filesystem interface {
|
||||
// Filesystem represents either the local Filesystem or the data backed by a project.
|
||||
type Filesystem interface {
|
||||
Close() error
|
||||
Open(ctx clingy.Context, loc Location) (readHandle, error)
|
||||
Create(ctx clingy.Context, loc Location) (writeHandle, error)
|
||||
ListObjects(ctx context.Context, prefix Location, recursive bool) (objectIterator, error)
|
||||
ListUploads(ctx context.Context, prefix Location, recursive bool) (objectIterator, error)
|
||||
IsLocalDir(ctx context.Context, loc Location) bool
|
||||
Open(ctx clingy.Context, loc ulloc.Location) (ReadHandle, error)
|
||||
Create(ctx clingy.Context, loc ulloc.Location) (WriteHandle, error)
|
||||
ListObjects(ctx context.Context, prefix ulloc.Location, recursive bool) (ObjectIterator, error)
|
||||
ListUploads(ctx context.Context, prefix ulloc.Location, recursive bool) (ObjectIterator, error)
|
||||
IsLocalDir(ctx context.Context, loc ulloc.Location) bool
|
||||
}
|
||||
|
||||
//
|
||||
// object info
|
||||
//
|
||||
|
||||
// objectInfo is a simpler *uplink.Object that contains the minimal information the
|
||||
// ObjectInfo is a simpler *uplink.Object that contains the minimal information the
|
||||
// uplink command needs that multiple types can be converted to.
|
||||
type objectInfo struct {
|
||||
Loc Location
|
||||
type ObjectInfo struct {
|
||||
Loc ulloc.Location
|
||||
IsPrefix bool
|
||||
Created time.Time
|
||||
ContentLength int64
|
||||
}
|
||||
|
||||
// uplinkObjectToObjectInfo returns an objectInfo converted from an *uplink.Object.
|
||||
func uplinkObjectToObjectInfo(bucket string, obj *uplink.Object) objectInfo {
|
||||
return objectInfo{
|
||||
Loc: Location{
|
||||
bucket: bucket,
|
||||
key: obj.Key,
|
||||
remote: true,
|
||||
},
|
||||
func uplinkObjectToObjectInfo(bucket string, obj *uplink.Object) ObjectInfo {
|
||||
return ObjectInfo{
|
||||
Loc: ulloc.NewRemote(bucket, obj.Key),
|
||||
IsPrefix: obj.IsPrefix,
|
||||
Created: obj.System.Created,
|
||||
ContentLength: obj.System.ContentLength,
|
||||
@ -54,13 +51,9 @@ func uplinkObjectToObjectInfo(bucket string, obj *uplink.Object) objectInfo {
|
||||
}
|
||||
|
||||
// uplinkUploadInfoToObjectInfo returns an objectInfo converted from an *uplink.Object.
|
||||
func uplinkUploadInfoToObjectInfo(bucket string, upl *uplink.UploadInfo) objectInfo {
|
||||
return objectInfo{
|
||||
Loc: Location{
|
||||
bucket: bucket,
|
||||
key: upl.Key,
|
||||
remote: true,
|
||||
},
|
||||
func uplinkUploadInfoToObjectInfo(bucket string, upl *uplink.UploadInfo) ObjectInfo {
|
||||
return ObjectInfo{
|
||||
Loc: ulloc.NewRemote(bucket, upl.Key),
|
||||
IsPrefix: upl.IsPrefix,
|
||||
Created: upl.System.Created,
|
||||
ContentLength: upl.System.ContentLength,
|
||||
@ -71,11 +64,11 @@ func uplinkUploadInfoToObjectInfo(bucket string, upl *uplink.UploadInfo) objectI
|
||||
// read handles
|
||||
//
|
||||
|
||||
// readHandle is something that can be read from.
|
||||
type readHandle interface {
|
||||
// ReadHandle is something that can be read from.
|
||||
type ReadHandle interface {
|
||||
io.Reader
|
||||
io.Closer
|
||||
Info() objectInfo
|
||||
Info() ObjectInfo
|
||||
}
|
||||
|
||||
// uplinkReadHandle implements readHandle for *uplink.Downloads.
|
||||
@ -94,12 +87,12 @@ func newUplinkReadHandle(bucket string, dl *uplink.Download) *uplinkReadHandle {
|
||||
|
||||
func (u *uplinkReadHandle) Read(p []byte) (int, error) { return u.dl.Read(p) }
|
||||
func (u *uplinkReadHandle) Close() error { return u.dl.Close() }
|
||||
func (u *uplinkReadHandle) Info() objectInfo { return uplinkObjectToObjectInfo(u.bucket, u.dl.Info()) }
|
||||
func (u *uplinkReadHandle) Info() ObjectInfo { return uplinkObjectToObjectInfo(u.bucket, u.dl.Info()) }
|
||||
|
||||
// osReadHandle implements readHandle for *os.Files.
|
||||
type osReadHandle struct {
|
||||
raw *os.File
|
||||
info objectInfo
|
||||
info ObjectInfo
|
||||
}
|
||||
|
||||
// newOsReadHandle constructs an *osReadHandle from an *os.File.
|
||||
@ -110,8 +103,8 @@ func newOSReadHandle(fh *os.File) (*osReadHandle, error) {
|
||||
}
|
||||
return &osReadHandle{
|
||||
raw: fh,
|
||||
info: objectInfo{
|
||||
Loc: Location{path: fh.Name()},
|
||||
info: ObjectInfo{
|
||||
Loc: ulloc.NewLocal(fh.Name()),
|
||||
IsPrefix: false,
|
||||
Created: fi.ModTime(), // TODO: os specific crtime
|
||||
ContentLength: fi.Size(),
|
||||
@ -121,7 +114,7 @@ func newOSReadHandle(fh *os.File) (*osReadHandle, error) {
|
||||
|
||||
func (o *osReadHandle) Read(p []byte) (int, error) { return o.raw.Read(p) }
|
||||
func (o *osReadHandle) Close() error { return o.raw.Close() }
|
||||
func (o *osReadHandle) Info() objectInfo { return o.info }
|
||||
func (o *osReadHandle) Info() ObjectInfo { return o.info }
|
||||
|
||||
// genericReadHandle implements readHandle for an io.Reader.
|
||||
type genericReadHandle struct{ r io.Reader }
|
||||
@ -133,14 +126,14 @@ func newGenericReadHandle(r io.Reader) *genericReadHandle {
|
||||
|
||||
func (g *genericReadHandle) Read(p []byte) (int, error) { return g.r.Read(p) }
|
||||
func (g *genericReadHandle) Close() error { return nil }
|
||||
func (g *genericReadHandle) Info() objectInfo { return objectInfo{ContentLength: -1} }
|
||||
func (g *genericReadHandle) Info() ObjectInfo { return ObjectInfo{ContentLength: -1} }
|
||||
|
||||
//
|
||||
// write handles
|
||||
//
|
||||
|
||||
// writeHandle is anything that can be written to with commit/abort semantics.
|
||||
type writeHandle interface {
|
||||
// WriteHandle is anything that can be written to with commit/abort semantics.
|
||||
type WriteHandle interface {
|
||||
io.Writer
|
||||
Commit() error
|
||||
Abort() error
|
||||
@ -212,11 +205,11 @@ func (g *genericWriteHandle) Abort() error { return nil }
|
||||
// object iteration
|
||||
//
|
||||
|
||||
// objectIterator is an interface type for iterating over objectInfo values.
|
||||
type objectIterator interface {
|
||||
// ObjectIterator is an interface type for iterating over objectInfo values.
|
||||
type ObjectIterator interface {
|
||||
Next() bool
|
||||
Err() error
|
||||
Item() objectInfo
|
||||
Item() ObjectInfo
|
||||
}
|
||||
|
||||
// filteredObjectIterator removes any iteration entries that do not begin with the filter.
|
||||
@ -225,7 +218,7 @@ type objectIterator interface {
|
||||
type filteredObjectIterator struct {
|
||||
trim string
|
||||
filter string
|
||||
iter objectIterator
|
||||
iter ObjectIterator
|
||||
}
|
||||
|
||||
func (f *filteredObjectIterator) Next() bool {
|
||||
@ -245,15 +238,9 @@ func (f *filteredObjectIterator) Next() bool {
|
||||
|
||||
func (f *filteredObjectIterator) Err() error { return f.iter.Err() }
|
||||
|
||||
func (f *filteredObjectIterator) Item() objectInfo {
|
||||
func (f *filteredObjectIterator) Item() ObjectInfo {
|
||||
item := f.iter.Item()
|
||||
path := item.Loc
|
||||
if path.remote {
|
||||
path.key = path.key[len(f.trim):]
|
||||
} else {
|
||||
path.path = path.path[len(f.trim):]
|
||||
}
|
||||
item.Loc = path
|
||||
item.Loc = item.Loc.RemoveKeyPrefix(f.trim)
|
||||
return item
|
||||
}
|
||||
|
||||
@ -262,4 +249,4 @@ type emptyObjectIterator struct{}
|
||||
|
||||
func (emptyObjectIterator) Next() bool { return false }
|
||||
func (emptyObjectIterator) Err() error { return nil }
|
||||
func (emptyObjectIterator) Item() objectInfo { return objectInfo{} }
|
||||
func (emptyObjectIterator) Item() ObjectInfo { return ObjectInfo{} }
|
||||
@ -1,7 +1,7 @@
|
||||
// Copyright (C) 2021 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package main
|
||||
package ulfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
@ -11,12 +11,19 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/zeebo/errs"
|
||||
|
||||
"storj.io/storj/cmd/uplinkng/ulloc"
|
||||
)
|
||||
|
||||
// filesystemLocal implements something close to a filesystem but backed by the local disk.
|
||||
type filesystemLocal struct{}
|
||||
// Local implements something close to a filesystem but backed by the local disk.
|
||||
type Local struct{}
|
||||
|
||||
func (l *filesystemLocal) abs(path string) (string, error) {
|
||||
// NewLocal constructs a Local filesystem.
|
||||
func NewLocal() *Local {
|
||||
return &Local{}
|
||||
}
|
||||
|
||||
func (l *Local) abs(path string) (string, error) {
|
||||
abs, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return "", errs.Wrap(err)
|
||||
@ -28,7 +35,8 @@ func (l *filesystemLocal) abs(path string) (string, error) {
|
||||
return abs, nil
|
||||
}
|
||||
|
||||
func (l *filesystemLocal) Open(ctx context.Context, path string) (readHandle, error) {
|
||||
// Open returns a read ReadHandle for the given local path.
|
||||
func (l *Local) Open(ctx context.Context, path string) (ReadHandle, error) {
|
||||
path, err := l.abs(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -41,7 +49,8 @@ func (l *filesystemLocal) Open(ctx context.Context, path string) (readHandle, er
|
||||
return newOSReadHandle(fh)
|
||||
}
|
||||
|
||||
func (l *filesystemLocal) Create(ctx context.Context, path string) (writeHandle, error) {
|
||||
// Create makes any directories necessary to create a file at path and returns a WriteHandle.
|
||||
func (l *Local) Create(ctx context.Context, path string) (WriteHandle, error) {
|
||||
path, err := l.abs(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -66,7 +75,9 @@ func (l *filesystemLocal) Create(ctx context.Context, path string) (writeHandle,
|
||||
return newOSWriteHandle(fh), nil
|
||||
}
|
||||
|
||||
func (l *filesystemLocal) ListObjects(ctx context.Context, path string, recursive bool) (objectIterator, error) {
|
||||
// ListObjects returns an ObjectIterator listing files and directories that have string prefix
|
||||
// with the provided path.
|
||||
func (l *Local) ListObjects(ctx context.Context, path string, recursive bool) (ObjectIterator, error) {
|
||||
path, err := l.abs(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -110,7 +121,8 @@ func (l *filesystemLocal) ListObjects(ctx context.Context, path string, recursiv
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (l *filesystemLocal) IsLocalDir(ctx context.Context, path string) bool {
|
||||
// IsLocalDir returns true if the path is a directory.
|
||||
func (l *Local) IsLocalDir(ctx context.Context, path string) bool {
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return false
|
||||
@ -141,7 +153,7 @@ func (fi *fileinfoObjectIterator) Next() bool {
|
||||
|
||||
func (fi *fileinfoObjectIterator) Err() error { return nil }
|
||||
|
||||
func (fi *fileinfoObjectIterator) Item() objectInfo {
|
||||
func (fi *fileinfoObjectIterator) Item() ObjectInfo {
|
||||
name := filepath.Join(fi.base, fi.current.Name())
|
||||
isDir := fi.current.IsDir()
|
||||
if isDir {
|
||||
@ -154,8 +166,8 @@ func (fi *fileinfoObjectIterator) Item() objectInfo {
|
||||
name = strings.ReplaceAll(name, string(filepath.Separator), "/")
|
||||
}
|
||||
|
||||
return objectInfo{
|
||||
Loc: Location{path: name},
|
||||
return ObjectInfo{
|
||||
Loc: ulloc.NewLocal(name),
|
||||
IsPrefix: isDir,
|
||||
Created: fi.current.ModTime(), // TODO: use real crtime
|
||||
ContentLength: fi.current.Size(),
|
||||
81
cmd/uplinkng/ulfs/mixed.go
Normal file
81
cmd/uplinkng/ulfs/mixed.go
Normal file
@ -0,0 +1,81 @@
|
||||
// Copyright (C) 2021 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package ulfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/zeebo/clingy"
|
||||
"github.com/zeebo/errs"
|
||||
|
||||
"storj.io/storj/cmd/uplinkng/ulloc"
|
||||
)
|
||||
|
||||
// Mixed dispatches to either the local or remote filesystem depending on the location.
|
||||
type Mixed struct {
|
||||
local *Local
|
||||
remote *Remote
|
||||
}
|
||||
|
||||
// NewMixed returns a Mixed backed by the provided local and remote filesystems.
|
||||
func NewMixed(local *Local, remote *Remote) *Mixed {
|
||||
return &Mixed{
|
||||
local: local,
|
||||
remote: remote,
|
||||
}
|
||||
}
|
||||
|
||||
// Close releases any resources that the Mixed contails.
|
||||
func (m *Mixed) Close() error {
|
||||
return m.remote.Close()
|
||||
}
|
||||
|
||||
// Open returns a ReadHandle to either a local file, remote object, or stdin.
|
||||
func (m *Mixed) Open(ctx clingy.Context, loc ulloc.Location) (ReadHandle, error) {
|
||||
if bucket, key, ok := loc.RemoteParts(); ok {
|
||||
return m.remote.Open(ctx, bucket, key)
|
||||
} else if path, ok := loc.LocalParts(); ok {
|
||||
return m.local.Open(ctx, path)
|
||||
}
|
||||
return newGenericReadHandle(ctx.Stdin()), nil
|
||||
}
|
||||
|
||||
// Create returns a WriteHandle to either a local file, remote object, or stdout.
|
||||
func (m *Mixed) Create(ctx clingy.Context, loc ulloc.Location) (WriteHandle, error) {
|
||||
if bucket, key, ok := loc.RemoteParts(); ok {
|
||||
return m.remote.Create(ctx, bucket, key)
|
||||
} else if path, ok := loc.LocalParts(); ok {
|
||||
return m.local.Create(ctx, path)
|
||||
}
|
||||
return newGenericWriteHandle(ctx.Stdout()), nil
|
||||
}
|
||||
|
||||
// ListObjects lists either files and directories with some local path prefix or remote objects
|
||||
// with a given bucket and key.
|
||||
func (m *Mixed) ListObjects(ctx context.Context, prefix ulloc.Location, recursive bool) (ObjectIterator, error) {
|
||||
if bucket, key, ok := prefix.RemoteParts(); ok {
|
||||
return m.remote.ListObjects(ctx, bucket, key, recursive), nil
|
||||
} else if path, ok := prefix.LocalParts(); ok {
|
||||
return m.local.ListObjects(ctx, path, recursive)
|
||||
}
|
||||
return nil, errs.New("unable to list objects for prefix %q", prefix)
|
||||
}
|
||||
|
||||
// ListUploads lists all of the pending uploads for remote objects with some given bucket and key.
|
||||
func (m *Mixed) ListUploads(ctx context.Context, prefix ulloc.Location, recursive bool) (ObjectIterator, error) {
|
||||
if bucket, key, ok := prefix.RemoteParts(); ok {
|
||||
return m.remote.ListUploads(ctx, bucket, key, recursive), nil
|
||||
} else if prefix.Local() {
|
||||
return emptyObjectIterator{}, nil
|
||||
}
|
||||
return nil, errs.New("unable to list uploads for prefix %q", prefix)
|
||||
}
|
||||
|
||||
// IsLocalDir returns true if the location is a directory that is local.
|
||||
func (m *Mixed) IsLocalDir(ctx context.Context, loc ulloc.Location) bool {
|
||||
if path, ok := loc.LocalParts(); ok {
|
||||
return m.local.IsLocalDir(ctx, path)
|
||||
}
|
||||
return false
|
||||
}
|
||||
@ -1,7 +1,7 @@
|
||||
// Copyright (C) 2021 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package main
|
||||
package ulfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
@ -12,16 +12,25 @@ import (
|
||||
"storj.io/uplink"
|
||||
)
|
||||
|
||||
// filesystemRemote implements something close to a filesystem but backed by an uplink project.
|
||||
type filesystemRemote struct {
|
||||
// Remote implements something close to a filesystem but backed by an uplink project.
|
||||
type Remote struct {
|
||||
project *uplink.Project
|
||||
}
|
||||
|
||||
func (r *filesystemRemote) Close() error {
|
||||
// NewRemote returns something close to a filesystem and returns objects using the project.
|
||||
func NewRemote(project *uplink.Project) *Remote {
|
||||
return &Remote{
|
||||
project: project,
|
||||
}
|
||||
}
|
||||
|
||||
// Close releases any resources that the Remote contains.
|
||||
func (r *Remote) Close() error {
|
||||
return r.project.Close()
|
||||
}
|
||||
|
||||
func (r *filesystemRemote) Open(ctx context.Context, bucket, key string) (readHandle, error) {
|
||||
// Open returns a ReadHandle for the object identified by a given bucket and key.
|
||||
func (r *Remote) Open(ctx context.Context, bucket, key string) (ReadHandle, error) {
|
||||
fh, err := r.project.DownloadObject(ctx, bucket, key, nil)
|
||||
if err != nil {
|
||||
return nil, errs.Wrap(err)
|
||||
@ -29,7 +38,8 @@ func (r *filesystemRemote) Open(ctx context.Context, bucket, key string) (readHa
|
||||
return newUplinkReadHandle(bucket, fh), nil
|
||||
}
|
||||
|
||||
func (r *filesystemRemote) Create(ctx context.Context, bucket, key string) (writeHandle, error) {
|
||||
// Create returns a WriteHandle for the object identified by a given bucket and key.
|
||||
func (r *Remote) Create(ctx context.Context, bucket, key string) (WriteHandle, error) {
|
||||
fh, err := r.project.UploadObject(ctx, bucket, key, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -37,7 +47,8 @@ func (r *filesystemRemote) Create(ctx context.Context, bucket, key string) (writ
|
||||
return newUplinkWriteHandle(fh), nil
|
||||
}
|
||||
|
||||
func (r *filesystemRemote) ListObjects(ctx context.Context, bucket, prefix string, recursive bool) objectIterator {
|
||||
// ListObjects lists all of the objects in some bucket that begin with the given prefix.
|
||||
func (r *Remote) ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ObjectIterator {
|
||||
parentPrefix := ""
|
||||
if idx := strings.LastIndexByte(prefix, '/'); idx >= 0 {
|
||||
parentPrefix = prefix[:idx+1]
|
||||
@ -60,7 +71,8 @@ func (r *filesystemRemote) ListObjects(ctx context.Context, bucket, prefix strin
|
||||
}
|
||||
}
|
||||
|
||||
func (r *filesystemRemote) ListPendingMultiparts(ctx context.Context, bucket, prefix string, recursive bool) objectIterator {
|
||||
// ListUploads lists all of the pending uploads in some bucket that begin with the given prefix.
|
||||
func (r *Remote) ListUploads(ctx context.Context, bucket, prefix string, recursive bool) ObjectIterator {
|
||||
parentPrefix := ""
|
||||
if idx := strings.LastIndexByte(prefix, '/'); idx >= 0 {
|
||||
parentPrefix = prefix[:idx+1]
|
||||
@ -99,7 +111,7 @@ func newUplinkObjectIterator(bucket string, iter *uplink.ObjectIterator) *uplink
|
||||
|
||||
func (u *uplinkObjectIterator) Next() bool { return u.iter.Next() }
|
||||
func (u *uplinkObjectIterator) Err() error { return u.iter.Err() }
|
||||
func (u *uplinkObjectIterator) Item() objectInfo {
|
||||
func (u *uplinkObjectIterator) Item() ObjectInfo {
|
||||
return uplinkObjectToObjectInfo(u.bucket, u.iter.Item())
|
||||
}
|
||||
|
||||
@ -119,6 +131,6 @@ func newUplinkUploadIterator(bucket string, iter *uplink.UploadIterator) *uplink
|
||||
|
||||
func (u *uplinkUploadIterator) Next() bool { return u.iter.Next() }
|
||||
func (u *uplinkUploadIterator) Err() error { return u.iter.Err() }
|
||||
func (u *uplinkUploadIterator) Item() objectInfo {
|
||||
func (u *uplinkUploadIterator) Item() ObjectInfo {
|
||||
return uplinkUploadInfoToObjectInfo(u.bucket, u.iter.Item())
|
||||
}
|
||||
@ -1,7 +1,7 @@
|
||||
// Copyright (C) 2021 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package main
|
||||
package ulloc
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@ -19,9 +19,30 @@ type Location struct {
|
||||
remote bool
|
||||
}
|
||||
|
||||
func parseLocation(location string) (p Location, err error) {
|
||||
// NewLocal returns a new Location that refers to a local path.
|
||||
func NewLocal(path string) Location {
|
||||
return Location{path: path}
|
||||
}
|
||||
|
||||
// NewRemote returns a new location that refers to a remote path.
|
||||
func NewRemote(bucket, key string) Location {
|
||||
return Location{
|
||||
bucket: bucket,
|
||||
key: key,
|
||||
remote: true,
|
||||
}
|
||||
}
|
||||
|
||||
// NewStd returns a new location that refers to stdin or stdout.
|
||||
func NewStd() Location {
|
||||
return Location{path: "-", key: "-"}
|
||||
}
|
||||
|
||||
// Parse turns the string form of the location into the structured Location
|
||||
// value and an error if it is unable to or the location is invalid.
|
||||
func Parse(location string) (p Location, err error) {
|
||||
if location == "-" {
|
||||
return Location{path: "-", key: "-"}, nil
|
||||
return NewStd(), nil
|
||||
}
|
||||
|
||||
// Locations, Chapter 2, Verses 9 to 21.
|
||||
@ -199,3 +220,55 @@ func (p Location) ListKeyName(prefix Location) (string, bool) {
|
||||
}
|
||||
return rem, false
|
||||
}
|
||||
|
||||
// RemoveKeyPrefix removes the prefix from the key or path in the location if they
|
||||
// begin with it.
|
||||
func (p Location) RemoveKeyPrefix(prefix string) Location {
|
||||
if p.remote {
|
||||
p.key = strings.TrimPrefix(p.key, prefix)
|
||||
} else {
|
||||
p.path = strings.TrimPrefix(p.path, prefix)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// RemoteParts returns the bucket and key for the location and a bool indicating
|
||||
// if those values are valid because the location is remote.
|
||||
func (p Location) RemoteParts() (bucket, key string, ok bool) {
|
||||
return p.bucket, p.key, p.Remote()
|
||||
}
|
||||
|
||||
// LocalParts returns the path for the location and a bool indicating if that
|
||||
// value is valid because the location is local.
|
||||
func (p Location) LocalParts() (path string, ok bool) {
|
||||
return p.path, p.Local()
|
||||
}
|
||||
|
||||
// Less returns true if the location is less than the passed in location.
|
||||
func (p Location) Less(q Location) bool {
|
||||
if !p.remote && q.remote {
|
||||
return true
|
||||
} else if !q.remote && p.remote {
|
||||
return false
|
||||
}
|
||||
|
||||
if p.bucket < q.bucket {
|
||||
return true
|
||||
} else if q.bucket < p.bucket {
|
||||
return false
|
||||
}
|
||||
|
||||
if p.key < q.key {
|
||||
return true
|
||||
} else if q.key < p.key {
|
||||
return false
|
||||
}
|
||||
|
||||
if p.path < q.path {
|
||||
return true
|
||||
} else if q.path < p.path {
|
||||
return false
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
261
cmd/uplinkng/ultest/filesystem.go
Normal file
261
cmd/uplinkng/ultest/filesystem.go
Normal file
@ -0,0 +1,261 @@
|
||||
// Copyright (C) 2021 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package ultest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/zeebo/clingy"
|
||||
"github.com/zeebo/errs"
|
||||
|
||||
"storj.io/storj/cmd/uplinkng/ulfs"
|
||||
"storj.io/storj/cmd/uplinkng/ulloc"
|
||||
)
|
||||
|
||||
//
|
||||
// ulfs.Filesystem
|
||||
//
|
||||
|
||||
type testFilesystem struct {
|
||||
stdin string
|
||||
ops []Operation
|
||||
created int64
|
||||
files map[ulloc.Location]byteFileData
|
||||
pending map[ulloc.Location][]*byteWriteHandle
|
||||
buckets map[string]struct{}
|
||||
}
|
||||
|
||||
func newTestFilesystem() *testFilesystem {
|
||||
return &testFilesystem{
|
||||
files: make(map[ulloc.Location]byteFileData),
|
||||
pending: make(map[ulloc.Location][]*byteWriteHandle),
|
||||
buckets: make(map[string]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
type byteFileData struct {
|
||||
data []byte
|
||||
created int64
|
||||
}
|
||||
|
||||
func (tfs *testFilesystem) ensureBucket(name string) {
|
||||
tfs.buckets[name] = struct{}{}
|
||||
}
|
||||
|
||||
func (tfs *testFilesystem) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tfs *testFilesystem) Open(ctx clingy.Context, loc ulloc.Location) (_ ulfs.ReadHandle, err error) {
|
||||
defer func() { tfs.ops = append(tfs.ops, newOp("open", loc, err)) }()
|
||||
|
||||
bf, ok := tfs.files[loc]
|
||||
if !ok {
|
||||
return nil, errs.New("file does not exist")
|
||||
}
|
||||
return &byteReadHandle{Buffer: bytes.NewBuffer(bf.data)}, nil
|
||||
}
|
||||
|
||||
func (tfs *testFilesystem) Create(ctx clingy.Context, loc ulloc.Location) (_ ulfs.WriteHandle, err error) {
|
||||
defer func() { tfs.ops = append(tfs.ops, newOp("create", loc, err)) }()
|
||||
|
||||
if bucket, _, ok := loc.RemoteParts(); ok {
|
||||
if _, ok := tfs.buckets[bucket]; !ok {
|
||||
return nil, errs.New("bucket %q does not exist", bucket)
|
||||
}
|
||||
}
|
||||
|
||||
tfs.created++
|
||||
wh := &byteWriteHandle{
|
||||
buf: bytes.NewBuffer(nil),
|
||||
loc: loc,
|
||||
tfs: tfs,
|
||||
cre: tfs.created,
|
||||
}
|
||||
|
||||
tfs.pending[loc] = append(tfs.pending[loc], wh)
|
||||
|
||||
return wh, nil
|
||||
}
|
||||
|
||||
func (tfs *testFilesystem) ListObjects(ctx context.Context, prefix ulloc.Location, recursive bool) (ulfs.ObjectIterator, error) {
|
||||
var infos []ulfs.ObjectInfo
|
||||
for loc, bf := range tfs.files {
|
||||
if loc.HasPrefix(prefix) {
|
||||
infos = append(infos, ulfs.ObjectInfo{
|
||||
Loc: loc,
|
||||
Created: time.Unix(bf.created, 0),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
sort.Sort(objectInfos(infos))
|
||||
|
||||
if !recursive {
|
||||
infos = collapseObjectInfos(prefix, infos)
|
||||
}
|
||||
|
||||
return &objectInfoIterator{infos: infos}, nil
|
||||
}
|
||||
|
||||
func (tfs *testFilesystem) ListUploads(ctx context.Context, prefix ulloc.Location, recursive bool) (ulfs.ObjectIterator, error) {
|
||||
var infos []ulfs.ObjectInfo
|
||||
for loc, whs := range tfs.pending {
|
||||
if loc.HasPrefix(prefix) {
|
||||
for _, wh := range whs {
|
||||
infos = append(infos, ulfs.ObjectInfo{
|
||||
Loc: loc,
|
||||
Created: time.Unix(wh.cre, 0),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort.Sort(objectInfos(infos))
|
||||
|
||||
if !recursive {
|
||||
infos = collapseObjectInfos(prefix, infos)
|
||||
}
|
||||
|
||||
return &objectInfoIterator{infos: infos}, nil
|
||||
}
|
||||
|
||||
func (tfs *testFilesystem) IsLocalDir(ctx context.Context, loc ulloc.Location) bool {
|
||||
// TODO: implement this
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
//
|
||||
// ulfs.ReadHandle
|
||||
//
|
||||
|
||||
type byteReadHandle struct {
|
||||
*bytes.Buffer
|
||||
}
|
||||
|
||||
func (b *byteReadHandle) Close() error { return nil }
|
||||
func (b *byteReadHandle) Info() ulfs.ObjectInfo { return ulfs.ObjectInfo{} }
|
||||
|
||||
//
|
||||
// ulfs.WriteHandle
|
||||
//
|
||||
|
||||
type byteWriteHandle struct {
|
||||
buf *bytes.Buffer
|
||||
loc ulloc.Location
|
||||
tfs *testFilesystem
|
||||
cre int64
|
||||
done bool
|
||||
}
|
||||
|
||||
func (b *byteWriteHandle) Write(p []byte) (int, error) {
|
||||
return b.buf.Write(p)
|
||||
}
|
||||
|
||||
func (b *byteWriteHandle) Commit() error {
|
||||
if err := b.close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b.tfs.ops = append(b.tfs.ops, newOp("commit", b.loc, nil))
|
||||
b.tfs.files[b.loc] = byteFileData{
|
||||
data: b.buf.Bytes(),
|
||||
created: b.cre,
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *byteWriteHandle) Abort() error {
|
||||
if err := b.close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b.tfs.ops = append(b.tfs.ops, newOp("append", b.loc, nil))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *byteWriteHandle) close() error {
|
||||
if b.done {
|
||||
return errs.New("already done")
|
||||
}
|
||||
b.done = true
|
||||
|
||||
handles := b.tfs.pending[b.loc]
|
||||
for i, v := range handles {
|
||||
if v == b {
|
||||
handles = append(handles[:i], handles[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if len(handles) > 0 {
|
||||
b.tfs.pending[b.loc] = handles
|
||||
} else {
|
||||
delete(b.tfs.pending, b.loc)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
//
|
||||
// ulfs.ObjectIterator
|
||||
//
|
||||
|
||||
type objectInfoIterator struct {
|
||||
infos []ulfs.ObjectInfo
|
||||
current ulfs.ObjectInfo
|
||||
}
|
||||
|
||||
func (li *objectInfoIterator) Next() bool {
|
||||
if len(li.infos) == 0 {
|
||||
return false
|
||||
}
|
||||
li.current, li.infos = li.infos[0], li.infos[1:]
|
||||
return true
|
||||
}
|
||||
|
||||
func (li *objectInfoIterator) Err() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (li *objectInfoIterator) Item() ulfs.ObjectInfo {
|
||||
return li.current
|
||||
}
|
||||
|
||||
type objectInfos []ulfs.ObjectInfo
|
||||
|
||||
func (ois objectInfos) Len() int { return len(ois) }
|
||||
func (ois objectInfos) Swap(i int, j int) { ois[i], ois[j] = ois[j], ois[i] }
|
||||
func (ois objectInfos) Less(i int, j int) bool { return ois[i].Loc.Less(ois[j].Loc) }
|
||||
|
||||
func collapseObjectInfos(prefix ulloc.Location, infos []ulfs.ObjectInfo) []ulfs.ObjectInfo {
|
||||
collapsing := false
|
||||
current := ""
|
||||
j := 0
|
||||
|
||||
for _, oi := range infos {
|
||||
first, ok := oi.Loc.ListKeyName(prefix)
|
||||
if ok {
|
||||
if collapsing && first == current {
|
||||
continue
|
||||
}
|
||||
|
||||
collapsing = true
|
||||
current = first
|
||||
|
||||
oi.IsPrefix = true
|
||||
}
|
||||
|
||||
oi.Loc = oi.Loc.SetKey(first)
|
||||
|
||||
infos[j] = oi
|
||||
j++
|
||||
}
|
||||
|
||||
return infos[:j]
|
||||
}
|
||||
91
cmd/uplinkng/ultest/result.go
Normal file
91
cmd/uplinkng/ultest/result.go
Normal file
@ -0,0 +1,91 @@
|
||||
// Copyright (C) 2021 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package ultest
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"storj.io/storj/cmd/uplinkng/ulloc"
|
||||
)
|
||||
|
||||
// Result captures all the output of running a command for inspection.
|
||||
type Result struct {
|
||||
Stdout string
|
||||
Stderr string
|
||||
Ok bool
|
||||
Err error
|
||||
Operations []Operation
|
||||
}
|
||||
|
||||
// RequireSuccess fails if the Result did not observe a successful execution.
|
||||
func (r Result) RequireSuccess(t *testing.T) {
|
||||
if !r.Ok {
|
||||
errs := parseErrors(r.Stdout)
|
||||
require.True(t, r.Ok, "test did not run successfully. errors:\n%s",
|
||||
strings.Join(errs, "\n"))
|
||||
}
|
||||
require.NoError(t, r.Err)
|
||||
}
|
||||
|
||||
// RequireFailure fails if the Result did not observe a failed execution.
|
||||
func (r Result) RequireFailure(t *testing.T) {
|
||||
require.False(t, r.Ok && r.Err == nil, "command ran with no error")
|
||||
}
|
||||
|
||||
// RequireStdout requires that the execution wrote to stdout the provided string.
|
||||
// Blank lines are ignored and all lines are space trimmed for the comparison.
|
||||
func (r Result) RequireStdout(t *testing.T, stdout string) {
|
||||
require.Equal(t, trimNewlineSpaces(stdout), trimNewlineSpaces(r.Stdout))
|
||||
}
|
||||
|
||||
// RequireStderr requires that the execution wrote to stderr the provided string.
|
||||
// Blank lines are ignored and all lines are space trimmed for the comparison.
|
||||
func (r Result) RequireStderr(t *testing.T, stderr string) {
|
||||
require.Equal(t, trimNewlineSpaces(stderr), trimNewlineSpaces(r.Stderr))
|
||||
}
|
||||
|
||||
func parseErrors(s string) []string {
|
||||
lines := strings.Split(s, "\n")
|
||||
start := 0
|
||||
for i, line := range lines {
|
||||
if line == "Errors:" {
|
||||
start = i + 1
|
||||
} else if len(line) > 0 && line[0] != ' ' {
|
||||
return lines[start:i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func trimNewlineSpaces(s string) string {
|
||||
lines := strings.Split(s, "\n")
|
||||
|
||||
j := 0
|
||||
for _, line := range lines {
|
||||
if trimmed := strings.TrimSpace(line); len(trimmed) > 0 {
|
||||
lines[j] = trimmed
|
||||
j++
|
||||
}
|
||||
}
|
||||
return strings.Join(lines[:j], "\n")
|
||||
}
|
||||
|
||||
// Operation represents some kind of filesystem operation that happened
|
||||
// on some location, and if the operation failed.
|
||||
type Operation struct {
|
||||
Kind string
|
||||
Loc string
|
||||
Error bool
|
||||
}
|
||||
|
||||
func newOp(kind string, loc ulloc.Location, err error) Operation {
|
||||
return Operation{
|
||||
Kind: kind,
|
||||
Loc: loc.String(),
|
||||
Error: err != nil,
|
||||
}
|
||||
}
|
||||
154
cmd/uplinkng/ultest/setup.go
Normal file
154
cmd/uplinkng/ultest/setup.go
Normal file
@ -0,0 +1,154 @@
|
||||
// Copyright (C) 2021 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package ultest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/zeebo/clingy"
|
||||
"github.com/zeebo/errs"
|
||||
|
||||
"storj.io/storj/cmd/uplinkng/ulfs"
|
||||
"storj.io/storj/cmd/uplinkng/ulloc"
|
||||
)
|
||||
|
||||
// Setup returns some State that can be run multiple times with different command
|
||||
// line arguments.
|
||||
func Setup(cmds func(clingy.Commands, clingy.Flags), opts ...ExecuteOption) State {
|
||||
return State{
|
||||
cmds: cmds,
|
||||
opts: opts,
|
||||
}
|
||||
}
|
||||
|
||||
// State represents some state and environment for a command to execute in.
|
||||
type State struct {
|
||||
cmds func(clingy.Commands, clingy.Flags)
|
||||
opts []ExecuteOption
|
||||
}
|
||||
|
||||
// With appends the provided options and returns a new State.
|
||||
func (st State) With(opts ...ExecuteOption) State {
|
||||
st.opts = append([]ExecuteOption(nil), st.opts...)
|
||||
st.opts = append(st.opts, opts...)
|
||||
return st
|
||||
}
|
||||
|
||||
// Succeed is the same as Run followed by result.RequireSuccess.
|
||||
func (st State) Succeed(t *testing.T, args ...string) Result {
|
||||
result := st.Run(t, args...)
|
||||
result.RequireSuccess(t)
|
||||
return result
|
||||
}
|
||||
|
||||
// Fail is the same as Run followed by result.RequireFailure.
|
||||
func (st State) Fail(t *testing.T, args ...string) Result {
|
||||
result := st.Run(t, args...)
|
||||
result.RequireFailure(t)
|
||||
return result
|
||||
}
|
||||
|
||||
// Run executes the command specified by the args and returns a Result.
|
||||
func (st State) Run(t *testing.T, args ...string) Result {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
var stdin bytes.Buffer
|
||||
var ops []Operation
|
||||
var ran bool
|
||||
|
||||
ok, err := clingy.Environment{
|
||||
Name: "uplink-test",
|
||||
Args: args,
|
||||
|
||||
Stdin: &stdin,
|
||||
Stdout: &stdout,
|
||||
Stderr: &stderr,
|
||||
|
||||
Wrap: func(ctx clingy.Context, cmd clingy.Cmd) error {
|
||||
tfs := newTestFilesystem()
|
||||
for _, opt := range st.opts {
|
||||
if err := opt.fn(ctx, tfs); err != nil {
|
||||
return errs.Wrap(err)
|
||||
}
|
||||
}
|
||||
tfs.ops = nil
|
||||
|
||||
if len(tfs.stdin) > 0 {
|
||||
_, _ = stdin.WriteString(tfs.stdin)
|
||||
}
|
||||
|
||||
if setter, ok := cmd.(interface {
|
||||
SetTestFilesystem(ulfs.Filesystem)
|
||||
}); ok {
|
||||
setter.SetTestFilesystem(tfs)
|
||||
}
|
||||
|
||||
ran = true
|
||||
err := cmd.Execute(ctx)
|
||||
ops = tfs.ops
|
||||
return err
|
||||
},
|
||||
}.Run(context.Background(), st.cmds)
|
||||
|
||||
if ok && err == nil {
|
||||
require.True(t, ran, "no command was executed: %q", args)
|
||||
}
|
||||
return Result{
|
||||
Stdout: stdout.String(),
|
||||
Stderr: stderr.String(),
|
||||
Ok: ok,
|
||||
Err: err,
|
||||
Operations: ops,
|
||||
}
|
||||
}
|
||||
|
||||
// ExecuteOption allows one to control the environment that a command executes in.
|
||||
type ExecuteOption struct {
|
||||
fn func(ctx clingy.Context, tfs *testFilesystem) error
|
||||
}
|
||||
|
||||
// WithStdin sets the command to execute with the provided string as standard input.
|
||||
func WithStdin(stdin string) ExecuteOption {
|
||||
return ExecuteOption{func(_ clingy.Context, tfs *testFilesystem) error {
|
||||
tfs.stdin = stdin
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// WithFile sets the command to execute with a file created at the given location.
|
||||
func WithFile(location string) ExecuteOption {
|
||||
return ExecuteOption{func(ctx clingy.Context, tfs *testFilesystem) error {
|
||||
loc, err := ulloc.Parse(location)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if bucket, _, ok := loc.RemoteParts(); ok {
|
||||
tfs.ensureBucket(bucket)
|
||||
}
|
||||
wh, err := tfs.Create(ctx, loc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return wh.Commit()
|
||||
}}
|
||||
}
|
||||
|
||||
// WithPendingFile sets the command to execute with a pending upload happening to
|
||||
// the provided location.
|
||||
func WithPendingFile(location string) ExecuteOption {
|
||||
return ExecuteOption{func(ctx clingy.Context, tfs *testFilesystem) error {
|
||||
loc, err := ulloc.Parse(location)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if bucket, _, ok := loc.RemoteParts(); ok {
|
||||
tfs.ensureBucket(bucket)
|
||||
}
|
||||
_, err = tfs.Create(ctx, loc)
|
||||
return err
|
||||
}}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user