Support empty path components (#2574)
This commit is contained in:
Isaac Hess
GitHub
parent
5710dc3a32
commit
b8fe349816
233
lib/uplink/list_test.go
Normal file
233
lib/uplink/list_test.go
Normal file
@ -0,0 +1,233 @@
|
||||
// Copyright (C) 2019 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package uplink_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"storj.io/storj/internal/testcontext"
|
||||
"storj.io/storj/internal/testplanet"
|
||||
"storj.io/storj/lib/uplink"
|
||||
"storj.io/storj/pkg/storj"
|
||||
)
|
||||
|
||||
type expectedResult struct {
|
||||
path string
|
||||
isPrefix bool
|
||||
}
|
||||
|
||||
type listChallenge struct {
|
||||
commands []string // list commands that should result in the same set of responses
|
||||
expectedResults []expectedResult // results that should come back for each command above
|
||||
}
|
||||
|
||||
type putGetListTest struct {
|
||||
bucket string // bucket to upload to
|
||||
paths []string // test will create a file at every path here
|
||||
listChallenges []listChallenge // set of list commands and expected results
|
||||
}
|
||||
|
||||
// TestPutGetList allows you to create a bucket, a set of paths, and a set of
|
||||
// list challenges that you want to try against that configuring. It will create
|
||||
// a unique file for each path, upload it, download it, and run the list
|
||||
// challenges against it.
|
||||
func TestPutGetList(t *testing.T) {
|
||||
testplanet.Run(t, testplanet.Config{
|
||||
SatelliteCount: 1,
|
||||
StorageNodeCount: 1,
|
||||
UplinkCount: 1},
|
||||
func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
||||
apiKey := planet.Uplinks[0].APIKey[planet.Satellites[0].ID()]
|
||||
satelliteAddr := planet.Satellites[0].Local().Address.Address
|
||||
|
||||
tests := []putGetListTest{
|
||||
{
|
||||
bucket: "bu1",
|
||||
paths: []string{
|
||||
"a-file",
|
||||
"a/b-file",
|
||||
"a/b/slash-file",
|
||||
"a/b////",
|
||||
"a/b//",
|
||||
"a/b//c-file",
|
||||
"a/b//c/",
|
||||
"//bob",
|
||||
"/",
|
||||
},
|
||||
listChallenges: []listChallenge{
|
||||
{
|
||||
commands: []string{"/"},
|
||||
expectedResults: []expectedResult{
|
||||
{
|
||||
path: "/",
|
||||
isPrefix: true,
|
||||
},
|
||||
{
|
||||
path: "",
|
||||
isPrefix: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
commands: []string{"//"},
|
||||
expectedResults: []expectedResult{
|
||||
{
|
||||
path: "bob",
|
||||
isPrefix: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
commands: []string{"a", "a/"},
|
||||
expectedResults: []expectedResult{
|
||||
{
|
||||
path: "b/",
|
||||
isPrefix: true,
|
||||
},
|
||||
{
|
||||
path: "b-file",
|
||||
isPrefix: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
commands: []string{"a/b", "a/b/"},
|
||||
expectedResults: []expectedResult{
|
||||
{
|
||||
path: "/",
|
||||
isPrefix: true,
|
||||
},
|
||||
{
|
||||
path: "slash-file",
|
||||
isPrefix: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
commands: []string{"a/b//"},
|
||||
expectedResults: []expectedResult{
|
||||
{
|
||||
path: "c/",
|
||||
isPrefix: true,
|
||||
},
|
||||
{
|
||||
path: "c-file",
|
||||
isPrefix: false,
|
||||
},
|
||||
{
|
||||
path: "/",
|
||||
isPrefix: true,
|
||||
},
|
||||
{
|
||||
path: "",
|
||||
isPrefix: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
commands: []string{"a/b///"},
|
||||
expectedResults: []expectedResult{
|
||||
{
|
||||
path: "/",
|
||||
isPrefix: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
commands: []string{"a/b////"},
|
||||
expectedResults: []expectedResult{
|
||||
{
|
||||
path: "",
|
||||
isPrefix: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
runTest(ctx, t, apiKey, satelliteAddr, test)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func runTest(ctx context.Context, t *testing.T, apiKey, satelliteAddr string,
|
||||
test putGetListTest) {
|
||||
|
||||
errCatch := func(fn func() error) { require.NoError(t, fn()) }
|
||||
|
||||
cfg := &uplink.Config{}
|
||||
cfg.Volatile.TLS.SkipPeerCAWhitelist = true
|
||||
|
||||
ul, err := uplink.NewUplink(ctx, cfg)
|
||||
require.NoError(t, err)
|
||||
defer errCatch(ul.Close)
|
||||
|
||||
key, err := uplink.ParseAPIKey(apiKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
p, err := ul.OpenProject(ctx, satelliteAddr, key)
|
||||
require.NoError(t, err)
|
||||
defer errCatch(p.Close)
|
||||
|
||||
_, err = p.CreateBucket(ctx, test.bucket, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
encKey, err := p.SaltedKeyFromPassphrase(ctx, "my secret passphrase")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Make an encryption context
|
||||
access := uplink.NewEncryptionAccessWithDefaultKey(*encKey)
|
||||
|
||||
bu, err := p.OpenBucket(ctx, test.bucket, access)
|
||||
require.NoError(t, err)
|
||||
|
||||
// First upload files to all the specified paths
|
||||
for _, path := range test.paths {
|
||||
err = bu.UploadObject(ctx, path, strings.NewReader(fmt.Sprintf("%s%s", path, "hi")), nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Now run the listChallenges and check the results
|
||||
for _, listChallenge := range test.listChallenges {
|
||||
for _, command := range listChallenge.commands {
|
||||
results, err := bu.ListObjects(ctx, &uplink.ListOptions{
|
||||
Direction: storj.After,
|
||||
Cursor: "",
|
||||
Prefix: command,
|
||||
Recursive: false,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
compareResults(t, results.Items, listChallenge.expectedResults)
|
||||
}
|
||||
}
|
||||
|
||||
// Download all files, make sure they work
|
||||
for _, path := range test.paths {
|
||||
r, err := bu.NewReader(ctx, path)
|
||||
require.NoError(t, err)
|
||||
downloaded, err := ioutil.ReadAll(r)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, fmt.Sprintf("%s%s", path, "hi"), string(downloaded))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func compareResults(t *testing.T, items []storj.Object, expected []expectedResult) {
|
||||
require.Equal(t, len(expected), len(items))
|
||||
sort.SliceStable(items, func(i, j int) bool { return items[i].Path < items[j].Path })
|
||||
sort.SliceStable(expected, func(i, j int) bool { return expected[i].path < expected[j].path })
|
||||
for i, item := range items {
|
||||
require.Equal(t, expected[i].path, item.Path)
|
||||
require.Equal(t, expected[i].isPrefix, item.IsPrefix)
|
||||
}
|
||||
}
|
||||
@ -10,6 +10,7 @@ import (
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gogo/protobuf/proto"
|
||||
@ -450,6 +451,13 @@ type ListItem struct {
|
||||
IsPrefix bool
|
||||
}
|
||||
|
||||
// pathForKey removes the trailing `/` from the raw path, which is required so
|
||||
// the derived key matches the final list path (which also has the trailing
|
||||
// encrypted `/` part of the path removed)
|
||||
func pathForKey(raw string) paths.Unencrypted {
|
||||
return paths.NewUnencrypted(strings.TrimSuffix(raw, "/"))
|
||||
}
|
||||
|
||||
// List all the paths inside l/, stripping off the l/ prefix
|
||||
func (s *streamStore) List(ctx context.Context, prefix Path, startAfter, endBefore string, pathCipher storj.CipherSuite, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
@ -460,7 +468,7 @@ func (s *streamStore) List(ctx context.Context, prefix Path, startAfter, endBefo
|
||||
metaFlags |= meta.UserDefined
|
||||
}
|
||||
|
||||
prefixKey, err := encryption.DerivePathKey(prefix.Bucket(), prefix.UnencryptedPath(), s.encStore)
|
||||
prefixKey, err := encryption.DerivePathKey(prefix.Bucket(), pathForKey(prefix.UnencryptedPath().Raw()), s.encStore)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
@ -470,6 +478,15 @@ func (s *streamStore) List(ctx context.Context, prefix Path, startAfter, endBefo
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
// If the raw unencrypted path ends in a `/` we need to remove the final
|
||||
// section of the encrypted path. For example, if we are listing the path
|
||||
// `/bob/`, the encrypted path results in `enc("")/enc("bob")/enc("")`. This
|
||||
// is an incorrect list prefix, what we really want is `enc("")/enc("bob")`
|
||||
if strings.HasSuffix(prefix.UnencryptedPath().Raw(), "/") {
|
||||
lastSlashIdx := strings.LastIndex(encPrefix.Raw(), "/")
|
||||
encPrefix = paths.NewEncrypted(encPrefix.Raw()[:lastSlashIdx])
|
||||
}
|
||||
|
||||
// We have to encrypt startAfter and endBefore but only if they don't contain a bucket.
|
||||
// They contain a bucket if and only if the prefix has no bucket. This is why they are raw
|
||||
// strings instead of a typed string: it's either a bucket or an unencrypted path component
|
||||
|
||||
Loading…
Reference in New Issue
Block a user