Metadata Access from Uplink CLI (#3310)

This commit is contained in:
Caleb Case 2019-11-06 11:04:12 -05:00 committed by GitHub
parent fd9f860fd6
commit 3b78addb2d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 378 additions and 1 deletions

View File

@ -5,6 +5,7 @@ package cmd
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
@ -24,6 +25,7 @@ import (
var (
progress *bool
expires *string
metadata *string
)
func init() {
@ -32,8 +34,10 @@ func init() {
Short: "Copies a local file or Storj object to another location locally or in Storj",
RunE: copyMain,
}, RootCmd)
progress = cpCmd.Flags().Bool("progress", true, "if true, show progress")
expires = cpCmd.Flags().String("expires", "", "optional expiration date of an object. Please use format (yyyy-mm-ddThh:mm:ssZhh:mm)")
metadata = cpCmd.Flags().String("metadata", "", "optional metadata for the object. Please use a single level JSON object of string to string only")
}
// upload transfers src from local machine to s3 compatible object dst
@ -102,6 +106,17 @@ func upload(ctx context.Context, src fpath.FPath, dst fpath.FPath, showProgress
opts.Expires = expiration.UTC()
}
if *metadata != "" {
var md map[string]string
err := json.Unmarshal([]byte(*metadata), &md)
if err != nil {
return err
}
opts.Metadata = md
}
opts.Volatile.RedundancyScheme = cfg.GetRedundancyScheme()
opts.Volatile.EncryptionParameters = cfg.GetEncryptionParameters()
@ -252,7 +267,7 @@ func copyObject(ctx context.Context, src fpath.FPath, dst fpath.FPath) (err erro
return nil
}
// copyMain is the function executed when cpCmd is called
// copyMain is the function executed when cpCmd is called.
func copyMain(cmd *cobra.Command, args []string) (err error) {
if len(args) == 0 {
return fmt.Errorf("No object specified for copy")

17
cmd/uplink/cmd/meta.go Normal file
View File

@ -0,0 +1,17 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package cmd
import (
"github.com/spf13/cobra"
)
var metaCmd *cobra.Command
func init() {
metaCmd = addCmd(&cobra.Command{
Use: "meta",
Short: "Metadata related commands",
}, RootCmd)
}

100
cmd/uplink/cmd/meta_get.go Normal file
View File

@ -0,0 +1,100 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package cmd
import (
"encoding/json"
"fmt"
"github.com/spf13/cobra"
"github.com/zeebo/errs"
"storj.io/storj/internal/fpath"
"storj.io/storj/pkg/process"
)
func init() {
addCmd(&cobra.Command{
Use: "get [KEY] PATH",
Short: "Get a Storj object's metadata",
RunE: metaGetMain,
}, metaCmd)
}
// metaGetMain is the function executed when metaGetCmd is called.
func metaGetMain(cmd *cobra.Command, args []string) (err error) {
var key *string
var path string
switch len(args) {
case 0:
return fmt.Errorf("No object specified")
case 1:
path = args[0]
case 2:
key = &args[0]
path = args[1]
default:
return fmt.Errorf("Too many arguments")
}
ctx, _ := process.Ctx(cmd)
src, err := fpath.New(path)
if err != nil {
return err
}
if src.IsLocal() {
return fmt.Errorf("The source destination must be a Storj URL")
}
project, bucket, err := cfg.GetProjectAndBucket(ctx, src.Bucket())
if err != nil {
return err
}
defer closeProjectAndBucket(project, bucket)
object, err := bucket.OpenObject(ctx, src.Path())
if err != nil {
return err
}
defer func() { err = errs.Combine(err, object.Close()) }()
if key != nil {
var keyNorm string
err := json.Unmarshal([]byte("\""+*key+"\""), &keyNorm)
if err != nil {
return err
}
value, ok := object.Meta.Metadata[keyNorm]
if ok != true {
return fmt.Errorf("Key does not exist")
}
str, err := json.Marshal(value)
if err != nil {
return err
}
fmt.Printf("%s\n", str[1:len(str)-1])
return nil
}
if object.Meta.Metadata != nil {
str, err := json.MarshalIndent(object.Meta.Metadata, "", " ")
if err != nil {
return err
}
fmt.Printf("%s\n", string(str))
return nil
}
fmt.Printf("{}\n")
return nil
}

156
cmd/uplink/cmd/meta_test.go Normal file
View File

@ -0,0 +1,156 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package cmd_test
import (
"encoding/json"
"fmt"
"os/exec"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"storj.io/storj/internal/testcontext"
"storj.io/storj/internal/testplanet"
"storj.io/storj/internal/testrand"
)
func min(x, y int) int {
if x < y {
return x
}
return y
}
func TestSetGetMeta(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1,
StorageNodeCount: 4,
UplinkCount: 1,
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
uplinkExe := ctx.Compile("storj.io/storj/cmd/uplink")
// Configure uplink.
{
output, err := exec.Command(uplinkExe,
"--config-dir", ctx.Dir("uplink"),
"setup",
"--non-interactive",
"--scope", planet.Uplinks[0].GetConfig(planet.Satellites[0]).Scope,
).CombinedOutput()
t.Log(string(output))
require.NoError(t, err)
}
// Create bucket.
bucketName := testrand.BucketName()
{
output, err := exec.Command(uplinkExe,
"--config-dir", ctx.Dir("uplink"),
"mb",
"sj://"+bucketName,
).CombinedOutput()
t.Log(string(output))
require.NoError(t, err)
}
// Upload file with metadata.
metadata := testrand.Metadata()
metadataBs, err := json.Marshal(metadata)
require.NoError(t, err)
metadataStr := string(metadataBs)
var metadataNorm map[string]string
err = json.Unmarshal(metadataBs, &metadataNorm)
require.NoError(t, err)
path := testrand.Path()
uri := "sj://" + bucketName + "/" + path
{
output, err := exec.Command(uplinkExe,
"--config-dir", ctx.Dir("uplink"),
"cp",
"--metadata", metadataStr,
"-", uri,
).CombinedOutput()
t.Log(string(output))
require.NoError(t, err)
}
// Get all metadata.
{
cmd := exec.Command(uplinkExe,
"--config-dir", ctx.Dir("uplink"),
"meta", "get", uri,
)
t.Log(cmd)
output, err := cmd.Output()
t.Log(string(output))
if !assert.NoError(t, err) {
if ee, ok := err.(*exec.ExitError); ok {
t.Log(ee)
t.Log(string(ee.Stderr))
}
return
}
var md map[string]string
err = json.Unmarshal(output, &md)
require.NoError(t, err)
assert.Equal(t, metadataNorm, md)
}
// Get specific metadata.
//
// NOTE: The CLI expects JSON encoded strings for input and
// output. The key and value returned from the CLI have to be
// converted from the JSON encoded string into the Go native
// string for comparison.
for key, value := range metadataNorm {
key, value := key, value
t.Run(fmt.Sprintf("Fetching key %q", key[:min(len(key), 8)]), func(t *testing.T) {
keyNorm, err := json.Marshal(key)
require.NoError(t, err)
cmd := exec.Command(uplinkExe,
"--config-dir", ctx.Dir("uplink"),
"meta", "get", string(keyNorm[1:len(keyNorm)-1]), uri,
)
t.Log(cmd)
output, err := cmd.Output()
assert.NoError(t, err)
if err != nil {
if ee, ok := err.(*exec.ExitError); ok {
t.Log(ee)
t.Log(string(ee.Stderr))
}
return
}
// Remove trailing newline.
if len(output) > 0 && string(output[len(output)-1]) == "\n" {
output = output[:len(output)-1]
}
var outputNorm string
err = json.Unmarshal([]byte("\""+string(output)+"\""), &outputNorm)
require.NoError(t, err)
assert.Equal(t, value, outputNorm)
})
}
})
}

View File

@ -115,3 +115,92 @@ func UUID() uuid.UUID {
Read(uuid[:])
return uuid
}
// BucketName creates a random bucket name mostly confirming to the
// restrictions of S3:
// https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
//
// NOTE: This may not generate values that cover all valid values (for Storj or
// S3). This is a best effort to cover most cases we believe our design
// requires and will need to be revisited when a more explicit design spec is
// created.
func BucketName() string {
const (
edges = "abcdefghijklmnopqrstuvwxyz0123456789"
body = "abcdefghijklmnopqrstuvwxyz0123456789-"
min = 3
max = 63
)
size := rand.Intn(max-min) + min
b := make([]byte, size)
for i := range b {
switch i {
case 0:
fallthrough
case size - 1:
b[i] = edges[rand.Intn(len(edges))]
default:
b[i] = body[rand.Intn(len(body))]
}
}
return string(b)
}
// Metadata creates random metadata mostly conforming to the restrictions of S3:
// https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#object-metadata
//
// NOTE: This may not generate values that cover all valid values (for Storj or
// S3). This is a best effort to cover most cases we believe our design
// requires and will need to be revisited when a more explicit design spec is
// created.
func Metadata() map[string]string {
const (
max = 2 * 1024
)
total := rand.Intn(max)
metadata := make(map[string]string)
for used := 0; total-used > 1; {
keySize := rand.Intn(total-(used+1)) + 1
key := BytesInt(keySize)
used += len(key)
valueSize := rand.Intn(total - used)
value := BytesInt(valueSize)
used += len(value)
metadata[string(key)] = string(value)
}
return metadata
}
// Path creates a random path mostly conforming to the retrictions of S3:
// https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#object-keys
//
// NOTE: This may not generate values that cover all valid values (for Storj or
// S3). This is a best effort to cover most cases we believe our design
// requires and will need to be revisited when a more explicit design spec is
// created.
func Path() string {
const (
max = 1 * 1024
)
total := rand.Intn(max)
path := ""
for used := 0; len(path) < total; {
if used != 0 {
path += "/"
}
path += SegmentID(rand.Intn(total - used)).String()
}
return path[:total]
}