From 3b78addb2ddf8a782fcfc6a2918c5e589b77224e Mon Sep 17 00:00:00 2001 From: Caleb Case Date: Wed, 6 Nov 2019 11:04:12 -0500 Subject: [PATCH] Metadata Access from Uplink CLI (#3310) --- cmd/uplink/cmd/cp.go | 17 +++- cmd/uplink/cmd/meta.go | 17 ++++ cmd/uplink/cmd/meta_get.go | 100 +++++++++++++++++++++++ cmd/uplink/cmd/meta_test.go | 156 ++++++++++++++++++++++++++++++++++++ internal/testrand/rand.go | 89 ++++++++++++++++++++ 5 files changed, 378 insertions(+), 1 deletion(-) create mode 100644 cmd/uplink/cmd/meta.go create mode 100644 cmd/uplink/cmd/meta_get.go create mode 100644 cmd/uplink/cmd/meta_test.go diff --git a/cmd/uplink/cmd/cp.go b/cmd/uplink/cmd/cp.go index c3cea21e4..800f70376 100644 --- a/cmd/uplink/cmd/cp.go +++ b/cmd/uplink/cmd/cp.go @@ -5,6 +5,7 @@ package cmd import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -24,6 +25,7 @@ import ( var ( progress *bool expires *string + metadata *string ) func init() { @@ -32,8 +34,10 @@ func init() { Short: "Copies a local file or Storj object to another location locally or in Storj", RunE: copyMain, }, RootCmd) + progress = cpCmd.Flags().Bool("progress", true, "if true, show progress") expires = cpCmd.Flags().String("expires", "", "optional expiration date of an object. Please use format (yyyy-mm-ddThh:mm:ssZhh:mm)") + metadata = cpCmd.Flags().String("metadata", "", "optional metadata for the object. Please use a single level JSON object of string to string only") } // upload transfers src from local machine to s3 compatible object dst @@ -102,6 +106,17 @@ func upload(ctx context.Context, src fpath.FPath, dst fpath.FPath, showProgress opts.Expires = expiration.UTC() } + if *metadata != "" { + var md map[string]string + + err := json.Unmarshal([]byte(*metadata), &md) + if err != nil { + return err + } + + opts.Metadata = md + } + opts.Volatile.RedundancyScheme = cfg.GetRedundancyScheme() opts.Volatile.EncryptionParameters = cfg.GetEncryptionParameters() @@ -252,7 +267,7 @@ func copyObject(ctx context.Context, src fpath.FPath, dst fpath.FPath) (err erro return nil } -// copyMain is the function executed when cpCmd is called +// copyMain is the function executed when cpCmd is called. func copyMain(cmd *cobra.Command, args []string) (err error) { if len(args) == 0 { return fmt.Errorf("No object specified for copy") diff --git a/cmd/uplink/cmd/meta.go b/cmd/uplink/cmd/meta.go new file mode 100644 index 000000000..56ec71cbe --- /dev/null +++ b/cmd/uplink/cmd/meta.go @@ -0,0 +1,17 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package cmd + +import ( + "github.com/spf13/cobra" +) + +var metaCmd *cobra.Command + +func init() { + metaCmd = addCmd(&cobra.Command{ + Use: "meta", + Short: "Metadata related commands", + }, RootCmd) +} diff --git a/cmd/uplink/cmd/meta_get.go b/cmd/uplink/cmd/meta_get.go new file mode 100644 index 000000000..eaf42a849 --- /dev/null +++ b/cmd/uplink/cmd/meta_get.go @@ -0,0 +1,100 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package cmd + +import ( + "encoding/json" + "fmt" + + "github.com/spf13/cobra" + "github.com/zeebo/errs" + + "storj.io/storj/internal/fpath" + "storj.io/storj/pkg/process" +) + +func init() { + addCmd(&cobra.Command{ + Use: "get [KEY] PATH", + Short: "Get a Storj object's metadata", + RunE: metaGetMain, + }, metaCmd) +} + +// metaGetMain is the function executed when metaGetCmd is called. +func metaGetMain(cmd *cobra.Command, args []string) (err error) { + var key *string + var path string + + switch len(args) { + case 0: + return fmt.Errorf("No object specified") + case 1: + path = args[0] + case 2: + key = &args[0] + path = args[1] + default: + return fmt.Errorf("Too many arguments") + } + + ctx, _ := process.Ctx(cmd) + + src, err := fpath.New(path) + if err != nil { + return err + } + if src.IsLocal() { + return fmt.Errorf("The source destination must be a Storj URL") + } + + project, bucket, err := cfg.GetProjectAndBucket(ctx, src.Bucket()) + if err != nil { + return err + } + defer closeProjectAndBucket(project, bucket) + + object, err := bucket.OpenObject(ctx, src.Path()) + if err != nil { + return err + } + defer func() { err = errs.Combine(err, object.Close()) }() + + if key != nil { + var keyNorm string + err := json.Unmarshal([]byte("\""+*key+"\""), &keyNorm) + if err != nil { + return err + } + + value, ok := object.Meta.Metadata[keyNorm] + if ok != true { + return fmt.Errorf("Key does not exist") + } + + str, err := json.Marshal(value) + if err != nil { + return err + } + + fmt.Printf("%s\n", str[1:len(str)-1]) + + return nil + } + + if object.Meta.Metadata != nil { + str, err := json.MarshalIndent(object.Meta.Metadata, "", " ") + if err != nil { + return err + } + + fmt.Printf("%s\n", string(str)) + + return nil + } + + fmt.Printf("{}\n") + + return nil +} diff --git a/cmd/uplink/cmd/meta_test.go b/cmd/uplink/cmd/meta_test.go new file mode 100644 index 000000000..bbb159724 --- /dev/null +++ b/cmd/uplink/cmd/meta_test.go @@ -0,0 +1,156 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package cmd_test + +import ( + "encoding/json" + "fmt" + "os/exec" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "storj.io/storj/internal/testcontext" + "storj.io/storj/internal/testplanet" + "storj.io/storj/internal/testrand" +) + +func min(x, y int) int { + if x < y { + return x + } + + return y +} + +func TestSetGetMeta(t *testing.T) { + testplanet.Run(t, testplanet.Config{ + SatelliteCount: 1, + StorageNodeCount: 4, + UplinkCount: 1, + }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { + uplinkExe := ctx.Compile("storj.io/storj/cmd/uplink") + + // Configure uplink. + { + output, err := exec.Command(uplinkExe, + "--config-dir", ctx.Dir("uplink"), + "setup", + "--non-interactive", + "--scope", planet.Uplinks[0].GetConfig(planet.Satellites[0]).Scope, + ).CombinedOutput() + t.Log(string(output)) + require.NoError(t, err) + } + + // Create bucket. + bucketName := testrand.BucketName() + + { + output, err := exec.Command(uplinkExe, + "--config-dir", ctx.Dir("uplink"), + "mb", + "sj://"+bucketName, + ).CombinedOutput() + t.Log(string(output)) + require.NoError(t, err) + } + + // Upload file with metadata. + metadata := testrand.Metadata() + + metadataBs, err := json.Marshal(metadata) + require.NoError(t, err) + + metadataStr := string(metadataBs) + + var metadataNorm map[string]string + err = json.Unmarshal(metadataBs, &metadataNorm) + require.NoError(t, err) + + path := testrand.Path() + uri := "sj://" + bucketName + "/" + path + + { + output, err := exec.Command(uplinkExe, + "--config-dir", ctx.Dir("uplink"), + "cp", + "--metadata", metadataStr, + "-", uri, + ).CombinedOutput() + t.Log(string(output)) + require.NoError(t, err) + } + + // Get all metadata. + { + cmd := exec.Command(uplinkExe, + "--config-dir", ctx.Dir("uplink"), + "meta", "get", uri, + ) + t.Log(cmd) + + output, err := cmd.Output() + t.Log(string(output)) + if !assert.NoError(t, err) { + if ee, ok := err.(*exec.ExitError); ok { + t.Log(ee) + t.Log(string(ee.Stderr)) + } + + return + } + + var md map[string]string + err = json.Unmarshal(output, &md) + require.NoError(t, err) + + assert.Equal(t, metadataNorm, md) + } + + // Get specific metadata. + // + // NOTE: The CLI expects JSON encoded strings for input and + // output. The key and value returned from the CLI have to be + // converted from the JSON encoded string into the Go native + // string for comparison. + for key, value := range metadataNorm { + key, value := key, value + + t.Run(fmt.Sprintf("Fetching key %q", key[:min(len(key), 8)]), func(t *testing.T) { + keyNorm, err := json.Marshal(key) + require.NoError(t, err) + + cmd := exec.Command(uplinkExe, + "--config-dir", ctx.Dir("uplink"), + "meta", "get", string(keyNorm[1:len(keyNorm)-1]), uri, + ) + t.Log(cmd) + + output, err := cmd.Output() + assert.NoError(t, err) + if err != nil { + if ee, ok := err.(*exec.ExitError); ok { + t.Log(ee) + t.Log(string(ee.Stderr)) + } + + return + } + + // Remove trailing newline. + if len(output) > 0 && string(output[len(output)-1]) == "\n" { + output = output[:len(output)-1] + } + + var outputNorm string + err = json.Unmarshal([]byte("\""+string(output)+"\""), &outputNorm) + require.NoError(t, err) + + assert.Equal(t, value, outputNorm) + }) + } + }) +} diff --git a/internal/testrand/rand.go b/internal/testrand/rand.go index 049f7c2b9..a859cec31 100644 --- a/internal/testrand/rand.go +++ b/internal/testrand/rand.go @@ -115,3 +115,92 @@ func UUID() uuid.UUID { Read(uuid[:]) return uuid } + +// BucketName creates a random bucket name mostly confirming to the +// restrictions of S3: +// https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html +// +// NOTE: This may not generate values that cover all valid values (for Storj or +// S3). This is a best effort to cover most cases we believe our design +// requires and will need to be revisited when a more explicit design spec is +// created. +func BucketName() string { + const ( + edges = "abcdefghijklmnopqrstuvwxyz0123456789" + body = "abcdefghijklmnopqrstuvwxyz0123456789-" + min = 3 + max = 63 + ) + + size := rand.Intn(max-min) + min + + b := make([]byte, size) + for i := range b { + switch i { + case 0: + fallthrough + case size - 1: + b[i] = edges[rand.Intn(len(edges))] + default: + b[i] = body[rand.Intn(len(body))] + } + } + + return string(b) +} + +// Metadata creates random metadata mostly conforming to the restrictions of S3: +// https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#object-metadata +// +// NOTE: This may not generate values that cover all valid values (for Storj or +// S3). This is a best effort to cover most cases we believe our design +// requires and will need to be revisited when a more explicit design spec is +// created. +func Metadata() map[string]string { + const ( + max = 2 * 1024 + ) + + total := rand.Intn(max) + metadata := make(map[string]string) + + for used := 0; total-used > 1; { + keySize := rand.Intn(total-(used+1)) + 1 + key := BytesInt(keySize) + used += len(key) + + valueSize := rand.Intn(total - used) + value := BytesInt(valueSize) + used += len(value) + + metadata[string(key)] = string(value) + } + + return metadata +} + +// Path creates a random path mostly conforming to the retrictions of S3: +// https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#object-keys +// +// NOTE: This may not generate values that cover all valid values (for Storj or +// S3). This is a best effort to cover most cases we believe our design +// requires and will need to be revisited when a more explicit design spec is +// created. +func Path() string { + const ( + max = 1 * 1024 + ) + + total := rand.Intn(max) + path := "" + + for used := 0; len(path) < total; { + if used != 0 { + path += "/" + } + + path += SegmentID(rand.Intn(total - used)).String() + } + + return path[:total] +}