JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

lib/uplink: deprecate docs

Browse Source 
guess what docs you get if you google "storj uplink"?

Change-Id: I600f1811f74a516d224189872f97727d57a153b1
This commit is contained in:
JT Olio 2020-04-13 13:00:21 -06:00
parent e35cbfb6ee
commit d08b2de558
7 changed files with 4 additions and 747 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
lib/uplink/doc.go
View File

@ -2,119 +2,9 @@
// See LICENSE for copying information.
/*
Package uplink is the main entrypoint to interacting with Storj Labs' decentralized
storage network.
Projects
An (*Uplink) reference lets you open a *Project, which should have already been created via
the web interface of one of the Storj Labs or Tardigrade network Satellites. You may be able
to create or access your us-central-1 account here: https://us-central-1.tardigrade.io/
Opening a *Project requires a specific Satellite address (e.g. "us-central-1.tardigrade.io:7777")
and an API key. The API key will grant specific access to certain operations and resources within
a project. Projects allow you to manage and open Buckets.
Example:
ul, err := uplink.NewUplink(ctx, nil)
if err != nil {
return err
}
defer ul.Close()
p, err := ul.OpenProject(ctx, "us-central-1.tardigrade.io:7777", apiKey)
if err != nil {
return err
}
defer p.Close()
API Keys
An API key is a "macaroon" (see https://ai.google/research/pubs/pub41892). As such, API keys
can be restricted such that users of the restricted API key only have access to a subset of
what the parent API key allowed. It is possible to restrict a macaroon to specific operations,
buckets, paths, path prefixes, or time windows.
If you need a valid API key, please visit your chosen Satellite's web interface.
Example:
adminKey, err := uplink.ParseAPIKey("13YqeJ3Xk4KHocypZMdQZZqfC1goMvxbYSCWWEjSmew6rVvJp3GCK")
if err != nil {
return "", err
}
readOnlyKey, err := adminKey.Restrict(macaroon.Caveat{
DisallowWrites: true,
DisallowLists: true,
DisallowDeletes: true,
})
if err != nil {
return "", err
}
// return a new restricted key that is read only
return readOnlyKey.Serialize()
Restricting an API key to a path prefix is most easily accomplished using an
EncryptionAccess, so see EncryptionAccess for more.
Buckets
A bucket represents a collection of objects. You can upload, download, list, and delete objects of
any size or shape. Objects within buckets are represented by keys, where keys can optionally be
listed using the "/" delimiter. Objects are always end-to-end encrypted.
b, err := p.OpenBucket(ctx, "staging", access)
if err != nil {
return err
}
defer b.Close()
EncryptionAccess
Where an APIKey controls what resources and operations a Satellite will allow a user to access
and perform, an EncryptionAccess controls what buckets, path prefixes, and objects a user has the
ability to decrypt. An EncryptionAccess is a serializable collection of hierarchically-determined
encryption keys, where by default the key starts at the root.
As an example, the following code creates an encryption access context (and API key) that is
restricted to objects with the prefix "/logs/" inside the staging bucket.
access := uplink.NewEncryptionAccessWithDefaultKey(defaultKey)
logServerKey, logServerAccess, err := access.Restrict(
readOnlyKey, uplink.EncryptionRestriction{
Bucket: "staging",
Path: "/logs/",
})
if err != nil {
return "", err
}
return logServerAccess.Serialize()
The keys to decrypt data in other buckets or in other path prefixes are not contained in this
new serialized encryption access context. This new encryption access context only provides the
information for just what is necessary.
Objects
Objects support a couple kilobytes of arbitrary key/value metadata, an arbitrary-size primary
data streams, with seeking. If you want to access only a small subrange of the data you
uploaded, you can download only the range of the data you need in a fast and performant way.
This allows you to stream video straight out of the network with little overhead.
obj, err := b.OpenObject(ctx, "/logs/webserver.log")
if err != nil {
return err
}
defer obj.Close()
reader, err := obj.DownloadRange(ctx, 0, -1)
if err != nil {
return err
}
defer reader.Close()
Package uplink (this specific package) is deprecated and kept for backwards
compatibility only. You're almost certainly looking for the newer
storj.io/uplink package, where docs can be found at
https://pkg.go.dev/storj.io/uplink
*/
package uplink

67
lib/uplink/ex_create_bucket_test.go
View File

@ -1,67 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package uplink_test
import (
"context"
"fmt"
"io"
"os"
"github.com/zeebo/errs"
"storj.io/storj/lib/uplink"
)
func CreateBucketExample(ctx context.Context,
satelliteAddress, apiKey string,
cfg *uplink.Config, out io.Writer) (err error) {
errCatch := func(fn func() error) { err = errs.Combine(err, fn()) }
// First, create an Uplink handle.
ul, err := uplink.NewUplink(ctx, cfg)
if err != nil {
return err
}
defer errCatch(ul.Close)
// Then, parse the API key. API keys are "macaroons" that allow you to create
// new, restricted API keys.
key, err := uplink.ParseAPIKey(apiKey)
if err != nil {
return err
}
// Next, open the project in question. Projects are identified by a specific
// Satellite and API key
p, err := ul.OpenProject(ctx, satelliteAddress, key)
if err != nil {
return err
}
defer errCatch(p.Close)
// Last, create the bucket!
_, err = p.CreateBucket(ctx, "testbucket", nil)
if err != nil {
return err
}
fmt.Fprintln(out, "success!")
return nil
}
func Example_createBucket() {
// The satellite address is the address of the satellite your API key is
// valid on
satelliteAddress := "us-central-1.tardigrade.io:7777"
// The API key can be created in the web interface
apiKey := "qPSUM3k0bZyOIyil2xrVWiSuc9HuB2yBP3qDrA2Gc"
err := CreateBucketExample(context.Background(), satelliteAddress, apiKey, &uplink.Config{}, os.Stdout)
if err != nil {
panic(err)
}
}

68
lib/uplink/ex_delete_bucket_test.go
View File

@ -1,68 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package uplink_test
import (
"context"
"fmt"
"io"
"os"
"github.com/zeebo/errs"
"storj.io/storj/lib/uplink"
)
func DeleteBucketExample(ctx context.Context,
satelliteAddress, apiKey string,
cfg *uplink.Config, out io.Writer) (err error) {
errCatch := func(fn func() error) { err = errs.Combine(err, fn()) }
// First, create an Uplink handle.
ul, err := uplink.NewUplink(ctx, cfg)
if err != nil {
return err
}
defer errCatch(ul.Close)
// Then, parse the API key. API keys are "macaroons" that allow you to create
// new, restricted API keys.
key, err := uplink.ParseAPIKey(apiKey)
if err != nil {
return err
}
// Next, open the project in question. Projects are identified by a specific
// Satellite and API key
p, err := ul.OpenProject(ctx, satelliteAddress, key)
if err != nil {
return err
}
defer errCatch(p.Close)
// Last, delete the bucket!
err = p.DeleteBucket(ctx, "testbucket")
if err != nil {
return err
}
fmt.Fprintln(out, "success!")
return nil
}
func Example_deleteBucket() {
// The satellite address is the address of the satellite your API key is
// valid on
satelliteAddress := "us-central-1.tardigrade.io:7777"
// The API key can be created in the web interface
apiKey := "qPSUM3k0bZyOIyil2xrVWiSuc9HuB2yBP3qDrA2Gc"
err := DeleteBucketExample(context.Background(), satelliteAddress, apiKey,
&uplink.Config{}, os.Stdout)
if err != nil {
panic(err)
}
}

80
lib/uplink/ex_list_buckets_test.go
View File

@ -1,80 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package uplink_test
import (
"context"
"fmt"
"io"
"os"
"github.com/zeebo/errs"
"storj.io/common/storj"
"storj.io/storj/lib/uplink"
)
func ListBucketsExample(ctx context.Context,
satelliteAddress, apiKey string,
cfg *uplink.Config, out io.Writer) (err error) {
errCatch := func(fn func() error) { err = errs.Combine(err, fn()) }
// First, create an Uplink handle.
ul, err := uplink.NewUplink(ctx, cfg)
if err != nil {
return err
}
defer errCatch(ul.Close)
// Then, parse the API key. API keys are "macaroons" that allow you to create
// new, restricted API keys.
key, err := uplink.ParseAPIKey(apiKey)
if err != nil {
return err
}
// Next, open the project in question. Projects are identified by a specific
// Satellite and API key
p, err := ul.OpenProject(ctx, satelliteAddress, key)
if err != nil {
return err
}
defer errCatch(p.Close)
// Last, list the buckets! Bucket listing is paginated, so you'll need to
// use pagination.
list := uplink.BucketListOptions{
Direction: storj.Forward}
for {
result, err := p.ListBuckets(ctx, &list)
if err != nil {
return err
}
for _, bucket := range result.Items {
fmt.Fprintf(out, "Bucket: %v\n", bucket.Name)
}
if !result.More {
break
}
list = list.NextPage(result)
}
return nil
}
func Example_listBuckets() {
// The satellite address is the address of the satellite your API key is
// valid on
satelliteAddress := "us-central-1.tardigrade.io:7777"
// The API key can be created in the web interface
apiKey := "qPSUM3k0bZyOIyil2xrVWiSuc9HuB2yBP3qDrA2Gc"
err := ListBucketsExample(context.Background(), satelliteAddress, apiKey,
&uplink.Config{}, os.Stdout)
if err != nil {
panic(err)
}
}

179
lib/uplink/ex_password_test.go
View File

@ -1,179 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package uplink_test
import (
"context"
"fmt"
"io"
"io/ioutil"
"os"
"strings"
"github.com/zeebo/errs"
"storj.io/common/storj"
"storj.io/storj/lib/uplink"
)
func CreateEncryptionKeyExampleByAdmin1(ctx context.Context,
satelliteAddress, apiKey string, cfg *uplink.Config, out io.Writer) (
serializedAccess string, err error) {
errCatch := func(fn func() error) { err = errs.Combine(err, fn()) }
// First, create an Uplink handle.
ul, err := uplink.NewUplink(ctx, cfg)
if err != nil {
return "", err
}
defer errCatch(ul.Close)
// Parse the API key. API keys are "macaroons" that allow you to create new,
// restricted API keys.
key, err := uplink.ParseAPIKey(apiKey)
if err != nil {
return "", err
}
// Open the project in question. Projects are identified by a specific
// Satellite and API key
p, err := ul.OpenProject(ctx, satelliteAddress, key)
if err != nil {
return "", err
}
defer errCatch(p.Close)
// Make a key
encKey, err := p.SaltedKeyFromPassphrase(ctx, "my secret passphrase")
if err != nil {
return "", err
}
// Make an encryption context
access := uplink.NewEncryptionAccessWithDefaultKey(*encKey)
access.SetDefaultPathCipher(storj.EncAESGCM)
// serialize it
serializedAccess, err = access.Serialize()
if err != nil {
return "", err
}
// Create a bucket
_, err = p.CreateBucket(ctx, "prod", nil)
if err != nil {
return "", err
}
// Open bucket
bucket, err := p.OpenBucket(ctx, "prod", access)
if err != nil {
return "", err
}
defer errCatch(bucket.Close)
// Upload a file
err = bucket.UploadObject(ctx, "webserver/logs/log.txt",
strings.NewReader("hello world"), nil)
if err != nil {
return "", err
}
fmt.Fprintln(out, "success!")
return serializedAccess, nil
}
func CreateEncryptionKeyExampleByAdmin2(ctx context.Context,
satelliteAddress, apiKey string, serializedAccess string,
cfg *uplink.Config, out io.Writer) (err error) {
errCatch := func(fn func() error) { err = errs.Combine(err, fn()) }
// First, create an Uplink handle.
ul, err := uplink.NewUplink(ctx, cfg)
if err != nil {
return err
}
defer errCatch(ul.Close)
// Parse the API key. API keys are "macaroons" that allow you to create new,
// restricted API keys.
key, err := uplink.ParseAPIKey(apiKey)
if err != nil {
return err
}
// Open the project in question. Projects are identified by a specific
// Satellite and API key
p, err := ul.OpenProject(ctx, satelliteAddress, key)
if err != nil {
return err
}
defer errCatch(p.Close)
// Parse the encryption context
access, err := uplink.ParseEncryptionAccess(serializedAccess)
if err != nil {
return err
}
// Open bucket
bucket, err := p.OpenBucket(ctx, "prod", access)
if err != nil {
return err
}
defer errCatch(bucket.Close)
// Open file
obj, err := bucket.OpenObject(ctx, "webserver/logs/log.txt")
if err != nil {
return err
}
defer errCatch(obj.Close)
// Get a reader for the entire file
r, err := obj.DownloadRange(ctx, 0, -1)
if err != nil {
return err
}
defer errCatch(r.Close)
// Read the file
data, err := ioutil.ReadAll(r)
if err != nil {
return err
}
// Print it!
fmt.Fprintln(out, string(data))
return nil
}
func Example_createEncryptionKey() {
// The satellite address is the address of the satellite your API key is
// valid on
satelliteAddress := "us-central-1.tardigrade.io:7777"
// The API key can be created in the web interface
admin1APIKey := "qPSUM3k0bZyOIyil2xrVWiSuc9HuB2yBP3qDrA2Gc"
admin2APIKey := "udP0lzCC2rgwRZfdY70PcwWrXzrq9cl5usbiFaeyo"
ctx := context.Background()
// Admin1 is going to create an encryption context and share it
access, err := CreateEncryptionKeyExampleByAdmin1(ctx, satelliteAddress,
admin1APIKey, &uplink.Config{}, os.Stdout)
if err != nil {
panic(err)
}
// Admin2 is going to use the provided encryption context to load the
// uploaded file
err = CreateEncryptionKeyExampleByAdmin2(ctx, satelliteAddress,
admin2APIKey, access, &uplink.Config{}, os.Stdout)
if err != nil {
panic(err)
}
}

163
lib/uplink/ex_restrict_test.go
View File

@ -1,163 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package uplink_test
import (
"context"
"fmt"
"io"
"io/ioutil"
"os"
"github.com/zeebo/errs"
"storj.io/common/macaroon"
"storj.io/storj/lib/uplink"
)
func RestrictAccessExampleByAdmin(ctx context.Context,
satelliteAddress, apiKey, adminAccess string,
cfg *uplink.Config, out io.Writer) (
serializedScope string, err error) {
// Parse the API key. API keys are "macaroons" that allow you to create new,
// restricted API keys.
key, err := uplink.ParseAPIKey(apiKey)
if err != nil {
return "", err
}
// Restrict the API key to be read only and to be for just the prod and
// staging buckets for the path webserver/logs/
userAPIKey, err := key.Restrict(macaroon.Caveat{
DisallowWrites: true,
DisallowDeletes: true,
})
if err != nil {
return "", err
}
// Load the existing encryption access context
access, err := uplink.ParseEncryptionAccess(adminAccess)
if err != nil {
return "", err
}
// Restrict the encryption access context to just the prod and staging
// buckets for webserver/logs/
userAPIKey, userAccess, err := access.Restrict(userAPIKey,
uplink.EncryptionRestriction{
Bucket: "prod",
PathPrefix: "webserver/logs",
},
uplink.EncryptionRestriction{
Bucket: "staging",
PathPrefix: "webserver/logs",
},
)
if err != nil {
return "", err
}
userScope := &uplink.Scope{
SatelliteAddr: satelliteAddress,
APIKey: userAPIKey,
EncryptionAccess: userAccess,
}
// Serialize the scope
serializedScope, err = userScope.Serialize()
if err != nil {
return "", err
}
fmt.Fprintln(out, "success!")
return serializedScope, nil
}
func RestrictAccessExampleByUser(ctx context.Context,
serializedScope string, cfg *uplink.Config, out io.Writer) (err error) {
errCatch := func(fn func() error) { err = errs.Combine(err, fn()) }
// First, create an Uplink handle.
ul, err := uplink.NewUplink(ctx, cfg)
if err != nil {
return err
}
defer errCatch(ul.Close)
// Parse the scope.
scope, err := uplink.ParseScope(serializedScope)
if err != nil {
return err
}
// Open the project in question. Projects are identified by a specific
// Satellite and API key
p, err := ul.OpenProject(ctx, scope.SatelliteAddr, scope.APIKey)
if err != nil {
return err
}
defer errCatch(p.Close)
// Open bucket
bucket, err := p.OpenBucket(ctx, "prod", scope.EncryptionAccess)
if err != nil {
return err
}
defer errCatch(bucket.Close)
// Open file
obj, err := bucket.OpenObject(ctx, "webserver/logs/log.txt")
if err != nil {
return err
}
defer errCatch(obj.Close)
// Get a reader for the entire file
r, err := obj.DownloadRange(ctx, 0, -1)
if err != nil {
return err
}
defer errCatch(r.Close)
// Read the file
data, err := ioutil.ReadAll(r)
if err != nil {
return err
}
// Print it!
fmt.Fprintln(out, string(data))
return nil
}
func Example_restrictAccess() {
// The satellite address is the address of the satellite your API key is
// valid on
satelliteAddress := "us-central-1.tardigrade.io:7777"
// The API key can be created in the web interface
adminAPIKey := "qPSUM3k0bZyOIyil2xrVWiSuc9HuB2yBP3qDrA2Gc"
// The encryption access context was created using
// NewEncryptionAccessWithDefaultKey and
// (*Project).SaltedKeyFromPassphrase() earlier
adminAccess := "HYGoqCEz43mCE40Hc5lQD3DtUYynx9Vo1GjOx75hQ"
ctx := context.Background()
// Admin1 is going to create a scope and share it
userScope, err := RestrictAccessExampleByAdmin(ctx, satelliteAddress,
adminAPIKey, adminAccess, &uplink.Config{}, os.Stdout)
if err != nil {
panic(err)
}
// Admin2 is going to use the provided scope to load the uploaded file
err = RestrictAccessExampleByUser(ctx, userScope, &uplink.Config{}, os.Stdout)
if err != nil {
panic(err)
}
}

76
lib/uplink/examples_test.go
View File

@ -1,76 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package uplink_test
import (
"bytes"
"testing"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
"storj.io/common/testcontext"
"storj.io/storj/lib/uplink"
"storj.io/storj/private/testplanet"
)
func TestBucketExamples(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1,
StorageNodeCount: 1,
UplinkCount: 1},
func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
cfg := uplink.Config{}
cfg.Volatile.Log = zaptest.NewLogger(t)
cfg.Volatile.TLS.SkipPeerCAWhitelist = true
satelliteAddr := planet.Satellites[0].Local().Address.Address
apiKey := planet.Uplinks[0].APIKey[planet.Satellites[0].ID()].Serialize()
out := bytes.NewBuffer(nil)
err := ListBucketsExample(ctx, satelliteAddr, apiKey, &cfg, out)
require.NoError(t, err)
require.Equal(t, out.String(), "")
out = bytes.NewBuffer(nil)
err = CreateBucketExample(ctx, satelliteAddr, apiKey, &cfg, out)
require.NoError(t, err)
require.Equal(t, out.String(), "success!\n")
out = bytes.NewBuffer(nil)
err = ListBucketsExample(ctx, satelliteAddr, apiKey, &cfg, out)
require.NoError(t, err)
require.Equal(t, out.String(), "Bucket: testbucket\n")
out = bytes.NewBuffer(nil)
err = DeleteBucketExample(ctx, satelliteAddr, apiKey, &cfg, out)
require.NoError(t, err)
require.Equal(t, out.String(), "success!\n")
out = bytes.NewBuffer(nil)
err = ListBucketsExample(ctx, satelliteAddr, apiKey, &cfg, out)
require.NoError(t, err)
require.Equal(t, out.String(), "")
out = bytes.NewBuffer(nil)
access, err := CreateEncryptionKeyExampleByAdmin1(ctx, satelliteAddr, apiKey, &cfg, out)
require.NoError(t, err)
require.Equal(t, out.String(), "success!\n")
out = bytes.NewBuffer(nil)
err = CreateEncryptionKeyExampleByAdmin2(ctx, satelliteAddr, apiKey, access, &cfg, out)
require.NoError(t, err)
require.Equal(t, out.String(), "hello world\n")
out = bytes.NewBuffer(nil)
userScope, err := RestrictAccessExampleByAdmin(ctx, satelliteAddr, apiKey, access, &cfg, out)
require.NoError(t, err)
require.Equal(t, out.String(), "success!\n")
out = bytes.NewBuffer(nil)
err = RestrictAccessExampleByUser(ctx, userScope, &cfg, out)
require.NoError(t, err)
require.Equal(t, out.String(), "hello world\n")
})
}