storj/lib/uplink/project.go

260 lines
8.3 KiB
Go
Raw Normal View History

// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package uplink
import (
"context"
"github.com/skyrings/skyring-common/tools/uuid"
"github.com/vivint/infectious"
"storj.io/common/encryption"
"storj.io/common/memory"
"storj.io/common/rpc"
"storj.io/common/storj"
"storj.io/uplink/ecclient"
"storj.io/uplink/eestream"
"storj.io/uplink/metainfo"
"storj.io/uplink/metainfo/kvmetainfo"
"storj.io/uplink/storage/segments"
"storj.io/uplink/storage/streams"
)
// Project represents a specific project access session.
type Project struct {
uplinkCfg *Config
dialer rpc.Dialer
metainfo *metainfo.Client
project *kvmetainfo.Project
}
// BucketConfig holds information about a bucket's configuration. This is
// filled in by the caller for use with CreateBucket(), or filled in by the
// library as Bucket.Config when a bucket is returned from OpenBucket().
type BucketConfig struct {
// PathCipher indicates which cipher suite is to be used for path
// encryption within the new Bucket. If not set, AES-GCM encryption
// will be used.
PathCipher storj.CipherSuite
// EncryptionParameters specifies the default encryption parameters to
// be used for data encryption of new Objects in this bucket.
EncryptionParameters storj.EncryptionParameters
// Volatile groups config values that are likely to change semantics
// or go away entirely between releases. Be careful when using them!
Volatile struct {
// RedundancyScheme defines the default Reed-Solomon and/or
// Forward Error Correction encoding parameters to be used by
// objects in this Bucket.
RedundancyScheme storj.RedundancyScheme
// SegmentsSize is the default segment size to use for new
// objects in this Bucket.
SegmentsSize memory.Size
}
}
2019-04-26 18:15:41 +01:00
func (cfg *BucketConfig) clone() *BucketConfig {
clone := *cfg
return &clone
}
// TODO: is this the best way to do this?
2019-04-26 18:15:41 +01:00
func (cfg *BucketConfig) setDefaults() {
if cfg.PathCipher == storj.EncUnspecified {
cfg.PathCipher = defaultCipher
}
2019-04-26 18:15:41 +01:00
if cfg.EncryptionParameters.CipherSuite == storj.EncUnspecified {
cfg.EncryptionParameters.CipherSuite = defaultCipher
}
2019-04-26 18:15:41 +01:00
if cfg.Volatile.RedundancyScheme.RequiredShares == 0 {
cfg.Volatile.RedundancyScheme.RequiredShares = 29
}
2019-04-26 18:15:41 +01:00
if cfg.Volatile.RedundancyScheme.RepairShares == 0 {
cfg.Volatile.RedundancyScheme.RepairShares = 35
}
2019-04-26 18:15:41 +01:00
if cfg.Volatile.RedundancyScheme.OptimalShares == 0 {
cfg.Volatile.RedundancyScheme.OptimalShares = 80
}
2019-04-26 18:15:41 +01:00
if cfg.Volatile.RedundancyScheme.TotalShares == 0 {
cfg.Volatile.RedundancyScheme.TotalShares = 95
}
2019-04-26 18:15:41 +01:00
if cfg.Volatile.RedundancyScheme.ShareSize == 0 {
cfg.Volatile.RedundancyScheme.ShareSize = 256 * memory.B.Int32()
}
2019-06-20 22:50:13 +01:00
if cfg.EncryptionParameters.BlockSize == 0 {
cfg.EncryptionParameters.BlockSize = cfg.Volatile.RedundancyScheme.ShareSize * int32(cfg.Volatile.RedundancyScheme.RequiredShares)
}
2019-04-26 18:15:41 +01:00
if cfg.Volatile.SegmentsSize.Int() == 0 {
cfg.Volatile.SegmentsSize = 64 * memory.MiB
}
}
// CreateBucket creates a new bucket if authorized.
func (p *Project) CreateBucket(ctx context.Context, name string, cfg *BucketConfig) (bucket storj.Bucket, err error) {
defer mon.Task()(&ctx)(&err)
if cfg == nil {
cfg = &BucketConfig{}
}
2019-04-26 18:15:41 +01:00
cfg = cfg.clone()
cfg.setDefaults()
var partnerID uuid.UUID
if p.uplinkCfg.Volatile.PartnerID != "" {
id, err := uuid.Parse(p.uplinkCfg.Volatile.PartnerID)
if err != nil {
return storj.Bucket{}, Error.Wrap(err)
}
partnerID = *id
}
bucket = storj.Bucket{
PartnerID: partnerID,
PathCipher: cfg.PathCipher,
DefaultEncryptionParameters: cfg.EncryptionParameters,
DefaultRedundancyScheme: cfg.Volatile.RedundancyScheme,
DefaultSegmentsSize: cfg.Volatile.SegmentsSize.Int64(),
}
return p.project.CreateBucket(ctx, name, &bucket)
}
// DeleteBucket deletes a bucket if authorized. If the bucket contains any
// Objects at the time of deletion, they may be lost permanently.
func (p *Project) DeleteBucket(ctx context.Context, bucket string) (err error) {
defer mon.Task()(&ctx)(&err)
return p.project.DeleteBucket(ctx, bucket)
}
// BucketListOptions controls options to the ListBuckets() call.
type BucketListOptions = storj.BucketListOptions
// ListBuckets will list authorized buckets.
func (p *Project) ListBuckets(ctx context.Context, opts *BucketListOptions) (bl storj.BucketList, err error) {
defer mon.Task()(&ctx)(&err)
if opts == nil {
opts = &BucketListOptions{Direction: storj.Forward}
}
return p.project.ListBuckets(ctx, *opts)
}
// GetBucketInfo returns info about the requested bucket if authorized.
func (p *Project) GetBucketInfo(ctx context.Context, bucket string) (b storj.Bucket, bi *BucketConfig, err error) {
defer mon.Task()(&ctx)(&err)
b, err = p.project.GetBucket(ctx, bucket)
if err != nil {
return b, nil, err
}
cfg := &BucketConfig{
PathCipher: b.PathCipher,
EncryptionParameters: b.DefaultEncryptionParameters,
}
cfg.Volatile.RedundancyScheme = b.DefaultRedundancyScheme
cfg.Volatile.SegmentsSize = memory.Size(b.DefaultSegmentsSize)
return b, cfg, nil
}
2019-06-25 16:36:23 +01:00
// TODO: move the bucket related OpenBucket to bucket.go
// OpenBucket returns a Bucket handle with the given EncryptionAccess
// information.
func (p *Project) OpenBucket(ctx context.Context, bucketName string, access *EncryptionAccess) (b *Bucket, err error) {
defer mon.Task()(&ctx)(&err)
bucketInfo, cfg, err := p.GetBucketInfo(ctx, bucketName)
if err != nil {
return nil, err
}
// partnerID set and bucket's attribution is not set
if bucketInfo.PartnerID.IsZero() {
err = p.trySetBucketAttribution(ctx, bucketName)
if err != nil {
return nil, err
}
}
encryptionParameters := cfg.EncryptionParameters
ec := ecclient.NewClient(p.uplinkCfg.Volatile.Log.Named("ecclient"), p.dialer, p.uplinkCfg.Volatile.MaxMemory.Int())
fc, err := infectious.NewFEC(int(cfg.Volatile.RedundancyScheme.RequiredShares), int(cfg.Volatile.RedundancyScheme.TotalShares))
if err != nil {
return nil, err
}
rs, err := eestream.NewRedundancyStrategy(
eestream.NewRSScheme(fc, int(cfg.Volatile.RedundancyScheme.ShareSize)),
int(cfg.Volatile.RedundancyScheme.RepairShares),
int(cfg.Volatile.RedundancyScheme.OptimalShares))
if err != nil {
return nil, err
}
maxEncryptedSegmentSize, err := encryption.CalcEncryptedSize(cfg.Volatile.SegmentsSize.Int64(),
cfg.EncryptionParameters)
if err != nil {
return nil, err
}
segmentStore := segments.NewSegmentStore(p.metainfo, ec, rs)
streamStore, err := streams.NewStreamStore(p.metainfo, segmentStore, cfg.Volatile.SegmentsSize.Int64(), access.store, int(encryptionParameters.BlockSize), encryptionParameters.CipherSuite, p.uplinkCfg.Volatile.MaxInlineSize.Int(), maxEncryptedSegmentSize)
if err != nil {
return nil, err
}
return &Bucket{
BucketConfig: *cfg,
Name: bucketInfo.Name,
Created: bucketInfo.Created,
bucket: bucketInfo,
metainfo: kvmetainfo.New(p.project, p.metainfo, streamStore, segmentStore, access.store),
streams: streamStore,
}, nil
}
lib/uplink: encryption context (#2349) * lib/uplink: encryption context Change-Id: I5c23dca3286a46b713b30c4997e9ae6e630b2280 * lib/uplink: bucket operation examples Change-Id: Ia0f6e69f365dcff0cf11c731f51b30842bce053b * lib/uplink: encryption key sharing test cases Change-Id: I3a172d565f33f4e591402cdcb9460664a7cc7fbe * fix encrypted path prefix restriction issue Change-Id: I8f3921f9d52aaf4b84039de608b8cbbc88769554 * implement panics in libuplink encryption code todo on cipher suite selection as well as an api concern Change-Id: Ifa39eb3cc4b3443f7d96f9304df9b2ac4ec4085d * implement GetProjectInfo api call to get salt Change-Id: Ic5f6b3be9ea35df48c1aa214ab5d355fb328e2cf * some fixes and accessors for encryption store Change-Id: I3bb61f6712a037900e2a96e72ad4029ec1d3f718 * general fixes to builds/tests/etc Change-Id: I9930fa96acb3b221d9a001f8e274af5729cc8a47 * java bindings changes Change-Id: Ia2bd4c9c69739c8d3154d79616cff1f36fb403b6 * get libuplink examples passing Change-Id: I828f09a144160e0a5dd932324f78491ae2ec8a07 * fix proto.lock file Change-Id: I2fbbf4d0976a7d0473c2645e6dcb21aaa3be7651 * fix proto.lock again Change-Id: I92702cf49e1a340eef6379c2be4f7c4a268112a9 * fix golint issues Change-Id: I631ff9f43307a58e3b25a58cbb4a4cc2495f5eb6 * more linting fixes Change-Id: I51f8f30b367b5bca14c94b15417b9a4c9e7aa0ce * bug fixed by structs bump Change-Id: Ibb03c691fce7606c35c08721b3ef0781ab48a38a * retrigger Change-Id: Ieee0470b6a2d07168a1578552e8e7f271ae93a13 * retrigger Change-Id: I753d63853171e6a436c104ce176048892eb974c5 * semantic merge conflict Change-Id: I9419448496de90340569047a6a16a1b858a7978a * update total to match prod defaults Change-Id: I693d55c1ebb28b5803ee1d26e9e198decf82308b * retrigger Change-Id: I28b74d5d6202f61aa3866fe407d423f6a0a14b9e * retrigger Change-Id: I6fd054885c715f602e2cef623fd464c42e88742c * retrigger Change-Id: I6a01bae88c72406d4ed5a8f13bf8a2b3c650bd2d
2019-06-27 18:36:51 +01:00
func (p *Project) retrieveSalt(ctx context.Context) (salt []byte, err error) {
defer mon.Task()(&ctx)(&err)
info, err := p.metainfo.GetProjectInfo(ctx)
if err != nil {
return nil, err
}
return info.ProjectSalt, nil
}
// SaltedKeyFromPassphrase returns a key generated from the given passphrase using a stable, project-specific salt
func (p *Project) SaltedKeyFromPassphrase(ctx context.Context, passphrase string) (_ *storj.Key, err error) {
defer mon.Task()(&ctx)(&err)
salt, err := p.retrieveSalt(ctx)
if err != nil {
return nil, err
}
key, err := encryption.DeriveRootKey([]byte(passphrase), salt, "", uint8(p.uplinkCfg.Volatile.PBKDFConcurrency))
lib/uplink: encryption context (#2349) * lib/uplink: encryption context Change-Id: I5c23dca3286a46b713b30c4997e9ae6e630b2280 * lib/uplink: bucket operation examples Change-Id: Ia0f6e69f365dcff0cf11c731f51b30842bce053b * lib/uplink: encryption key sharing test cases Change-Id: I3a172d565f33f4e591402cdcb9460664a7cc7fbe * fix encrypted path prefix restriction issue Change-Id: I8f3921f9d52aaf4b84039de608b8cbbc88769554 * implement panics in libuplink encryption code todo on cipher suite selection as well as an api concern Change-Id: Ifa39eb3cc4b3443f7d96f9304df9b2ac4ec4085d * implement GetProjectInfo api call to get salt Change-Id: Ic5f6b3be9ea35df48c1aa214ab5d355fb328e2cf * some fixes and accessors for encryption store Change-Id: I3bb61f6712a037900e2a96e72ad4029ec1d3f718 * general fixes to builds/tests/etc Change-Id: I9930fa96acb3b221d9a001f8e274af5729cc8a47 * java bindings changes Change-Id: Ia2bd4c9c69739c8d3154d79616cff1f36fb403b6 * get libuplink examples passing Change-Id: I828f09a144160e0a5dd932324f78491ae2ec8a07 * fix proto.lock file Change-Id: I2fbbf4d0976a7d0473c2645e6dcb21aaa3be7651 * fix proto.lock again Change-Id: I92702cf49e1a340eef6379c2be4f7c4a268112a9 * fix golint issues Change-Id: I631ff9f43307a58e3b25a58cbb4a4cc2495f5eb6 * more linting fixes Change-Id: I51f8f30b367b5bca14c94b15417b9a4c9e7aa0ce * bug fixed by structs bump Change-Id: Ibb03c691fce7606c35c08721b3ef0781ab48a38a * retrigger Change-Id: Ieee0470b6a2d07168a1578552e8e7f271ae93a13 * retrigger Change-Id: I753d63853171e6a436c104ce176048892eb974c5 * semantic merge conflict Change-Id: I9419448496de90340569047a6a16a1b858a7978a * update total to match prod defaults Change-Id: I693d55c1ebb28b5803ee1d26e9e198decf82308b * retrigger Change-Id: I28b74d5d6202f61aa3866fe407d423f6a0a14b9e * retrigger Change-Id: I6fd054885c715f602e2cef623fd464c42e88742c * retrigger Change-Id: I6a01bae88c72406d4ed5a8f13bf8a2b3c650bd2d
2019-06-27 18:36:51 +01:00
if err != nil {
return nil, err
}
return key, nil
lib/uplink: encryption context (#2349) * lib/uplink: encryption context Change-Id: I5c23dca3286a46b713b30c4997e9ae6e630b2280 * lib/uplink: bucket operation examples Change-Id: Ia0f6e69f365dcff0cf11c731f51b30842bce053b * lib/uplink: encryption key sharing test cases Change-Id: I3a172d565f33f4e591402cdcb9460664a7cc7fbe * fix encrypted path prefix restriction issue Change-Id: I8f3921f9d52aaf4b84039de608b8cbbc88769554 * implement panics in libuplink encryption code todo on cipher suite selection as well as an api concern Change-Id: Ifa39eb3cc4b3443f7d96f9304df9b2ac4ec4085d * implement GetProjectInfo api call to get salt Change-Id: Ic5f6b3be9ea35df48c1aa214ab5d355fb328e2cf * some fixes and accessors for encryption store Change-Id: I3bb61f6712a037900e2a96e72ad4029ec1d3f718 * general fixes to builds/tests/etc Change-Id: I9930fa96acb3b221d9a001f8e274af5729cc8a47 * java bindings changes Change-Id: Ia2bd4c9c69739c8d3154d79616cff1f36fb403b6 * get libuplink examples passing Change-Id: I828f09a144160e0a5dd932324f78491ae2ec8a07 * fix proto.lock file Change-Id: I2fbbf4d0976a7d0473c2645e6dcb21aaa3be7651 * fix proto.lock again Change-Id: I92702cf49e1a340eef6379c2be4f7c4a268112a9 * fix golint issues Change-Id: I631ff9f43307a58e3b25a58cbb4a4cc2495f5eb6 * more linting fixes Change-Id: I51f8f30b367b5bca14c94b15417b9a4c9e7aa0ce * bug fixed by structs bump Change-Id: Ibb03c691fce7606c35c08721b3ef0781ab48a38a * retrigger Change-Id: Ieee0470b6a2d07168a1578552e8e7f271ae93a13 * retrigger Change-Id: I753d63853171e6a436c104ce176048892eb974c5 * semantic merge conflict Change-Id: I9419448496de90340569047a6a16a1b858a7978a * update total to match prod defaults Change-Id: I693d55c1ebb28b5803ee1d26e9e198decf82308b * retrigger Change-Id: I28b74d5d6202f61aa3866fe407d423f6a0a14b9e * retrigger Change-Id: I6fd054885c715f602e2cef623fd464c42e88742c * retrigger Change-Id: I6a01bae88c72406d4ed5a8f13bf8a2b3c650bd2d
2019-06-27 18:36:51 +01:00
}
// trySetBucketAttribution sets the bucket attribution if PartnerID or UserAgent is available.
func (p *Project) trySetBucketAttribution(ctx context.Context, bucketName string) (err error) {
defer mon.Task()(&ctx)(&err)
if p.uplinkCfg.Volatile.PartnerID == "" && p.uplinkCfg.Volatile.UserAgent == "" {
return nil
}
var partnerID uuid.UUID
if p.uplinkCfg.Volatile.PartnerID != "" {
id, err := uuid.Parse(p.uplinkCfg.Volatile.PartnerID)
if err != nil {
return Error.Wrap(err)
}
partnerID = *id
}
// UserAgent is sent via RequestHeader
return p.metainfo.SetBucketAttribution(ctx, metainfo.SetBucketAttributionParams{
Bucket: bucketName,
PartnerID: partnerID,
})
}