storj/private/testplanet/uplink.go

// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information

package testplanet

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"runtime/pprof"
	"strconv"
	"time"

	"github.com/zeebo/errs"
	"go.uber.org/zap"

	"storj.io/common/grant"
	"storj.io/common/identity"
	"storj.io/common/macaroon"
	"storj.io/common/peertls/tlsopts"
	"storj.io/common/rpc"
	"storj.io/common/storj"
	"storj.io/common/uuid"
	"storj.io/storj/satellite/console"
	"storj.io/uplink"
	"storj.io/uplink/private/metaclient"
	"storj.io/uplink/private/piecestore"
	"storj.io/uplink/private/testuplink"
)

// UplinkConfig testplanet configuration for uplink.
type UplinkConfig struct {
	DefaultPathCipher storj.CipherSuite
}

// Uplink is a registered user on all satellites,
// which contains the necessary accesses and project info.
type Uplink struct {
	Log      *zap.Logger
	Identity *identity.FullIdentity
	Dialer   rpc.Dialer
	Config   uplink.Config

	APIKey map[storj.NodeID]*macaroon.APIKey
	Access map[storj.NodeID]*uplink.Access
	User   map[storj.NodeID]UserLogin

	// Projects is indexed by the satellite number.
	Projects []*Project
}

// Project contains all necessary information about a user.
type Project struct {
	client *Uplink

	ID    uuid.UUID
	Owner ProjectOwner

	Satellite Peer
	APIKey    string

	RawAPIKey *macaroon.APIKey
}

// ProjectOwner contains information about the project owner.
type ProjectOwner struct {
	ID    uuid.UUID
	Email string
}

// UserLogin contains information about the user login.
type UserLogin struct {
	Email    string
	Password string
}

// DialMetainfo dials the satellite with the appropriate api key.
func (project *Project) DialMetainfo(ctx context.Context) (_ *metaclient.Client, err error) {
	defer mon.Task()(&ctx)(&err)
	return project.client.DialMetainfo(ctx, project.Satellite, project.RawAPIKey)
}

// newUplinks creates initializes uplinks, requires peer to have at least one satellite.
func (planet *Planet) newUplinks(ctx context.Context, prefix string, count int) (_ []*Uplink, err error) {
	defer mon.Task()(&ctx)(&err)

	var xs []*Uplink
	for i := 0; i < count; i++ {
		name := prefix + strconv.Itoa(i)

		log := planet.log.Named(name)

		var uplink *Uplink
		var err error
		pprof.Do(ctx, pprof.Labels("peer", name), func(ctx context.Context) {
			uplink, err = planet.newUplink(ctx, i, log, name)
		})
		if err != nil {
			return nil, errs.Wrap(err)
		}
		xs = append(xs, uplink)
	}

	return xs, nil
}

// newUplink creates a new uplink.
func (planet *Planet) newUplink(ctx context.Context, index int, log *zap.Logger, name string) (_ *Uplink, err error) {
	defer mon.Task()(&ctx)(&err)

	identity, err := planet.NewIdentity()
	if err != nil {
		return nil, errs.Wrap(err)
	}

	tlsOptions, err := tlsopts.NewOptions(identity, tlsopts.Config{
		PeerIDVersions: strconv.Itoa(int(planet.config.IdentityVersion.Number)),
	}, nil)
	if err != nil {
		return nil, errs.Wrap(err)
	}

	planetUplink := &Uplink{
		Log:      planet.log.Named(name),
		Identity: identity,
		APIKey:   map[storj.NodeID]*macaroon.APIKey{},
		Access:   map[storj.NodeID]*uplink.Access{},
		User:     map[storj.NodeID]UserLogin{},
	}

	planetUplink.Log.Debug("id=" + identity.ID.String())

	planetUplink.Dialer = rpc.NewDefaultDialer(tlsOptions)

	for j, satellite := range planet.Satellites {
		consoleAPI := satellite.API.Console

		projectName := fmt.Sprintf("%s_%d", name, j)
		user, err := satellite.AddUser(ctx, console.CreateUser{
			FullName: fmt.Sprintf("User %s", projectName),
			Email:    fmt.Sprintf("user@%s.test", projectName),
		}, 10)
		if err != nil {
			return nil, errs.Wrap(err)
		}

		planetUplink.User[satellite.ID()] = UserLogin{
			Email:    user.Email,
			Password: user.FullName,
		}

		project, err := satellite.AddProject(ctx, user.ID, projectName)
		if err != nil {
			return nil, errs.Wrap(err)
		}

		userCtx, err := satellite.UserContext(ctx, user.ID)
		if err != nil {
			return nil, errs.Wrap(err)
		}
		_, apiKey, err := consoleAPI.Service.CreateAPIKey(userCtx, project.ID, "root")
		if err != nil {
			return nil, errs.Wrap(err)
		}

		planetUplink.APIKey[satellite.ID()] = apiKey

		planetUplink.Projects = append(planetUplink.Projects, &Project{
			client: planetUplink,

			ID: project.ID,
			Owner: ProjectOwner{
				ID:    user.ID,
				Email: user.Email,
			},

			Satellite: satellite,
			APIKey:    apiKey.Serialize(),

			RawAPIKey: apiKey,
		})

		var config UplinkConfig
		if planet.config.Reconfigure.Uplink != nil {
			planet.config.Reconfigure.Uplink(log, index, &config)
		}

		// create access grant manually to avoid dialing satellite for
		// project id and deriving key with argon2.IDKey method
		encAccess := grant.NewEncryptionAccessWithDefaultKey(&storj.Key{})
		if config.DefaultPathCipher == storj.EncUnspecified {
			encAccess.SetDefaultPathCipher(storj.EncAESGCM)
		} else {
			encAccess.SetDefaultPathCipher(config.DefaultPathCipher)
		}

		grantAccess := grant.Access{
			SatelliteAddress: satellite.URL(),
			APIKey:           apiKey,
			EncAccess:        encAccess,
		}

		serializedAccess, err := grantAccess.Serialize()
		if err != nil {
			return nil, errs.Wrap(err)
		}

		access, err := uplink.ParseAccess(serializedAccess)
		if err != nil {
			return nil, errs.Wrap(err)
		}

		planetUplink.Access[satellite.ID()] = access
	}

	planet.Uplinks = append(planet.Uplinks, planetUplink)

	return planetUplink, nil
}

// ID returns uplink id.
func (client *Uplink) ID() storj.NodeID { return client.Identity.ID }

// Addr returns uplink address.
func (client *Uplink) Addr() string { return "" }

// Shutdown shuts down all uplink dependencies.
func (client *Uplink) Shutdown() error { return nil }

// DialMetainfo dials destination with apikey and returns metainfo Client.
func (client *Uplink) DialMetainfo(ctx context.Context, destination Peer, apikey *macaroon.APIKey) (_ *metaclient.Client, err error) {
	defer mon.Task()(&ctx)(&err)
	return metaclient.DialNodeURL(ctx, client.Dialer, destination.NodeURL().String(), apikey, "Test/1.0")
}

// DialPiecestore dials destination storagenode and returns a piecestore client.
func (client *Uplink) DialPiecestore(ctx context.Context, destination Peer) (_ *piecestore.Client, err error) {
	defer mon.Task()(&ctx)(&err)
	return piecestore.Dial(ctx, client.Dialer, destination.NodeURL(), piecestore.DefaultConfig)
}

// OpenProject opens project with predefined access grant and gives access to pure uplink API.
func (client *Uplink) OpenProject(ctx context.Context, satellite *Satellite) (_ *uplink.Project, err error) {
	defer mon.Task()(&ctx)(&err)
	_, found := testuplink.GetMaxSegmentSize(ctx)
	if !found {
		ctx = testuplink.WithMaxSegmentSize(ctx, satellite.Config.Metainfo.MaxSegmentSize)
	}
	return uplink.OpenProject(ctx, client.Access[satellite.ID()])
}

// Upload data to specific satellite.
func (client *Uplink) Upload(ctx context.Context, satellite *Satellite, bucket string, path storj.Path, data []byte) (err error) {
	defer mon.Task()(&ctx)(&err)
	return client.UploadWithExpiration(ctx, satellite, bucket, path, data, time.Time{})
}

// UploadWithExpiration data to specific satellite and expiration time.
func (client *Uplink) UploadWithExpiration(ctx context.Context, satellite *Satellite, bucketName string, path storj.Path, data []byte, expiration time.Time) (err error) {
	defer mon.Task()(&ctx)(&err)

	_, found := testuplink.GetMaxSegmentSize(ctx)
	if !found {
		ctx = testuplink.WithMaxSegmentSize(ctx, satellite.Config.Metainfo.MaxSegmentSize)
	}

	project, err := client.GetProject(ctx, satellite)
	if err != nil {
		return errs.Wrap(err)
	}
	defer func() { err = errs.Combine(err, project.Close()) }()

	_, err = project.EnsureBucket(ctx, bucketName)
	if err != nil {
		return errs.Wrap(err)
	}

	upload, err := project.UploadObject(ctx, bucketName, path, &uplink.UploadOptions{
		Expires: expiration,
	})
	if err != nil {
		return errs.Wrap(err)
	}

	_, err = io.Copy(upload, bytes.NewReader(data))
	if err != nil {
		abortErr := upload.Abort()
		err = errs.Combine(err, abortErr)
		return errs.Wrap(err)
	}

	return upload.Commit()
}

// Download data from specific satellite.
func (client *Uplink) Download(ctx context.Context, satellite *Satellite, bucketName string, path storj.Path) (_ []byte, err error) {
	defer mon.Task()(&ctx)(&err)

	project, err := client.GetProject(ctx, satellite)
	if err != nil {
		return nil, errs.Wrap(err)
	}
	defer func() { err = errs.Combine(err, project.Close()) }()

	download, err := project.DownloadObject(ctx, bucketName, path, nil)
	if err != nil {
		return nil, errs.Wrap(err)
	}
	defer func() { err = errs.Combine(err, download.Close()) }()

	data, err := io.ReadAll(download)
	if err != nil {
		return []byte{}, err
	}
	return data, nil
}

// DownloadStream returns stream for downloading data.
func (client *Uplink) DownloadStream(ctx context.Context, satellite *Satellite, bucketName string, path storj.Path) (_ io.ReadCloser, cleanup func() error, err error) {
	defer mon.Task()(&ctx)(&err)

	project, err := client.GetProject(ctx, satellite)
	if err != nil {
		return nil, nil, errs.Wrap(err)
	}

	cleanup = func() error {
		err = errs.Combine(err,
			project.Close(),
		)
		return errs.Wrap(err)
	}

	downloader, err := project.DownloadObject(ctx, bucketName, path, nil)
	return downloader, cleanup, err
}

// DownloadStreamRange returns stream for downloading data.
func (client *Uplink) DownloadStreamRange(ctx context.Context, satellite *Satellite, bucketName string, path storj.Path, start, limit int64) (_ io.ReadCloser, cleanup func() error, err error) {
	defer mon.Task()(&ctx)(&err)

	project, err := client.GetProject(ctx, satellite)
	if err != nil {
		return nil, nil, errs.Wrap(err)
	}

	cleanup = func() error {
		return errs.Combine(err, project.Close())
	}

	downloader, err := project.DownloadObject(ctx, bucketName, path, &uplink.DownloadOptions{
		Offset: start,
		Length: limit,
	})
	return downloader, cleanup, errs.Wrap(err)
}

// DeleteObject deletes an object at the path in a bucket.
func (client *Uplink) DeleteObject(ctx context.Context, satellite *Satellite, bucketName string, path storj.Path) (err error) {
	defer mon.Task()(&ctx)(&err)

	project, err := client.GetProject(ctx, satellite)
	if err != nil {
		return errs.Wrap(err)
	}
	defer func() { err = errs.Combine(err, project.Close()) }()

	_, err = project.DeleteObject(ctx, bucketName, path)
	if err != nil {
		return errs.Wrap(err)
	}
	return errs.Wrap(err)
}

// CopyObject copies an object.
func (client *Uplink) CopyObject(ctx context.Context, satellite *Satellite, oldBucket, oldKey, newBucket, newKey string) (err error) {
	defer mon.Task()(&ctx)(&err)

	project, err := client.GetProject(ctx, satellite)
	if err != nil {
		return err
	}
	defer func() { err = errs.Combine(err, project.Close()) }()

	_, err = project.CopyObject(ctx, oldBucket, oldKey, newBucket, newKey, nil)
	return err
}

// CreateBucket creates a new bucket.
func (client *Uplink) CreateBucket(ctx context.Context, satellite *Satellite, bucketName string) (err error) {
	defer mon.Task()(&ctx)(&err)

	project, err := client.GetProject(ctx, satellite)
	if err != nil {
		return errs.Wrap(err)
	}
	defer func() { err = errs.Combine(err, project.Close()) }()

	_, err = project.CreateBucket(ctx, bucketName)
	if err != nil {
		return errs.Wrap(err)
	}
	return nil
}

// DeleteBucket deletes a bucket.
func (client *Uplink) DeleteBucket(ctx context.Context, satellite *Satellite, bucketName string) (err error) {
	defer mon.Task()(&ctx)(&err)

	project, err := client.GetProject(ctx, satellite)
	if err != nil {
		return errs.Wrap(err)
	}
	defer func() { err = errs.Combine(err, project.Close()) }()

	_, err = project.DeleteBucket(ctx, bucketName)
	if err != nil {
		return errs.Wrap(err)
	}
	return nil
}

// ListBuckets returns a list of all buckets in a project.
func (client *Uplink) ListBuckets(ctx context.Context, satellite *Satellite) (_ []*uplink.Bucket, err error) {
	defer mon.Task()(&ctx)(&err)

	var buckets = []*uplink.Bucket{}
	project, err := client.GetProject(ctx, satellite)
	if err != nil {
		return buckets, errs.Wrap(err)
	}
	defer func() { err = errs.Combine(err, project.Close()) }()

	iter := project.ListBuckets(ctx, &uplink.ListBucketsOptions{})
	for iter.Next() {
		buckets = append(buckets, iter.Item())
	}
	return buckets, iter.Err()
}

// ListObjects returns a list of all objects in a bucket.
func (client *Uplink) ListObjects(ctx context.Context, satellite *Satellite, bucketName string) (_ []*uplink.Object, err error) {
	defer mon.Task()(&ctx)(&err)

	var objects = []*uplink.Object{}
	project, err := client.GetProject(ctx, satellite)
	if err != nil {
		return objects, errs.Wrap(err)
	}
	defer func() { err = errs.Combine(err, project.Close()) }()

	iter := project.ListObjects(ctx, bucketName, &uplink.ListObjectsOptions{})
	for iter.Next() {
		objects = append(objects, iter.Item())
	}
	return objects, iter.Err()
}

// GetProject returns a uplink.Project which allows interactions with a specific project.
func (client *Uplink) GetProject(ctx context.Context, satellite *Satellite) (_ *uplink.Project, err error) {
	defer mon.Task()(&ctx)(&err)

	access := client.Access[satellite.ID()]

	project, err := client.Config.OpenProject(ctx, access)
	if err != nil {
		return nil, errs.Wrap(err)
	}
	return project, nil
}