storj/satellite/metainfo/config.go

// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.

package metainfo

import (
	"context"
	"io"
	"time"

	"go.uber.org/zap"

	"storj.io/common/memory"
	"storj.io/storj/private/dbutil"
	"storj.io/storj/satellite/metainfo/metabase"
	"storj.io/storj/satellite/metainfo/objectdeletion"
	"storj.io/storj/satellite/metainfo/piecedeletion"
	"storj.io/storj/storage"
	"storj.io/storj/storage/cockroachkv"
	"storj.io/storj/storage/postgreskv"
)

const (
	// BoltPointerBucket is the string representing the bucket used for `PointerEntries` in BoltDB.
	BoltPointerBucket = "pointers"
)

// RSConfig is a configuration struct that keeps details about default
// redundancy strategy information.
type RSConfig struct {
	MaxBufferMem     memory.Size `help:"maximum buffer memory to be allocated for read buffers" default:"4.00MiB"`
	ErasureShareSize memory.Size `help:"the size of each new erasure share in bytes" default:"256B"`
	MinThreshold     int         `help:"the minimum pieces required to recover a segment. k." releaseDefault:"29" devDefault:"4"`
	RepairThreshold  int         `help:"the minimum safe pieces before a repair is triggered. m." releaseDefault:"35" devDefault:"6"`
	SuccessThreshold int         `help:"the desired total pieces for a segment. o." releaseDefault:"80" devDefault:"8"`
	TotalThreshold   int         `help:"the largest amount of pieces to encode to. n." releaseDefault:"110" devDefault:"10"`

	// TODO left for validation until we will remove CreateSegmentOld
	MinTotalThreshold int  `help:"the largest amount of pieces to encode to. n (lower bound for validation)." releaseDefault:"95" devDefault:"10"`
	MaxTotalThreshold int  `help:"the largest amount of pieces to encode to. n (upper bound for validation)." releaseDefault:"130" devDefault:"10"`
	Validate          bool `help:"validate redundancy scheme configuration" default:"true"`
}

// RateLimiterConfig is a configuration struct for endpoint rate limiting.
type RateLimiterConfig struct {
	Enabled         bool          `help:"whether rate limiting is enabled." releaseDefault:"true" devDefault:"true"`
	Rate            float64       `help:"request rate per project per second." releaseDefault:"1000" devDefault:"100"`
	CacheCapacity   int           `help:"number of projects to cache." releaseDefault:"10000" devDefault:"10"`
	CacheExpiration time.Duration `help:"how long to cache the projects limiter." releaseDefault:"10m" devDefault:"10s"`
}

// ProjectLimitConfig is a configuration struct for default project limits.
type ProjectLimitConfig struct {
	MaxBuckets          int         `help:"max bucket count for a project." default:"100"`
	DefaultMaxUsage     memory.Size `help:"the default storage usage limit" releaseDefault:"50.00GB" devDefault:"200GB"`
	DefaultMaxBandwidth memory.Size `help:"the default bandwidth usage limit" releaseDefault:"50.00GB" devDefault:"200GB"`
}

// Config is a configuration struct that is everything you need to start a metainfo.
type Config struct {
	DatabaseURL          string                `help:"the database connection string to use" default:"postgres://"`
	MinRemoteSegmentSize memory.Size           `default:"1240" help:"minimum remote segment size"`
	MaxInlineSegmentSize memory.Size           `default:"4KiB" help:"maximum inline segment size"`
	MaxSegmentSize       memory.Size           `default:"64MiB" help:"maximum segment size"`
	MaxMetadataSize      memory.Size           `default:"2KiB" help:"maximum segment metadata size"`
	MaxCommitInterval    time.Duration         `default:"48h" help:"maximum time allowed to pass between creating and committing a segment"`
	Overlay              bool                  `default:"true" help:"toggle flag if overlay is enabled"`
	RS                   RSConfig              `help:"redundancy scheme configuration"`
	Loop                 LoopConfig            `help:"loop configuration"`
	RateLimiter          RateLimiterConfig     `help:"rate limiter configuration"`
	ProjectLimits        ProjectLimitConfig    `help:"project limit configuration"`
	PieceDeletion        piecedeletion.Config  `help:"piece deletion configuration"`
	ObjectDeletion       objectdeletion.Config `help:"object deletion configuration"`
}

// PointerDB stores pointers.
//
// architecture: Database
type PointerDB interface {
	// MigrateToLatest migrates to latest schema version.
	MigrateToLatest(ctx context.Context) error

	storage.KeyValueStore
}

// OpenStore returns database for storing pointer data.
func OpenStore(ctx context.Context, logger *zap.Logger, dbURLString string) (db PointerDB, err error) {
	_, source, implementation, err := dbutil.SplitConnStr(dbURLString)
	if err != nil {
		return nil, err
	}

	switch implementation {
	case dbutil.Postgres:
		db, err = postgreskv.Open(ctx, source)
	case dbutil.Cockroach:
		db, err = cockroachkv.Open(ctx, source)
	default:
		err = Error.New("unsupported db implementation: %s", dbURLString)
	}

	if err != nil {
		return nil, err
	}

	logger.Debug("Connected to:", zap.String("db source", source))
	return db, nil
}

// MetabaseDB stores objects and segments.
type MetabaseDB interface {
	io.Closer
	// MigrateToLatest migrates to latest schema version.
	MigrateToLatest(ctx context.Context) error

	// InternalImplementation returns *metabase.DB.
	// TODO: remove.
	InternalImplementation() interface{}
}

// OpenMetabase returns database for storing objects and segments.
func OpenMetabase(ctx context.Context, log *zap.Logger, dbURLString string) (db MetabaseDB, err error) {
	_, source, implementation, err := dbutil.SplitConnStr(dbURLString)
	if err != nil {
		return nil, err
	}

	switch implementation {
	case dbutil.Postgres:
		db, err = metabase.Open(ctx, log, "pgx", dbURLString)
	case dbutil.Cockroach:
		db, err = metabase.Open(ctx, log, "cockroach", dbURLString)
	default:
		err = Error.New("unsupported db implementation: %s", dbURLString)
	}

	if err != nil {
		return nil, err
	}

	log.Debug("Connected to:", zap.String("db source", source))
	return db, nil
}