// Copyright (C) 2019 Storj Labs, Inc. // See LICENSE for copying information. package metainfo import ( "context" "io" "time" "go.uber.org/zap" "storj.io/common/memory" "storj.io/storj/private/dbutil" "storj.io/storj/satellite/metainfo/metabase" "storj.io/storj/satellite/metainfo/objectdeletion" "storj.io/storj/satellite/metainfo/piecedeletion" "storj.io/storj/storage" "storj.io/storj/storage/cockroachkv" "storj.io/storj/storage/postgreskv" ) const ( // BoltPointerBucket is the string representing the bucket used for `PointerEntries` in BoltDB. BoltPointerBucket = "pointers" ) // RSConfig is a configuration struct that keeps details about default // redundancy strategy information. type RSConfig struct { MaxBufferMem memory.Size `help:"maximum buffer memory to be allocated for read buffers" default:"4.00MiB"` ErasureShareSize memory.Size `help:"the size of each new erasure share in bytes" default:"256B"` MinThreshold int `help:"the minimum pieces required to recover a segment. k." releaseDefault:"29" devDefault:"4"` RepairThreshold int `help:"the minimum safe pieces before a repair is triggered. m." releaseDefault:"35" devDefault:"6"` SuccessThreshold int `help:"the desired total pieces for a segment. o." releaseDefault:"80" devDefault:"8"` TotalThreshold int `help:"the largest amount of pieces to encode to. n." releaseDefault:"110" devDefault:"10"` // TODO left for validation until we will remove CreateSegmentOld MinTotalThreshold int `help:"the largest amount of pieces to encode to. n (lower bound for validation)." releaseDefault:"95" devDefault:"10"` MaxTotalThreshold int `help:"the largest amount of pieces to encode to. n (upper bound for validation)." releaseDefault:"130" devDefault:"10"` Validate bool `help:"validate redundancy scheme configuration" default:"true"` } // RateLimiterConfig is a configuration struct for endpoint rate limiting. type RateLimiterConfig struct { Enabled bool `help:"whether rate limiting is enabled." releaseDefault:"true" devDefault:"true"` Rate float64 `help:"request rate per project per second." releaseDefault:"1000" devDefault:"100"` CacheCapacity int `help:"number of projects to cache." releaseDefault:"10000" devDefault:"10"` CacheExpiration time.Duration `help:"how long to cache the projects limiter." releaseDefault:"10m" devDefault:"10s"` } // ProjectLimitConfig is a configuration struct for default project limits. type ProjectLimitConfig struct { MaxBuckets int `help:"max bucket count for a project." default:"100"` DefaultMaxUsage memory.Size `help:"the default storage usage limit" releaseDefault:"50.00GB" devDefault:"200GB"` DefaultMaxBandwidth memory.Size `help:"the default bandwidth usage limit" releaseDefault:"50.00GB" devDefault:"200GB"` } // Config is a configuration struct that is everything you need to start a metainfo. type Config struct { DatabaseURL string `help:"the database connection string to use" default:"postgres://"` MinRemoteSegmentSize memory.Size `default:"1240" help:"minimum remote segment size"` MaxInlineSegmentSize memory.Size `default:"4KiB" help:"maximum inline segment size"` MaxSegmentSize memory.Size `default:"64MiB" help:"maximum segment size"` MaxMetadataSize memory.Size `default:"2KiB" help:"maximum segment metadata size"` MaxCommitInterval time.Duration `default:"48h" help:"maximum time allowed to pass between creating and committing a segment"` Overlay bool `default:"true" help:"toggle flag if overlay is enabled"` RS RSConfig `help:"redundancy scheme configuration"` Loop LoopConfig `help:"loop configuration"` RateLimiter RateLimiterConfig `help:"rate limiter configuration"` ProjectLimits ProjectLimitConfig `help:"project limit configuration"` PieceDeletion piecedeletion.Config `help:"piece deletion configuration"` ObjectDeletion objectdeletion.Config `help:"object deletion configuration"` } // PointerDB stores pointers. // // architecture: Database type PointerDB interface { // MigrateToLatest migrates to latest schema version. MigrateToLatest(ctx context.Context) error storage.KeyValueStore } // OpenStore returns database for storing pointer data. func OpenStore(ctx context.Context, logger *zap.Logger, dbURLString string) (db PointerDB, err error) { _, source, implementation, err := dbutil.SplitConnStr(dbURLString) if err != nil { return nil, err } switch implementation { case dbutil.Postgres: db, err = postgreskv.Open(ctx, source) case dbutil.Cockroach: db, err = cockroachkv.Open(ctx, source) default: err = Error.New("unsupported db implementation: %s", dbURLString) } if err != nil { return nil, err } logger.Debug("Connected to:", zap.String("db source", source)) return db, nil } // MetabaseDB stores objects and segments. type MetabaseDB interface { io.Closer // MigrateToLatest migrates to latest schema version. MigrateToLatest(ctx context.Context) error } // OpenMetabase returns database for storing objects and segments. func OpenMetabase(ctx context.Context, logger *zap.Logger, dbURLString string) (db MetabaseDB, err error) { _, source, implementation, err := dbutil.SplitConnStr(dbURLString) if err != nil { return nil, err } switch implementation { case dbutil.Postgres: db, err = metabase.Open(ctx, "pgx", dbURLString) case dbutil.Cockroach: db, err = metabase.Open(ctx, "cockroach", dbURLString) default: err = Error.New("unsupported db implementation: %s", dbURLString) } if err != nil { return nil, err } logger.Debug("Connected to:", zap.String("db source", source)) return db, nil }