internal/testplanet: split planet.go file to avoid package naming conflicts (#2279)
This commit is contained in:
Egon Elbre
GitHub
parent
5f47b7028d
commit
5b030062c0
158
internal/testplanet/bootstrap.go
Normal file
158
internal/testplanet/bootstrap.go
Normal file
@ -0,0 +1,158 @@
|
||||
// Copyright (C) 2019 Storj Labs, Inc.
|
||||
// See LICENSE for copying information
|
||||
|
||||
package testplanet
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"storj.io/storj/bootstrap"
|
||||
"storj.io/storj/bootstrap/bootstrapdb"
|
||||
"storj.io/storj/bootstrap/bootstrapweb/bootstrapserver"
|
||||
"storj.io/storj/internal/version"
|
||||
"storj.io/storj/pkg/kademlia"
|
||||
"storj.io/storj/pkg/peertls/extensions"
|
||||
"storj.io/storj/pkg/peertls/tlsopts"
|
||||
"storj.io/storj/pkg/server"
|
||||
"storj.io/storj/versioncontrol"
|
||||
)
|
||||
|
||||
// newBootstrap initializes the bootstrap node
|
||||
func (planet *Planet) newBootstrap() (peer *bootstrap.Peer, err error) {
|
||||
defer func() {
|
||||
planet.peers = append(planet.peers, closablePeer{peer: peer})
|
||||
}()
|
||||
|
||||
prefix := "bootstrap"
|
||||
log := planet.log.Named(prefix)
|
||||
dbDir := filepath.Join(planet.directory, prefix)
|
||||
|
||||
if err := os.MkdirAll(dbDir, 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
identity, err := planet.NewIdentity()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var db bootstrap.DB
|
||||
if planet.config.Reconfigure.NewBootstrapDB != nil {
|
||||
db, err = planet.config.Reconfigure.NewBootstrapDB(0)
|
||||
} else {
|
||||
db, err = bootstrapdb.NewInMemory(dbDir)
|
||||
}
|
||||
|
||||
err = db.CreateTables()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
planet.databases = append(planet.databases, db)
|
||||
|
||||
config := bootstrap.Config{
|
||||
Server: server.Config{
|
||||
Address: "127.0.0.1:0",
|
||||
PrivateAddress: "127.0.0.1:0",
|
||||
|
||||
Config: tlsopts.Config{
|
||||
RevocationDBURL: "bolt://" + filepath.Join(dbDir, "revocation.db"),
|
||||
UsePeerCAWhitelist: true,
|
||||
PeerCAWhitelistPath: planet.whitelistPath,
|
||||
PeerIDVersions: "latest",
|
||||
Extensions: extensions.Config{
|
||||
Revocation: false,
|
||||
WhitelistSignedLeaf: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
Kademlia: kademlia.Config{
|
||||
BootstrapBackoffBase: 500 * time.Millisecond,
|
||||
BootstrapBackoffMax: 2 * time.Second,
|
||||
Alpha: 5,
|
||||
DBPath: dbDir, // TODO: replace with master db
|
||||
Operator: kademlia.OperatorConfig{
|
||||
Email: prefix + "@mail.test",
|
||||
Wallet: "0x" + strings.Repeat("00", 20),
|
||||
},
|
||||
},
|
||||
Web: bootstrapserver.Config{
|
||||
Address: "127.0.0.1:0",
|
||||
StaticDir: "./web/bootstrap", // TODO: for development only
|
||||
},
|
||||
Version: planet.NewVersionConfig(),
|
||||
}
|
||||
if planet.config.Reconfigure.Bootstrap != nil {
|
||||
planet.config.Reconfigure.Bootstrap(0, &config)
|
||||
}
|
||||
|
||||
var verInfo version.Info
|
||||
verInfo = planet.NewVersionInfo()
|
||||
|
||||
peer, err = bootstrap.New(log, identity, db, config, verInfo)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Debug("id=" + peer.ID().String() + " addr=" + peer.Addr())
|
||||
|
||||
return peer, nil
|
||||
}
|
||||
|
||||
// newVersionControlServer initializes the Versioning Server
|
||||
func (planet *Planet) newVersionControlServer() (peer *versioncontrol.Peer, err error) {
|
||||
|
||||
prefix := "versioncontrol"
|
||||
log := planet.log.Named(prefix)
|
||||
dbDir := filepath.Join(planet.directory, prefix)
|
||||
|
||||
if err := os.MkdirAll(dbDir, 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
config := &versioncontrol.Config{
|
||||
Address: "127.0.0.1:0",
|
||||
Versions: versioncontrol.ServiceVersions{
|
||||
Bootstrap: "v0.0.1",
|
||||
Satellite: "v0.0.1",
|
||||
Storagenode: "v0.0.1",
|
||||
Uplink: "v0.0.1",
|
||||
Gateway: "v0.0.1",
|
||||
},
|
||||
}
|
||||
peer, err = versioncontrol.New(log, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Debug(" addr= " + peer.Addr())
|
||||
|
||||
return peer, nil
|
||||
}
|
||||
|
||||
// NewVersionInfo returns the Version Info for this planet with tuned metrics.
|
||||
func (planet *Planet) NewVersionInfo() version.Info {
|
||||
info := version.Info{
|
||||
Timestamp: time.Now(),
|
||||
CommitHash: "testplanet",
|
||||
Version: version.SemVer{
|
||||
Major: 0,
|
||||
Minor: 0,
|
||||
Patch: 1},
|
||||
Release: false,
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
// NewVersionConfig returns the Version Config for this planet with tuned metrics.
|
||||
func (planet *Planet) NewVersionConfig() version.Config {
|
||||
return version.Config{
|
||||
ServerAddress: fmt.Sprintf("http://%s/", planet.VersionControl.Addr()),
|
||||
RequestTimeout: time.Second * 15,
|
||||
CheckInterval: time.Minute * 5,
|
||||
}
|
||||
}
|
||||
@ -7,14 +7,11 @@ package testplanet
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@ -24,41 +21,13 @@ import (
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"storj.io/storj/bootstrap"
|
||||
"storj.io/storj/bootstrap/bootstrapdb"
|
||||
"storj.io/storj/bootstrap/bootstrapweb/bootstrapserver"
|
||||
"storj.io/storj/internal/memory"
|
||||
"storj.io/storj/internal/testidentity"
|
||||
"storj.io/storj/internal/version"
|
||||
"storj.io/storj/pkg/accounting/rollup"
|
||||
"storj.io/storj/pkg/accounting/tally"
|
||||
"storj.io/storj/pkg/audit"
|
||||
"storj.io/storj/pkg/datarepair/checker"
|
||||
"storj.io/storj/pkg/datarepair/repairer"
|
||||
"storj.io/storj/pkg/discovery"
|
||||
"storj.io/storj/pkg/identity"
|
||||
"storj.io/storj/pkg/kademlia"
|
||||
"storj.io/storj/pkg/overlay"
|
||||
"storj.io/storj/pkg/pb"
|
||||
"storj.io/storj/pkg/peertls/extensions"
|
||||
"storj.io/storj/pkg/peertls/tlsopts"
|
||||
"storj.io/storj/pkg/server"
|
||||
"storj.io/storj/pkg/storj"
|
||||
"storj.io/storj/satellite"
|
||||
"storj.io/storj/satellite/console"
|
||||
"storj.io/storj/satellite/console/consoleweb"
|
||||
"storj.io/storj/satellite/mailservice"
|
||||
"storj.io/storj/satellite/marketing/marketingweb"
|
||||
"storj.io/storj/satellite/metainfo"
|
||||
satorders "storj.io/storj/satellite/orders"
|
||||
"storj.io/storj/satellite/satellitedb"
|
||||
"storj.io/storj/satellite/vouchers"
|
||||
"storj.io/storj/storagenode"
|
||||
"storj.io/storj/storagenode/collector"
|
||||
"storj.io/storj/storagenode/console/consoleserver"
|
||||
"storj.io/storj/storagenode/monitor"
|
||||
"storj.io/storj/storagenode/orders"
|
||||
"storj.io/storj/storagenode/piecestore"
|
||||
"storj.io/storj/storagenode/storagenodedb"
|
||||
"storj.io/storj/versioncontrol"
|
||||
)
|
||||
|
||||
@ -368,444 +337,6 @@ func (planet *Planet) Shutdown() error {
|
||||
return errlist.Err()
|
||||
}
|
||||
|
||||
// newUplinks creates initializes uplinks, requires peer to have at least one satellite
|
||||
func (planet *Planet) newUplinks(prefix string, count, storageNodeCount int) ([]*Uplink, error) {
|
||||
var xs []*Uplink
|
||||
for i := 0; i < count; i++ {
|
||||
uplink, err := planet.newUplink(prefix+strconv.Itoa(i), storageNodeCount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
xs = append(xs, uplink)
|
||||
}
|
||||
|
||||
return xs, nil
|
||||
}
|
||||
|
||||
// newSatellites initializes satellites
|
||||
func (planet *Planet) newSatellites(count int) ([]*satellite.Peer, error) {
|
||||
// TODO: move into separate file
|
||||
var xs []*satellite.Peer
|
||||
defer func() {
|
||||
for _, x := range xs {
|
||||
planet.peers = append(planet.peers, closablePeer{peer: x})
|
||||
}
|
||||
}()
|
||||
|
||||
for i := 0; i < count; i++ {
|
||||
prefix := "satellite" + strconv.Itoa(i)
|
||||
log := planet.log.Named(prefix)
|
||||
|
||||
storageDir := filepath.Join(planet.directory, prefix)
|
||||
if err := os.MkdirAll(storageDir, 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
identity, err := planet.NewIdentity()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var db satellite.DB
|
||||
if planet.config.Reconfigure.NewSatelliteDB != nil {
|
||||
db, err = planet.config.Reconfigure.NewSatelliteDB(log.Named("db"), i)
|
||||
} else {
|
||||
db, err = satellitedb.NewInMemory(log.Named("db"))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.CreateTables()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
planet.databases = append(planet.databases, db)
|
||||
|
||||
config := satellite.Config{
|
||||
Server: server.Config{
|
||||
Address: "127.0.0.1:0",
|
||||
PrivateAddress: "127.0.0.1:0",
|
||||
|
||||
Config: tlsopts.Config{
|
||||
RevocationDBURL: "bolt://" + filepath.Join(storageDir, "revocation.db"),
|
||||
UsePeerCAWhitelist: true,
|
||||
PeerCAWhitelistPath: planet.whitelistPath,
|
||||
PeerIDVersions: "latest",
|
||||
Extensions: extensions.Config{
|
||||
Revocation: false,
|
||||
WhitelistSignedLeaf: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
Kademlia: kademlia.Config{
|
||||
Alpha: 5,
|
||||
BootstrapBackoffBase: 500 * time.Millisecond,
|
||||
BootstrapBackoffMax: 2 * time.Second,
|
||||
DBPath: storageDir, // TODO: replace with master db
|
||||
Operator: kademlia.OperatorConfig{
|
||||
Email: prefix + "@mail.test",
|
||||
Wallet: "0x" + strings.Repeat("00", 20),
|
||||
},
|
||||
},
|
||||
Overlay: overlay.Config{
|
||||
Node: overlay.NodeSelectionConfig{
|
||||
UptimeRatio: 0,
|
||||
UptimeCount: 0,
|
||||
AuditSuccessRatio: 0,
|
||||
AuditCount: 0,
|
||||
NewNodePercentage: 0,
|
||||
OnlineWindow: time.Hour,
|
||||
DistinctIP: false,
|
||||
|
||||
AuditReputationRepairWeight: 1,
|
||||
AuditReputationUplinkWeight: 1,
|
||||
AuditReputationAlpha0: 1,
|
||||
AuditReputationBeta0: 0,
|
||||
AuditReputationLambda: 1,
|
||||
AuditReputationWeight: 1,
|
||||
UptimeReputationRepairWeight: 1,
|
||||
UptimeReputationUplinkWeight: 1,
|
||||
UptimeReputationAlpha0: 1,
|
||||
UptimeReputationBeta0: 0,
|
||||
UptimeReputationLambda: 1,
|
||||
UptimeReputationWeight: 1,
|
||||
},
|
||||
},
|
||||
Discovery: discovery.Config{
|
||||
DiscoveryInterval: 1 * time.Second,
|
||||
RefreshInterval: 1 * time.Second,
|
||||
RefreshLimit: 100,
|
||||
},
|
||||
Metainfo: metainfo.Config{
|
||||
DatabaseURL: "bolt://" + filepath.Join(storageDir, "pointers.db"),
|
||||
MinRemoteSegmentSize: 0, // TODO: fix tests to work with 1024
|
||||
MaxInlineSegmentSize: 8000,
|
||||
Overlay: true,
|
||||
BwExpiration: 45,
|
||||
},
|
||||
Orders: satorders.Config{
|
||||
Expiration: 45 * 24 * time.Hour,
|
||||
},
|
||||
Checker: checker.Config{
|
||||
Interval: 30 * time.Second,
|
||||
IrreparableInterval: 15 * time.Second,
|
||||
},
|
||||
Repairer: repairer.Config{
|
||||
MaxRepair: 10,
|
||||
Interval: time.Hour,
|
||||
Timeout: 10 * time.Second, // Repairs can take up to 4 seconds. Leaving room for outliers
|
||||
MaxBufferMem: 4 * memory.MiB,
|
||||
},
|
||||
Audit: audit.Config{
|
||||
MaxRetriesStatDB: 0,
|
||||
Interval: 30 * time.Second,
|
||||
MinBytesPerSecond: 1 * memory.KB,
|
||||
},
|
||||
Tally: tally.Config{
|
||||
Interval: 30 * time.Second,
|
||||
},
|
||||
Rollup: rollup.Config{
|
||||
Interval: 2 * time.Minute,
|
||||
MaxAlphaUsage: 25 * memory.GB,
|
||||
DeleteTallies: false,
|
||||
},
|
||||
Mail: mailservice.Config{
|
||||
SMTPServerAddress: "smtp.mail.test:587",
|
||||
From: "Labs <storj@mail.test>",
|
||||
AuthType: "simulate",
|
||||
TemplatePath: filepath.Join(developmentRoot, "web/satellite/static/emails"),
|
||||
},
|
||||
Console: consoleweb.Config{
|
||||
Address: "127.0.0.1:0",
|
||||
StaticDir: filepath.Join(developmentRoot, "web/satellite"),
|
||||
PasswordCost: console.TestPasswordCost,
|
||||
AuthTokenSecret: "my-suppa-secret-key",
|
||||
},
|
||||
Marketing: marketingweb.Config{
|
||||
Address: "127.0.0.1:0",
|
||||
StaticDir: filepath.Join(developmentRoot, "web/marketing"),
|
||||
},
|
||||
Vouchers: vouchers.Config{
|
||||
Expiration: 30,
|
||||
},
|
||||
Version: planet.NewVersionConfig(),
|
||||
}
|
||||
if planet.config.Reconfigure.Satellite != nil {
|
||||
planet.config.Reconfigure.Satellite(log, i, &config)
|
||||
}
|
||||
|
||||
verInfo := planet.NewVersionInfo()
|
||||
|
||||
peer, err := satellite.New(log, identity, db, &config, verInfo)
|
||||
if err != nil {
|
||||
return xs, err
|
||||
}
|
||||
|
||||
log.Debug("id=" + peer.ID().String() + " addr=" + peer.Addr())
|
||||
xs = append(xs, peer)
|
||||
}
|
||||
return xs, nil
|
||||
}
|
||||
|
||||
// newStorageNodes initializes storage nodes
|
||||
func (planet *Planet) newStorageNodes(count int, whitelistedSatelliteIDs []string) ([]*storagenode.Peer, error) {
|
||||
// TODO: move into separate file
|
||||
var xs []*storagenode.Peer
|
||||
defer func() {
|
||||
for _, x := range xs {
|
||||
planet.peers = append(planet.peers, closablePeer{peer: x})
|
||||
}
|
||||
}()
|
||||
|
||||
for i := 0; i < count; i++ {
|
||||
prefix := "storage" + strconv.Itoa(i)
|
||||
log := planet.log.Named(prefix)
|
||||
storageDir := filepath.Join(planet.directory, prefix)
|
||||
|
||||
if err := os.MkdirAll(storageDir, 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
identity, err := planet.NewIdentity()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var db storagenode.DB
|
||||
if planet.config.Reconfigure.NewStorageNodeDB != nil {
|
||||
db, err = planet.config.Reconfigure.NewStorageNodeDB(i)
|
||||
} else {
|
||||
db, err = storagenodedb.NewInMemory(log.Named("db"), storageDir)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.CreateTables()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
planet.databases = append(planet.databases, db)
|
||||
|
||||
config := storagenode.Config{
|
||||
Server: server.Config{
|
||||
Address: "127.0.0.1:0",
|
||||
PrivateAddress: "127.0.0.1:0",
|
||||
|
||||
Config: tlsopts.Config{
|
||||
RevocationDBURL: "bolt://" + filepath.Join(storageDir, "revocation.db"),
|
||||
UsePeerCAWhitelist: true,
|
||||
PeerCAWhitelistPath: planet.whitelistPath,
|
||||
PeerIDVersions: "*",
|
||||
Extensions: extensions.Config{
|
||||
Revocation: false,
|
||||
WhitelistSignedLeaf: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
Kademlia: kademlia.Config{
|
||||
BootstrapBackoffBase: 500 * time.Millisecond,
|
||||
BootstrapBackoffMax: 2 * time.Second,
|
||||
Alpha: 5,
|
||||
DBPath: storageDir, // TODO: replace with master db
|
||||
Operator: kademlia.OperatorConfig{
|
||||
Email: prefix + "@mail.test",
|
||||
Wallet: "0x" + strings.Repeat("00", 20),
|
||||
},
|
||||
},
|
||||
Storage: piecestore.OldConfig{
|
||||
Path: "", // TODO: this argument won't be needed with master storagenodedb
|
||||
AllocatedDiskSpace: 1 * memory.GB,
|
||||
AllocatedBandwidth: memory.TB,
|
||||
KBucketRefreshInterval: time.Hour,
|
||||
|
||||
SatelliteIDRestriction: true,
|
||||
WhitelistedSatelliteIDs: strings.Join(whitelistedSatelliteIDs, ","),
|
||||
},
|
||||
Collector: collector.Config{
|
||||
Interval: time.Minute,
|
||||
},
|
||||
Console: consoleserver.Config{
|
||||
Address: "127.0.0.1:0",
|
||||
StaticDir: filepath.Join(developmentRoot, "web/operator/"),
|
||||
},
|
||||
Storage2: piecestore.Config{
|
||||
Sender: orders.SenderConfig{
|
||||
Interval: time.Hour,
|
||||
Timeout: time.Hour,
|
||||
},
|
||||
Monitor: monitor.Config{
|
||||
MinimumBandwidth: 100 * memory.MB,
|
||||
MinimumDiskSpace: 100 * memory.MB,
|
||||
},
|
||||
},
|
||||
Version: planet.NewVersionConfig(),
|
||||
}
|
||||
if planet.config.Reconfigure.StorageNode != nil {
|
||||
planet.config.Reconfigure.StorageNode(i, &config)
|
||||
}
|
||||
|
||||
newIPCount := planet.config.Reconfigure.NewIPCount
|
||||
if newIPCount > 0 {
|
||||
if i >= count-newIPCount {
|
||||
config.Server.Address = fmt.Sprintf("127.0.0.%d:0", i+1)
|
||||
config.Server.PrivateAddress = fmt.Sprintf("127.0.0.%d:0", i+1)
|
||||
}
|
||||
}
|
||||
|
||||
verInfo := planet.NewVersionInfo()
|
||||
|
||||
peer, err := storagenode.New(log, identity, db, config, verInfo)
|
||||
if err != nil {
|
||||
return xs, err
|
||||
}
|
||||
|
||||
log.Debug("id=" + peer.ID().String() + " addr=" + peer.Addr())
|
||||
xs = append(xs, peer)
|
||||
}
|
||||
return xs, nil
|
||||
}
|
||||
|
||||
// newBootstrap initializes the bootstrap node
|
||||
func (planet *Planet) newBootstrap() (peer *bootstrap.Peer, err error) {
|
||||
// TODO: move into separate file
|
||||
defer func() {
|
||||
planet.peers = append(planet.peers, closablePeer{peer: peer})
|
||||
}()
|
||||
|
||||
prefix := "bootstrap"
|
||||
log := planet.log.Named(prefix)
|
||||
dbDir := filepath.Join(planet.directory, prefix)
|
||||
|
||||
if err := os.MkdirAll(dbDir, 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
identity, err := planet.NewIdentity()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var db bootstrap.DB
|
||||
if planet.config.Reconfigure.NewBootstrapDB != nil {
|
||||
db, err = planet.config.Reconfigure.NewBootstrapDB(0)
|
||||
} else {
|
||||
db, err = bootstrapdb.NewInMemory(dbDir)
|
||||
}
|
||||
|
||||
err = db.CreateTables()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
planet.databases = append(planet.databases, db)
|
||||
|
||||
config := bootstrap.Config{
|
||||
Server: server.Config{
|
||||
Address: "127.0.0.1:0",
|
||||
PrivateAddress: "127.0.0.1:0",
|
||||
|
||||
Config: tlsopts.Config{
|
||||
RevocationDBURL: "bolt://" + filepath.Join(dbDir, "revocation.db"),
|
||||
UsePeerCAWhitelist: true,
|
||||
PeerCAWhitelistPath: planet.whitelistPath,
|
||||
PeerIDVersions: "latest",
|
||||
Extensions: extensions.Config{
|
||||
Revocation: false,
|
||||
WhitelistSignedLeaf: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
Kademlia: kademlia.Config{
|
||||
BootstrapBackoffBase: 500 * time.Millisecond,
|
||||
BootstrapBackoffMax: 2 * time.Second,
|
||||
Alpha: 5,
|
||||
DBPath: dbDir, // TODO: replace with master db
|
||||
Operator: kademlia.OperatorConfig{
|
||||
Email: prefix + "@mail.test",
|
||||
Wallet: "0x" + strings.Repeat("00", 20),
|
||||
},
|
||||
},
|
||||
Web: bootstrapserver.Config{
|
||||
Address: "127.0.0.1:0",
|
||||
StaticDir: "./web/bootstrap", // TODO: for development only
|
||||
},
|
||||
Version: planet.NewVersionConfig(),
|
||||
}
|
||||
if planet.config.Reconfigure.Bootstrap != nil {
|
||||
planet.config.Reconfigure.Bootstrap(0, &config)
|
||||
}
|
||||
|
||||
var verInfo version.Info
|
||||
verInfo = planet.NewVersionInfo()
|
||||
|
||||
peer, err = bootstrap.New(log, identity, db, config, verInfo)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Debug("id=" + peer.ID().String() + " addr=" + peer.Addr())
|
||||
|
||||
return peer, nil
|
||||
}
|
||||
|
||||
// newVersionControlServer initializes the Versioning Server
|
||||
func (planet *Planet) newVersionControlServer() (peer *versioncontrol.Peer, err error) {
|
||||
|
||||
prefix := "versioncontrol"
|
||||
log := planet.log.Named(prefix)
|
||||
dbDir := filepath.Join(planet.directory, prefix)
|
||||
|
||||
if err := os.MkdirAll(dbDir, 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
config := &versioncontrol.Config{
|
||||
Address: "127.0.0.1:0",
|
||||
Versions: versioncontrol.ServiceVersions{
|
||||
Bootstrap: "v0.0.1",
|
||||
Satellite: "v0.0.1",
|
||||
Storagenode: "v0.0.1",
|
||||
Uplink: "v0.0.1",
|
||||
Gateway: "v0.0.1",
|
||||
},
|
||||
}
|
||||
peer, err = versioncontrol.New(log, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Debug(" addr= " + peer.Addr())
|
||||
|
||||
return peer, nil
|
||||
}
|
||||
|
||||
// NewVersionInfo returns the Version Info for this planet with tuned metrics.
|
||||
func (planet *Planet) NewVersionInfo() version.Info {
|
||||
info := version.Info{
|
||||
Timestamp: time.Now(),
|
||||
CommitHash: "testplanet",
|
||||
Version: version.SemVer{
|
||||
Major: 0,
|
||||
Minor: 0,
|
||||
Patch: 1},
|
||||
Release: false,
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
// NewVersionConfig returns the Version Config for this planet with tuned metrics.
|
||||
func (planet *Planet) NewVersionConfig() version.Config {
|
||||
return version.Config{
|
||||
ServerAddress: fmt.Sprintf("http://%s/", planet.VersionControl.Addr()),
|
||||
RequestTimeout: time.Second * 15,
|
||||
CheckInterval: time.Minute * 5,
|
||||
}
|
||||
}
|
||||
|
||||
// Identities returns the identity provider for this planet.
|
||||
func (planet *Planet) Identities() *testidentity.Identities {
|
||||
return planet.identities
|
||||
|
||||
200
internal/testplanet/satellite.go
Normal file
200
internal/testplanet/satellite.go
Normal file
@ -0,0 +1,200 @@
|
||||
// Copyright (C) 2019 Storj Labs, Inc.
|
||||
// See LICENSE for copying information
|
||||
|
||||
package testplanet
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"storj.io/storj/internal/memory"
|
||||
"storj.io/storj/pkg/accounting/rollup"
|
||||
"storj.io/storj/pkg/accounting/tally"
|
||||
"storj.io/storj/pkg/audit"
|
||||
"storj.io/storj/pkg/datarepair/checker"
|
||||
"storj.io/storj/pkg/datarepair/repairer"
|
||||
"storj.io/storj/pkg/discovery"
|
||||
"storj.io/storj/pkg/kademlia"
|
||||
"storj.io/storj/pkg/overlay"
|
||||
"storj.io/storj/pkg/peertls/extensions"
|
||||
"storj.io/storj/pkg/peertls/tlsopts"
|
||||
"storj.io/storj/pkg/server"
|
||||
"storj.io/storj/satellite"
|
||||
"storj.io/storj/satellite/console"
|
||||
"storj.io/storj/satellite/console/consoleweb"
|
||||
"storj.io/storj/satellite/mailservice"
|
||||
"storj.io/storj/satellite/marketing/marketingweb"
|
||||
"storj.io/storj/satellite/metainfo"
|
||||
"storj.io/storj/satellite/orders"
|
||||
"storj.io/storj/satellite/satellitedb"
|
||||
"storj.io/storj/satellite/vouchers"
|
||||
)
|
||||
|
||||
// newSatellites initializes satellites
|
||||
func (planet *Planet) newSatellites(count int) ([]*satellite.Peer, error) {
|
||||
var xs []*satellite.Peer
|
||||
defer func() {
|
||||
for _, x := range xs {
|
||||
planet.peers = append(planet.peers, closablePeer{peer: x})
|
||||
}
|
||||
}()
|
||||
|
||||
for i := 0; i < count; i++ {
|
||||
prefix := "satellite" + strconv.Itoa(i)
|
||||
log := planet.log.Named(prefix)
|
||||
|
||||
storageDir := filepath.Join(planet.directory, prefix)
|
||||
if err := os.MkdirAll(storageDir, 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
identity, err := planet.NewIdentity()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var db satellite.DB
|
||||
if planet.config.Reconfigure.NewSatelliteDB != nil {
|
||||
db, err = planet.config.Reconfigure.NewSatelliteDB(log.Named("db"), i)
|
||||
} else {
|
||||
db, err = satellitedb.NewInMemory(log.Named("db"))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.CreateTables()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
planet.databases = append(planet.databases, db)
|
||||
|
||||
config := satellite.Config{
|
||||
Server: server.Config{
|
||||
Address: "127.0.0.1:0",
|
||||
PrivateAddress: "127.0.0.1:0",
|
||||
|
||||
Config: tlsopts.Config{
|
||||
RevocationDBURL: "bolt://" + filepath.Join(storageDir, "revocation.db"),
|
||||
UsePeerCAWhitelist: true,
|
||||
PeerCAWhitelistPath: planet.whitelistPath,
|
||||
PeerIDVersions: "latest",
|
||||
Extensions: extensions.Config{
|
||||
Revocation: false,
|
||||
WhitelistSignedLeaf: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
Kademlia: kademlia.Config{
|
||||
Alpha: 5,
|
||||
BootstrapBackoffBase: 500 * time.Millisecond,
|
||||
BootstrapBackoffMax: 2 * time.Second,
|
||||
DBPath: storageDir, // TODO: replace with master db
|
||||
Operator: kademlia.OperatorConfig{
|
||||
Email: prefix + "@mail.test",
|
||||
Wallet: "0x" + strings.Repeat("00", 20),
|
||||
},
|
||||
},
|
||||
Overlay: overlay.Config{
|
||||
Node: overlay.NodeSelectionConfig{
|
||||
UptimeRatio: 0,
|
||||
UptimeCount: 0,
|
||||
AuditSuccessRatio: 0,
|
||||
AuditCount: 0,
|
||||
NewNodePercentage: 0,
|
||||
OnlineWindow: time.Hour,
|
||||
DistinctIP: false,
|
||||
|
||||
AuditReputationRepairWeight: 1,
|
||||
AuditReputationUplinkWeight: 1,
|
||||
AuditReputationAlpha0: 1,
|
||||
AuditReputationBeta0: 0,
|
||||
AuditReputationLambda: 1,
|
||||
AuditReputationWeight: 1,
|
||||
UptimeReputationRepairWeight: 1,
|
||||
UptimeReputationUplinkWeight: 1,
|
||||
UptimeReputationAlpha0: 1,
|
||||
UptimeReputationBeta0: 0,
|
||||
UptimeReputationLambda: 1,
|
||||
UptimeReputationWeight: 1,
|
||||
},
|
||||
},
|
||||
Discovery: discovery.Config{
|
||||
DiscoveryInterval: 1 * time.Second,
|
||||
RefreshInterval: 1 * time.Second,
|
||||
RefreshLimit: 100,
|
||||
},
|
||||
Metainfo: metainfo.Config{
|
||||
DatabaseURL: "bolt://" + filepath.Join(storageDir, "pointers.db"),
|
||||
MinRemoteSegmentSize: 0, // TODO: fix tests to work with 1024
|
||||
MaxInlineSegmentSize: 8000,
|
||||
Overlay: true,
|
||||
BwExpiration: 45,
|
||||
},
|
||||
Orders: orders.Config{
|
||||
Expiration: 45 * 24 * time.Hour,
|
||||
},
|
||||
Checker: checker.Config{
|
||||
Interval: 30 * time.Second,
|
||||
IrreparableInterval: 15 * time.Second,
|
||||
},
|
||||
Repairer: repairer.Config{
|
||||
MaxRepair: 10,
|
||||
Interval: time.Hour,
|
||||
Timeout: 10 * time.Second, // Repairs can take up to 4 seconds. Leaving room for outliers
|
||||
MaxBufferMem: 4 * memory.MiB,
|
||||
},
|
||||
Audit: audit.Config{
|
||||
MaxRetriesStatDB: 0,
|
||||
Interval: 30 * time.Second,
|
||||
MinBytesPerSecond: 1 * memory.KB,
|
||||
},
|
||||
Tally: tally.Config{
|
||||
Interval: 30 * time.Second,
|
||||
},
|
||||
Rollup: rollup.Config{
|
||||
Interval: 2 * time.Minute,
|
||||
MaxAlphaUsage: 25 * memory.GB,
|
||||
DeleteTallies: false,
|
||||
},
|
||||
Mail: mailservice.Config{
|
||||
SMTPServerAddress: "smtp.mail.test:587",
|
||||
From: "Labs <storj@mail.test>",
|
||||
AuthType: "simulate",
|
||||
TemplatePath: filepath.Join(developmentRoot, "web/satellite/static/emails"),
|
||||
},
|
||||
Console: consoleweb.Config{
|
||||
Address: "127.0.0.1:0",
|
||||
StaticDir: filepath.Join(developmentRoot, "web/satellite"),
|
||||
PasswordCost: console.TestPasswordCost,
|
||||
AuthTokenSecret: "my-suppa-secret-key",
|
||||
},
|
||||
Marketing: marketingweb.Config{
|
||||
Address: "127.0.0.1:0",
|
||||
StaticDir: filepath.Join(developmentRoot, "web/marketing"),
|
||||
},
|
||||
Vouchers: vouchers.Config{
|
||||
Expiration: 30,
|
||||
},
|
||||
Version: planet.NewVersionConfig(),
|
||||
}
|
||||
if planet.config.Reconfigure.Satellite != nil {
|
||||
planet.config.Reconfigure.Satellite(log, i, &config)
|
||||
}
|
||||
|
||||
verInfo := planet.NewVersionInfo()
|
||||
|
||||
peer, err := satellite.New(log, identity, db, &config, verInfo)
|
||||
if err != nil {
|
||||
return xs, err
|
||||
}
|
||||
|
||||
log.Debug("id=" + peer.ID().String() + " addr=" + peer.Addr())
|
||||
xs = append(xs, peer)
|
||||
}
|
||||
return xs, nil
|
||||
}
|
||||
145
internal/testplanet/storagenode.go
Normal file
145
internal/testplanet/storagenode.go
Normal file
@ -0,0 +1,145 @@
|
||||
// Copyright (C) 2019 Storj Labs, Inc.
|
||||
// See LICENSE for copying information
|
||||
|
||||
package testplanet
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"storj.io/storj/internal/memory"
|
||||
"storj.io/storj/pkg/kademlia"
|
||||
"storj.io/storj/pkg/peertls/extensions"
|
||||
"storj.io/storj/pkg/peertls/tlsopts"
|
||||
"storj.io/storj/pkg/server"
|
||||
"storj.io/storj/storagenode"
|
||||
"storj.io/storj/storagenode/collector"
|
||||
"storj.io/storj/storagenode/console/consoleserver"
|
||||
"storj.io/storj/storagenode/monitor"
|
||||
"storj.io/storj/storagenode/orders"
|
||||
"storj.io/storj/storagenode/piecestore"
|
||||
"storj.io/storj/storagenode/storagenodedb"
|
||||
)
|
||||
|
||||
// newStorageNodes initializes storage nodes
|
||||
func (planet *Planet) newStorageNodes(count int, whitelistedSatelliteIDs []string) ([]*storagenode.Peer, error) {
|
||||
var xs []*storagenode.Peer
|
||||
defer func() {
|
||||
for _, x := range xs {
|
||||
planet.peers = append(planet.peers, closablePeer{peer: x})
|
||||
}
|
||||
}()
|
||||
|
||||
for i := 0; i < count; i++ {
|
||||
prefix := "storage" + strconv.Itoa(i)
|
||||
log := planet.log.Named(prefix)
|
||||
storageDir := filepath.Join(planet.directory, prefix)
|
||||
|
||||
if err := os.MkdirAll(storageDir, 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
identity, err := planet.NewIdentity()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var db storagenode.DB
|
||||
if planet.config.Reconfigure.NewStorageNodeDB != nil {
|
||||
db, err = planet.config.Reconfigure.NewStorageNodeDB(i)
|
||||
} else {
|
||||
db, err = storagenodedb.NewInMemory(log.Named("db"), storageDir)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.CreateTables()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
planet.databases = append(planet.databases, db)
|
||||
|
||||
config := storagenode.Config{
|
||||
Server: server.Config{
|
||||
Address: "127.0.0.1:0",
|
||||
PrivateAddress: "127.0.0.1:0",
|
||||
|
||||
Config: tlsopts.Config{
|
||||
RevocationDBURL: "bolt://" + filepath.Join(storageDir, "revocation.db"),
|
||||
UsePeerCAWhitelist: true,
|
||||
PeerCAWhitelistPath: planet.whitelistPath,
|
||||
PeerIDVersions: "*",
|
||||
Extensions: extensions.Config{
|
||||
Revocation: false,
|
||||
WhitelistSignedLeaf: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
Kademlia: kademlia.Config{
|
||||
BootstrapBackoffBase: 500 * time.Millisecond,
|
||||
BootstrapBackoffMax: 2 * time.Second,
|
||||
Alpha: 5,
|
||||
DBPath: storageDir, // TODO: replace with master db
|
||||
Operator: kademlia.OperatorConfig{
|
||||
Email: prefix + "@mail.test",
|
||||
Wallet: "0x" + strings.Repeat("00", 20),
|
||||
},
|
||||
},
|
||||
Storage: piecestore.OldConfig{
|
||||
Path: "", // TODO: this argument won't be needed with master storagenodedb
|
||||
AllocatedDiskSpace: 1 * memory.GB,
|
||||
AllocatedBandwidth: memory.TB,
|
||||
KBucketRefreshInterval: time.Hour,
|
||||
|
||||
SatelliteIDRestriction: true,
|
||||
WhitelistedSatelliteIDs: strings.Join(whitelistedSatelliteIDs, ","),
|
||||
},
|
||||
Collector: collector.Config{
|
||||
Interval: time.Minute,
|
||||
},
|
||||
Console: consoleserver.Config{
|
||||
Address: "127.0.0.1:0",
|
||||
StaticDir: filepath.Join(developmentRoot, "web/operator/"),
|
||||
},
|
||||
Storage2: piecestore.Config{
|
||||
Sender: orders.SenderConfig{
|
||||
Interval: time.Hour,
|
||||
Timeout: time.Hour,
|
||||
},
|
||||
Monitor: monitor.Config{
|
||||
MinimumBandwidth: 100 * memory.MB,
|
||||
MinimumDiskSpace: 100 * memory.MB,
|
||||
},
|
||||
},
|
||||
Version: planet.NewVersionConfig(),
|
||||
}
|
||||
if planet.config.Reconfigure.StorageNode != nil {
|
||||
planet.config.Reconfigure.StorageNode(i, &config)
|
||||
}
|
||||
|
||||
newIPCount := planet.config.Reconfigure.NewIPCount
|
||||
if newIPCount > 0 {
|
||||
if i >= count-newIPCount {
|
||||
config.Server.Address = fmt.Sprintf("127.0.0.%d:0", i+1)
|
||||
config.Server.PrivateAddress = fmt.Sprintf("127.0.0.%d:0", i+1)
|
||||
}
|
||||
}
|
||||
|
||||
verInfo := planet.NewVersionInfo()
|
||||
|
||||
peer, err := storagenode.New(log, identity, db, config, verInfo)
|
||||
if err != nil {
|
||||
return xs, err
|
||||
}
|
||||
|
||||
log.Debug("id=" + peer.ID().String() + " addr=" + peer.Addr())
|
||||
xs = append(xs, peer)
|
||||
}
|
||||
return xs, nil
|
||||
}
|
||||
@ -43,6 +43,20 @@ type Uplink struct {
|
||||
APIKey map[storj.NodeID]string
|
||||
}
|
||||
|
||||
// newUplinks creates initializes uplinks, requires peer to have at least one satellite
|
||||
func (planet *Planet) newUplinks(prefix string, count, storageNodeCount int) ([]*Uplink, error) {
|
||||
var xs []*Uplink
|
||||
for i := 0; i < count; i++ {
|
||||
uplink, err := planet.newUplink(prefix+strconv.Itoa(i), storageNodeCount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
xs = append(xs, uplink)
|
||||
}
|
||||
|
||||
return xs, nil
|
||||
}
|
||||
|
||||
// newUplink creates a new uplink
|
||||
func (planet *Planet) newUplink(name string, storageNodeCount int) (*Uplink, error) {
|
||||
identity, err := planet.NewIdentity()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user