diff --git a/internal/testplanet/bootstrap.go b/internal/testplanet/bootstrap.go new file mode 100644 index 000000000..32de61f63 --- /dev/null +++ b/internal/testplanet/bootstrap.go @@ -0,0 +1,158 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information + +package testplanet + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "storj.io/storj/bootstrap" + "storj.io/storj/bootstrap/bootstrapdb" + "storj.io/storj/bootstrap/bootstrapweb/bootstrapserver" + "storj.io/storj/internal/version" + "storj.io/storj/pkg/kademlia" + "storj.io/storj/pkg/peertls/extensions" + "storj.io/storj/pkg/peertls/tlsopts" + "storj.io/storj/pkg/server" + "storj.io/storj/versioncontrol" +) + +// newBootstrap initializes the bootstrap node +func (planet *Planet) newBootstrap() (peer *bootstrap.Peer, err error) { + defer func() { + planet.peers = append(planet.peers, closablePeer{peer: peer}) + }() + + prefix := "bootstrap" + log := planet.log.Named(prefix) + dbDir := filepath.Join(planet.directory, prefix) + + if err := os.MkdirAll(dbDir, 0700); err != nil { + return nil, err + } + + identity, err := planet.NewIdentity() + if err != nil { + return nil, err + } + + var db bootstrap.DB + if planet.config.Reconfigure.NewBootstrapDB != nil { + db, err = planet.config.Reconfigure.NewBootstrapDB(0) + } else { + db, err = bootstrapdb.NewInMemory(dbDir) + } + + err = db.CreateTables() + if err != nil { + return nil, err + } + + planet.databases = append(planet.databases, db) + + config := bootstrap.Config{ + Server: server.Config{ + Address: "127.0.0.1:0", + PrivateAddress: "127.0.0.1:0", + + Config: tlsopts.Config{ + RevocationDBURL: "bolt://" + filepath.Join(dbDir, "revocation.db"), + UsePeerCAWhitelist: true, + PeerCAWhitelistPath: planet.whitelistPath, + PeerIDVersions: "latest", + Extensions: extensions.Config{ + Revocation: false, + WhitelistSignedLeaf: false, + }, + }, + }, + Kademlia: kademlia.Config{ + BootstrapBackoffBase: 500 * time.Millisecond, + BootstrapBackoffMax: 2 * time.Second, + Alpha: 5, + DBPath: dbDir, // TODO: replace with master db + Operator: kademlia.OperatorConfig{ + Email: prefix + "@mail.test", + Wallet: "0x" + strings.Repeat("00", 20), + }, + }, + Web: bootstrapserver.Config{ + Address: "127.0.0.1:0", + StaticDir: "./web/bootstrap", // TODO: for development only + }, + Version: planet.NewVersionConfig(), + } + if planet.config.Reconfigure.Bootstrap != nil { + planet.config.Reconfigure.Bootstrap(0, &config) + } + + var verInfo version.Info + verInfo = planet.NewVersionInfo() + + peer, err = bootstrap.New(log, identity, db, config, verInfo) + if err != nil { + return nil, err + } + + log.Debug("id=" + peer.ID().String() + " addr=" + peer.Addr()) + + return peer, nil +} + +// newVersionControlServer initializes the Versioning Server +func (planet *Planet) newVersionControlServer() (peer *versioncontrol.Peer, err error) { + + prefix := "versioncontrol" + log := planet.log.Named(prefix) + dbDir := filepath.Join(planet.directory, prefix) + + if err := os.MkdirAll(dbDir, 0700); err != nil { + return nil, err + } + + config := &versioncontrol.Config{ + Address: "127.0.0.1:0", + Versions: versioncontrol.ServiceVersions{ + Bootstrap: "v0.0.1", + Satellite: "v0.0.1", + Storagenode: "v0.0.1", + Uplink: "v0.0.1", + Gateway: "v0.0.1", + }, + } + peer, err = versioncontrol.New(log, config) + if err != nil { + return nil, err + } + + log.Debug(" addr= " + peer.Addr()) + + return peer, nil +} + +// NewVersionInfo returns the Version Info for this planet with tuned metrics. +func (planet *Planet) NewVersionInfo() version.Info { + info := version.Info{ + Timestamp: time.Now(), + CommitHash: "testplanet", + Version: version.SemVer{ + Major: 0, + Minor: 0, + Patch: 1}, + Release: false, + } + return info +} + +// NewVersionConfig returns the Version Config for this planet with tuned metrics. +func (planet *Planet) NewVersionConfig() version.Config { + return version.Config{ + ServerAddress: fmt.Sprintf("http://%s/", planet.VersionControl.Addr()), + RequestTimeout: time.Second * 15, + CheckInterval: time.Minute * 5, + } +} diff --git a/internal/testplanet/planet.go b/internal/testplanet/planet.go index 61b618074..7d9f0e642 100644 --- a/internal/testplanet/planet.go +++ b/internal/testplanet/planet.go @@ -7,14 +7,11 @@ package testplanet import ( "context" "errors" - "fmt" "io" "io/ioutil" "net" "os" "path/filepath" - "strconv" - "strings" "sync" "time" @@ -24,41 +21,13 @@ import ( "golang.org/x/sync/errgroup" "storj.io/storj/bootstrap" - "storj.io/storj/bootstrap/bootstrapdb" - "storj.io/storj/bootstrap/bootstrapweb/bootstrapserver" - "storj.io/storj/internal/memory" "storj.io/storj/internal/testidentity" - "storj.io/storj/internal/version" - "storj.io/storj/pkg/accounting/rollup" - "storj.io/storj/pkg/accounting/tally" - "storj.io/storj/pkg/audit" - "storj.io/storj/pkg/datarepair/checker" - "storj.io/storj/pkg/datarepair/repairer" - "storj.io/storj/pkg/discovery" "storj.io/storj/pkg/identity" - "storj.io/storj/pkg/kademlia" "storj.io/storj/pkg/overlay" "storj.io/storj/pkg/pb" - "storj.io/storj/pkg/peertls/extensions" - "storj.io/storj/pkg/peertls/tlsopts" - "storj.io/storj/pkg/server" "storj.io/storj/pkg/storj" "storj.io/storj/satellite" - "storj.io/storj/satellite/console" - "storj.io/storj/satellite/console/consoleweb" - "storj.io/storj/satellite/mailservice" - "storj.io/storj/satellite/marketing/marketingweb" - "storj.io/storj/satellite/metainfo" - satorders "storj.io/storj/satellite/orders" - "storj.io/storj/satellite/satellitedb" - "storj.io/storj/satellite/vouchers" "storj.io/storj/storagenode" - "storj.io/storj/storagenode/collector" - "storj.io/storj/storagenode/console/consoleserver" - "storj.io/storj/storagenode/monitor" - "storj.io/storj/storagenode/orders" - "storj.io/storj/storagenode/piecestore" - "storj.io/storj/storagenode/storagenodedb" "storj.io/storj/versioncontrol" ) @@ -368,444 +337,6 @@ func (planet *Planet) Shutdown() error { return errlist.Err() } -// newUplinks creates initializes uplinks, requires peer to have at least one satellite -func (planet *Planet) newUplinks(prefix string, count, storageNodeCount int) ([]*Uplink, error) { - var xs []*Uplink - for i := 0; i < count; i++ { - uplink, err := planet.newUplink(prefix+strconv.Itoa(i), storageNodeCount) - if err != nil { - return nil, err - } - xs = append(xs, uplink) - } - - return xs, nil -} - -// newSatellites initializes satellites -func (planet *Planet) newSatellites(count int) ([]*satellite.Peer, error) { - // TODO: move into separate file - var xs []*satellite.Peer - defer func() { - for _, x := range xs { - planet.peers = append(planet.peers, closablePeer{peer: x}) - } - }() - - for i := 0; i < count; i++ { - prefix := "satellite" + strconv.Itoa(i) - log := planet.log.Named(prefix) - - storageDir := filepath.Join(planet.directory, prefix) - if err := os.MkdirAll(storageDir, 0700); err != nil { - return nil, err - } - - identity, err := planet.NewIdentity() - if err != nil { - return nil, err - } - - var db satellite.DB - if planet.config.Reconfigure.NewSatelliteDB != nil { - db, err = planet.config.Reconfigure.NewSatelliteDB(log.Named("db"), i) - } else { - db, err = satellitedb.NewInMemory(log.Named("db")) - } - if err != nil { - return nil, err - } - - err = db.CreateTables() - if err != nil { - return nil, err - } - - planet.databases = append(planet.databases, db) - - config := satellite.Config{ - Server: server.Config{ - Address: "127.0.0.1:0", - PrivateAddress: "127.0.0.1:0", - - Config: tlsopts.Config{ - RevocationDBURL: "bolt://" + filepath.Join(storageDir, "revocation.db"), - UsePeerCAWhitelist: true, - PeerCAWhitelistPath: planet.whitelistPath, - PeerIDVersions: "latest", - Extensions: extensions.Config{ - Revocation: false, - WhitelistSignedLeaf: false, - }, - }, - }, - Kademlia: kademlia.Config{ - Alpha: 5, - BootstrapBackoffBase: 500 * time.Millisecond, - BootstrapBackoffMax: 2 * time.Second, - DBPath: storageDir, // TODO: replace with master db - Operator: kademlia.OperatorConfig{ - Email: prefix + "@mail.test", - Wallet: "0x" + strings.Repeat("00", 20), - }, - }, - Overlay: overlay.Config{ - Node: overlay.NodeSelectionConfig{ - UptimeRatio: 0, - UptimeCount: 0, - AuditSuccessRatio: 0, - AuditCount: 0, - NewNodePercentage: 0, - OnlineWindow: time.Hour, - DistinctIP: false, - - AuditReputationRepairWeight: 1, - AuditReputationUplinkWeight: 1, - AuditReputationAlpha0: 1, - AuditReputationBeta0: 0, - AuditReputationLambda: 1, - AuditReputationWeight: 1, - UptimeReputationRepairWeight: 1, - UptimeReputationUplinkWeight: 1, - UptimeReputationAlpha0: 1, - UptimeReputationBeta0: 0, - UptimeReputationLambda: 1, - UptimeReputationWeight: 1, - }, - }, - Discovery: discovery.Config{ - DiscoveryInterval: 1 * time.Second, - RefreshInterval: 1 * time.Second, - RefreshLimit: 100, - }, - Metainfo: metainfo.Config{ - DatabaseURL: "bolt://" + filepath.Join(storageDir, "pointers.db"), - MinRemoteSegmentSize: 0, // TODO: fix tests to work with 1024 - MaxInlineSegmentSize: 8000, - Overlay: true, - BwExpiration: 45, - }, - Orders: satorders.Config{ - Expiration: 45 * 24 * time.Hour, - }, - Checker: checker.Config{ - Interval: 30 * time.Second, - IrreparableInterval: 15 * time.Second, - }, - Repairer: repairer.Config{ - MaxRepair: 10, - Interval: time.Hour, - Timeout: 10 * time.Second, // Repairs can take up to 4 seconds. Leaving room for outliers - MaxBufferMem: 4 * memory.MiB, - }, - Audit: audit.Config{ - MaxRetriesStatDB: 0, - Interval: 30 * time.Second, - MinBytesPerSecond: 1 * memory.KB, - }, - Tally: tally.Config{ - Interval: 30 * time.Second, - }, - Rollup: rollup.Config{ - Interval: 2 * time.Minute, - MaxAlphaUsage: 25 * memory.GB, - DeleteTallies: false, - }, - Mail: mailservice.Config{ - SMTPServerAddress: "smtp.mail.test:587", - From: "Labs ", - AuthType: "simulate", - TemplatePath: filepath.Join(developmentRoot, "web/satellite/static/emails"), - }, - Console: consoleweb.Config{ - Address: "127.0.0.1:0", - StaticDir: filepath.Join(developmentRoot, "web/satellite"), - PasswordCost: console.TestPasswordCost, - AuthTokenSecret: "my-suppa-secret-key", - }, - Marketing: marketingweb.Config{ - Address: "127.0.0.1:0", - StaticDir: filepath.Join(developmentRoot, "web/marketing"), - }, - Vouchers: vouchers.Config{ - Expiration: 30, - }, - Version: planet.NewVersionConfig(), - } - if planet.config.Reconfigure.Satellite != nil { - planet.config.Reconfigure.Satellite(log, i, &config) - } - - verInfo := planet.NewVersionInfo() - - peer, err := satellite.New(log, identity, db, &config, verInfo) - if err != nil { - return xs, err - } - - log.Debug("id=" + peer.ID().String() + " addr=" + peer.Addr()) - xs = append(xs, peer) - } - return xs, nil -} - -// newStorageNodes initializes storage nodes -func (planet *Planet) newStorageNodes(count int, whitelistedSatelliteIDs []string) ([]*storagenode.Peer, error) { - // TODO: move into separate file - var xs []*storagenode.Peer - defer func() { - for _, x := range xs { - planet.peers = append(planet.peers, closablePeer{peer: x}) - } - }() - - for i := 0; i < count; i++ { - prefix := "storage" + strconv.Itoa(i) - log := planet.log.Named(prefix) - storageDir := filepath.Join(planet.directory, prefix) - - if err := os.MkdirAll(storageDir, 0700); err != nil { - return nil, err - } - - identity, err := planet.NewIdentity() - if err != nil { - return nil, err - } - - var db storagenode.DB - if planet.config.Reconfigure.NewStorageNodeDB != nil { - db, err = planet.config.Reconfigure.NewStorageNodeDB(i) - } else { - db, err = storagenodedb.NewInMemory(log.Named("db"), storageDir) - } - if err != nil { - return nil, err - } - - err = db.CreateTables() - if err != nil { - return nil, err - } - - planet.databases = append(planet.databases, db) - - config := storagenode.Config{ - Server: server.Config{ - Address: "127.0.0.1:0", - PrivateAddress: "127.0.0.1:0", - - Config: tlsopts.Config{ - RevocationDBURL: "bolt://" + filepath.Join(storageDir, "revocation.db"), - UsePeerCAWhitelist: true, - PeerCAWhitelistPath: planet.whitelistPath, - PeerIDVersions: "*", - Extensions: extensions.Config{ - Revocation: false, - WhitelistSignedLeaf: false, - }, - }, - }, - Kademlia: kademlia.Config{ - BootstrapBackoffBase: 500 * time.Millisecond, - BootstrapBackoffMax: 2 * time.Second, - Alpha: 5, - DBPath: storageDir, // TODO: replace with master db - Operator: kademlia.OperatorConfig{ - Email: prefix + "@mail.test", - Wallet: "0x" + strings.Repeat("00", 20), - }, - }, - Storage: piecestore.OldConfig{ - Path: "", // TODO: this argument won't be needed with master storagenodedb - AllocatedDiskSpace: 1 * memory.GB, - AllocatedBandwidth: memory.TB, - KBucketRefreshInterval: time.Hour, - - SatelliteIDRestriction: true, - WhitelistedSatelliteIDs: strings.Join(whitelistedSatelliteIDs, ","), - }, - Collector: collector.Config{ - Interval: time.Minute, - }, - Console: consoleserver.Config{ - Address: "127.0.0.1:0", - StaticDir: filepath.Join(developmentRoot, "web/operator/"), - }, - Storage2: piecestore.Config{ - Sender: orders.SenderConfig{ - Interval: time.Hour, - Timeout: time.Hour, - }, - Monitor: monitor.Config{ - MinimumBandwidth: 100 * memory.MB, - MinimumDiskSpace: 100 * memory.MB, - }, - }, - Version: planet.NewVersionConfig(), - } - if planet.config.Reconfigure.StorageNode != nil { - planet.config.Reconfigure.StorageNode(i, &config) - } - - newIPCount := planet.config.Reconfigure.NewIPCount - if newIPCount > 0 { - if i >= count-newIPCount { - config.Server.Address = fmt.Sprintf("127.0.0.%d:0", i+1) - config.Server.PrivateAddress = fmt.Sprintf("127.0.0.%d:0", i+1) - } - } - - verInfo := planet.NewVersionInfo() - - peer, err := storagenode.New(log, identity, db, config, verInfo) - if err != nil { - return xs, err - } - - log.Debug("id=" + peer.ID().String() + " addr=" + peer.Addr()) - xs = append(xs, peer) - } - return xs, nil -} - -// newBootstrap initializes the bootstrap node -func (planet *Planet) newBootstrap() (peer *bootstrap.Peer, err error) { - // TODO: move into separate file - defer func() { - planet.peers = append(planet.peers, closablePeer{peer: peer}) - }() - - prefix := "bootstrap" - log := planet.log.Named(prefix) - dbDir := filepath.Join(planet.directory, prefix) - - if err := os.MkdirAll(dbDir, 0700); err != nil { - return nil, err - } - - identity, err := planet.NewIdentity() - if err != nil { - return nil, err - } - - var db bootstrap.DB - if planet.config.Reconfigure.NewBootstrapDB != nil { - db, err = planet.config.Reconfigure.NewBootstrapDB(0) - } else { - db, err = bootstrapdb.NewInMemory(dbDir) - } - - err = db.CreateTables() - if err != nil { - return nil, err - } - - planet.databases = append(planet.databases, db) - - config := bootstrap.Config{ - Server: server.Config{ - Address: "127.0.0.1:0", - PrivateAddress: "127.0.0.1:0", - - Config: tlsopts.Config{ - RevocationDBURL: "bolt://" + filepath.Join(dbDir, "revocation.db"), - UsePeerCAWhitelist: true, - PeerCAWhitelistPath: planet.whitelistPath, - PeerIDVersions: "latest", - Extensions: extensions.Config{ - Revocation: false, - WhitelistSignedLeaf: false, - }, - }, - }, - Kademlia: kademlia.Config{ - BootstrapBackoffBase: 500 * time.Millisecond, - BootstrapBackoffMax: 2 * time.Second, - Alpha: 5, - DBPath: dbDir, // TODO: replace with master db - Operator: kademlia.OperatorConfig{ - Email: prefix + "@mail.test", - Wallet: "0x" + strings.Repeat("00", 20), - }, - }, - Web: bootstrapserver.Config{ - Address: "127.0.0.1:0", - StaticDir: "./web/bootstrap", // TODO: for development only - }, - Version: planet.NewVersionConfig(), - } - if planet.config.Reconfigure.Bootstrap != nil { - planet.config.Reconfigure.Bootstrap(0, &config) - } - - var verInfo version.Info - verInfo = planet.NewVersionInfo() - - peer, err = bootstrap.New(log, identity, db, config, verInfo) - if err != nil { - return nil, err - } - - log.Debug("id=" + peer.ID().String() + " addr=" + peer.Addr()) - - return peer, nil -} - -// newVersionControlServer initializes the Versioning Server -func (planet *Planet) newVersionControlServer() (peer *versioncontrol.Peer, err error) { - - prefix := "versioncontrol" - log := planet.log.Named(prefix) - dbDir := filepath.Join(planet.directory, prefix) - - if err := os.MkdirAll(dbDir, 0700); err != nil { - return nil, err - } - - config := &versioncontrol.Config{ - Address: "127.0.0.1:0", - Versions: versioncontrol.ServiceVersions{ - Bootstrap: "v0.0.1", - Satellite: "v0.0.1", - Storagenode: "v0.0.1", - Uplink: "v0.0.1", - Gateway: "v0.0.1", - }, - } - peer, err = versioncontrol.New(log, config) - if err != nil { - return nil, err - } - - log.Debug(" addr= " + peer.Addr()) - - return peer, nil -} - -// NewVersionInfo returns the Version Info for this planet with tuned metrics. -func (planet *Planet) NewVersionInfo() version.Info { - info := version.Info{ - Timestamp: time.Now(), - CommitHash: "testplanet", - Version: version.SemVer{ - Major: 0, - Minor: 0, - Patch: 1}, - Release: false, - } - return info -} - -// NewVersionConfig returns the Version Config for this planet with tuned metrics. -func (planet *Planet) NewVersionConfig() version.Config { - return version.Config{ - ServerAddress: fmt.Sprintf("http://%s/", planet.VersionControl.Addr()), - RequestTimeout: time.Second * 15, - CheckInterval: time.Minute * 5, - } -} - // Identities returns the identity provider for this planet. func (planet *Planet) Identities() *testidentity.Identities { return planet.identities diff --git a/internal/testplanet/satellite.go b/internal/testplanet/satellite.go new file mode 100644 index 000000000..65f194b29 --- /dev/null +++ b/internal/testplanet/satellite.go @@ -0,0 +1,200 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information + +package testplanet + +import ( + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "storj.io/storj/internal/memory" + "storj.io/storj/pkg/accounting/rollup" + "storj.io/storj/pkg/accounting/tally" + "storj.io/storj/pkg/audit" + "storj.io/storj/pkg/datarepair/checker" + "storj.io/storj/pkg/datarepair/repairer" + "storj.io/storj/pkg/discovery" + "storj.io/storj/pkg/kademlia" + "storj.io/storj/pkg/overlay" + "storj.io/storj/pkg/peertls/extensions" + "storj.io/storj/pkg/peertls/tlsopts" + "storj.io/storj/pkg/server" + "storj.io/storj/satellite" + "storj.io/storj/satellite/console" + "storj.io/storj/satellite/console/consoleweb" + "storj.io/storj/satellite/mailservice" + "storj.io/storj/satellite/marketing/marketingweb" + "storj.io/storj/satellite/metainfo" + "storj.io/storj/satellite/orders" + "storj.io/storj/satellite/satellitedb" + "storj.io/storj/satellite/vouchers" +) + +// newSatellites initializes satellites +func (planet *Planet) newSatellites(count int) ([]*satellite.Peer, error) { + var xs []*satellite.Peer + defer func() { + for _, x := range xs { + planet.peers = append(planet.peers, closablePeer{peer: x}) + } + }() + + for i := 0; i < count; i++ { + prefix := "satellite" + strconv.Itoa(i) + log := planet.log.Named(prefix) + + storageDir := filepath.Join(planet.directory, prefix) + if err := os.MkdirAll(storageDir, 0700); err != nil { + return nil, err + } + + identity, err := planet.NewIdentity() + if err != nil { + return nil, err + } + + var db satellite.DB + if planet.config.Reconfigure.NewSatelliteDB != nil { + db, err = planet.config.Reconfigure.NewSatelliteDB(log.Named("db"), i) + } else { + db, err = satellitedb.NewInMemory(log.Named("db")) + } + if err != nil { + return nil, err + } + + err = db.CreateTables() + if err != nil { + return nil, err + } + + planet.databases = append(planet.databases, db) + + config := satellite.Config{ + Server: server.Config{ + Address: "127.0.0.1:0", + PrivateAddress: "127.0.0.1:0", + + Config: tlsopts.Config{ + RevocationDBURL: "bolt://" + filepath.Join(storageDir, "revocation.db"), + UsePeerCAWhitelist: true, + PeerCAWhitelistPath: planet.whitelistPath, + PeerIDVersions: "latest", + Extensions: extensions.Config{ + Revocation: false, + WhitelistSignedLeaf: false, + }, + }, + }, + Kademlia: kademlia.Config{ + Alpha: 5, + BootstrapBackoffBase: 500 * time.Millisecond, + BootstrapBackoffMax: 2 * time.Second, + DBPath: storageDir, // TODO: replace with master db + Operator: kademlia.OperatorConfig{ + Email: prefix + "@mail.test", + Wallet: "0x" + strings.Repeat("00", 20), + }, + }, + Overlay: overlay.Config{ + Node: overlay.NodeSelectionConfig{ + UptimeRatio: 0, + UptimeCount: 0, + AuditSuccessRatio: 0, + AuditCount: 0, + NewNodePercentage: 0, + OnlineWindow: time.Hour, + DistinctIP: false, + + AuditReputationRepairWeight: 1, + AuditReputationUplinkWeight: 1, + AuditReputationAlpha0: 1, + AuditReputationBeta0: 0, + AuditReputationLambda: 1, + AuditReputationWeight: 1, + UptimeReputationRepairWeight: 1, + UptimeReputationUplinkWeight: 1, + UptimeReputationAlpha0: 1, + UptimeReputationBeta0: 0, + UptimeReputationLambda: 1, + UptimeReputationWeight: 1, + }, + }, + Discovery: discovery.Config{ + DiscoveryInterval: 1 * time.Second, + RefreshInterval: 1 * time.Second, + RefreshLimit: 100, + }, + Metainfo: metainfo.Config{ + DatabaseURL: "bolt://" + filepath.Join(storageDir, "pointers.db"), + MinRemoteSegmentSize: 0, // TODO: fix tests to work with 1024 + MaxInlineSegmentSize: 8000, + Overlay: true, + BwExpiration: 45, + }, + Orders: orders.Config{ + Expiration: 45 * 24 * time.Hour, + }, + Checker: checker.Config{ + Interval: 30 * time.Second, + IrreparableInterval: 15 * time.Second, + }, + Repairer: repairer.Config{ + MaxRepair: 10, + Interval: time.Hour, + Timeout: 10 * time.Second, // Repairs can take up to 4 seconds. Leaving room for outliers + MaxBufferMem: 4 * memory.MiB, + }, + Audit: audit.Config{ + MaxRetriesStatDB: 0, + Interval: 30 * time.Second, + MinBytesPerSecond: 1 * memory.KB, + }, + Tally: tally.Config{ + Interval: 30 * time.Second, + }, + Rollup: rollup.Config{ + Interval: 2 * time.Minute, + MaxAlphaUsage: 25 * memory.GB, + DeleteTallies: false, + }, + Mail: mailservice.Config{ + SMTPServerAddress: "smtp.mail.test:587", + From: "Labs ", + AuthType: "simulate", + TemplatePath: filepath.Join(developmentRoot, "web/satellite/static/emails"), + }, + Console: consoleweb.Config{ + Address: "127.0.0.1:0", + StaticDir: filepath.Join(developmentRoot, "web/satellite"), + PasswordCost: console.TestPasswordCost, + AuthTokenSecret: "my-suppa-secret-key", + }, + Marketing: marketingweb.Config{ + Address: "127.0.0.1:0", + StaticDir: filepath.Join(developmentRoot, "web/marketing"), + }, + Vouchers: vouchers.Config{ + Expiration: 30, + }, + Version: planet.NewVersionConfig(), + } + if planet.config.Reconfigure.Satellite != nil { + planet.config.Reconfigure.Satellite(log, i, &config) + } + + verInfo := planet.NewVersionInfo() + + peer, err := satellite.New(log, identity, db, &config, verInfo) + if err != nil { + return xs, err + } + + log.Debug("id=" + peer.ID().String() + " addr=" + peer.Addr()) + xs = append(xs, peer) + } + return xs, nil +} diff --git a/internal/testplanet/storagenode.go b/internal/testplanet/storagenode.go new file mode 100644 index 000000000..082ce7a49 --- /dev/null +++ b/internal/testplanet/storagenode.go @@ -0,0 +1,145 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information + +package testplanet + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "storj.io/storj/internal/memory" + "storj.io/storj/pkg/kademlia" + "storj.io/storj/pkg/peertls/extensions" + "storj.io/storj/pkg/peertls/tlsopts" + "storj.io/storj/pkg/server" + "storj.io/storj/storagenode" + "storj.io/storj/storagenode/collector" + "storj.io/storj/storagenode/console/consoleserver" + "storj.io/storj/storagenode/monitor" + "storj.io/storj/storagenode/orders" + "storj.io/storj/storagenode/piecestore" + "storj.io/storj/storagenode/storagenodedb" +) + +// newStorageNodes initializes storage nodes +func (planet *Planet) newStorageNodes(count int, whitelistedSatelliteIDs []string) ([]*storagenode.Peer, error) { + var xs []*storagenode.Peer + defer func() { + for _, x := range xs { + planet.peers = append(planet.peers, closablePeer{peer: x}) + } + }() + + for i := 0; i < count; i++ { + prefix := "storage" + strconv.Itoa(i) + log := planet.log.Named(prefix) + storageDir := filepath.Join(planet.directory, prefix) + + if err := os.MkdirAll(storageDir, 0700); err != nil { + return nil, err + } + + identity, err := planet.NewIdentity() + if err != nil { + return nil, err + } + + var db storagenode.DB + if planet.config.Reconfigure.NewStorageNodeDB != nil { + db, err = planet.config.Reconfigure.NewStorageNodeDB(i) + } else { + db, err = storagenodedb.NewInMemory(log.Named("db"), storageDir) + } + if err != nil { + return nil, err + } + + err = db.CreateTables() + if err != nil { + return nil, err + } + + planet.databases = append(planet.databases, db) + + config := storagenode.Config{ + Server: server.Config{ + Address: "127.0.0.1:0", + PrivateAddress: "127.0.0.1:0", + + Config: tlsopts.Config{ + RevocationDBURL: "bolt://" + filepath.Join(storageDir, "revocation.db"), + UsePeerCAWhitelist: true, + PeerCAWhitelistPath: planet.whitelistPath, + PeerIDVersions: "*", + Extensions: extensions.Config{ + Revocation: false, + WhitelistSignedLeaf: false, + }, + }, + }, + Kademlia: kademlia.Config{ + BootstrapBackoffBase: 500 * time.Millisecond, + BootstrapBackoffMax: 2 * time.Second, + Alpha: 5, + DBPath: storageDir, // TODO: replace with master db + Operator: kademlia.OperatorConfig{ + Email: prefix + "@mail.test", + Wallet: "0x" + strings.Repeat("00", 20), + }, + }, + Storage: piecestore.OldConfig{ + Path: "", // TODO: this argument won't be needed with master storagenodedb + AllocatedDiskSpace: 1 * memory.GB, + AllocatedBandwidth: memory.TB, + KBucketRefreshInterval: time.Hour, + + SatelliteIDRestriction: true, + WhitelistedSatelliteIDs: strings.Join(whitelistedSatelliteIDs, ","), + }, + Collector: collector.Config{ + Interval: time.Minute, + }, + Console: consoleserver.Config{ + Address: "127.0.0.1:0", + StaticDir: filepath.Join(developmentRoot, "web/operator/"), + }, + Storage2: piecestore.Config{ + Sender: orders.SenderConfig{ + Interval: time.Hour, + Timeout: time.Hour, + }, + Monitor: monitor.Config{ + MinimumBandwidth: 100 * memory.MB, + MinimumDiskSpace: 100 * memory.MB, + }, + }, + Version: planet.NewVersionConfig(), + } + if planet.config.Reconfigure.StorageNode != nil { + planet.config.Reconfigure.StorageNode(i, &config) + } + + newIPCount := planet.config.Reconfigure.NewIPCount + if newIPCount > 0 { + if i >= count-newIPCount { + config.Server.Address = fmt.Sprintf("127.0.0.%d:0", i+1) + config.Server.PrivateAddress = fmt.Sprintf("127.0.0.%d:0", i+1) + } + } + + verInfo := planet.NewVersionInfo() + + peer, err := storagenode.New(log, identity, db, config, verInfo) + if err != nil { + return xs, err + } + + log.Debug("id=" + peer.ID().String() + " addr=" + peer.Addr()) + xs = append(xs, peer) + } + return xs, nil +} diff --git a/internal/testplanet/uplink.go b/internal/testplanet/uplink.go index 338f7495c..c53201076 100644 --- a/internal/testplanet/uplink.go +++ b/internal/testplanet/uplink.go @@ -43,6 +43,20 @@ type Uplink struct { APIKey map[storj.NodeID]string } +// newUplinks creates initializes uplinks, requires peer to have at least one satellite +func (planet *Planet) newUplinks(prefix string, count, storageNodeCount int) ([]*Uplink, error) { + var xs []*Uplink + for i := 0; i < count; i++ { + uplink, err := planet.newUplink(prefix+strconv.Itoa(i), storageNodeCount) + if err != nil { + return nil, err + } + xs = append(xs, uplink) + } + + return xs, nil +} + // newUplink creates a new uplink func (planet *Planet) newUplink(name string, storageNodeCount int) (*Uplink, error) { identity, err := planet.NewIdentity()