JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

capt: reduce nodes to 10 (#793)

Browse Source 
Change-Id: Ief380fe29e3043657705cd7505c266fd774181a4
This commit is contained in:
JT Olio 2018-12-11 11:40:54 -07:00 committed by GitHub
parent ef5adfb65e
commit ceb590fa67
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 47 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/captplanet/run.go
View File

@ -33,7 +33,7 @@ import (
)
const (
storagenodeCount = 100
storagenodeCount = 10
)
// Satellite is for configuring client

17
cmd/captplanet/setup.go
View File

@ -145,6 +145,7 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
setupCfg.BasePath, "satellite", "pointerdb.db"),
"satellite.overlay.database-url": "bolt://" + filepath.Join(
setupCfg.BasePath, "satellite", "overlay.db"),
"satellite.kademlia.alpha": 3,
"satellite.repairer.queue-address": "redis://127.0.0.1:6378?db=1&password=abc123",
"satellite.repairer.overlay-addr": overlayAddr,
"satellite.repairer.pointer-db-addr": joinHostPort(
@ -152,7 +153,7 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
"satellite.repairer.api-key": setupCfg.APIKey,
"uplink.identity.cert-path": setupCfg.UplinkIdentity.CertPath,
"uplink.identity.key-path": setupCfg.UplinkIdentity.KeyPath,
"uplink.identity.address": joinHostPort(
"uplink.identity.server.address": joinHostPort(
setupCfg.ListenHost, startingPort),
"uplink.client.overlay-addr": joinHostPort(
setupCfg.ListenHost, startingPort+1),
@ -162,9 +163,18 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
setupCfg.BasePath, "uplink", "minio"),
"uplink.enc.key": setupCfg.EncKey,
"uplink.client.api-key": setupCfg.APIKey,
"uplink.rs.min-threshold": 1 * len(runCfg.StorageNodes) / 5,
"uplink.rs.repair-threshold": 2 * len(runCfg.StorageNodes) / 5,
"uplink.rs.success-threshold": 3 * len(runCfg.StorageNodes) / 5,
"uplink.rs.max-threshold": 4 * len(runCfg.StorageNodes) / 5,
"kademlia.bucket-size": 4,
"kademlia.replacement-cache-size": 1,
// TODO: this will eventually go away
"pointer-db.auth.api-key": setupCfg.APIKey,
// Repairer
// TODO: this is a source of bugs. this value should be pulled from
// kademlia instead
"piecestore.agreementsender.overlay_addr": overlayAddr,
"log.development": true,
@ -173,7 +183,7 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
for i := 0; i < len(runCfg.StorageNodes); i++ {
storagenodePath := filepath.Join(setupCfg.BasePath, fmt.Sprintf("f%d", i))
storagenode := fmt.Sprintf("storage-nodes.%03d.", i)
storagenode := fmt.Sprintf("storage-nodes.%02d.", i)
overrides[storagenode+"identity.cert-path"] = filepath.Join(
storagenodePath, "identity.cert")
overrides[storagenode+"identity.key-path"] = filepath.Join(
@ -183,6 +193,7 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
overrides[storagenode+"kademlia.bootstrap-addr"] = joinHostPort(
setupCfg.ListenHost, startingPort+1)
overrides[storagenode+"storage.path"] = filepath.Join(storagenodePath, "data")
overrides[storagenode+"kademlia.alpha"] = 3
}
return process.SaveConfig(runCmd.Flags(),

3
cmd/storagenode/main.go
View File

@ -61,11 +61,12 @@ var (
}
defaultConfDir string
defaultDiagDir = "$HOME/.storj/capt/f37/data"
defaultDiagDir string
)
func init() {
defaultConfDir = fpath.ApplicationDir("storj", "storagenode")
defaultDiagDir = filepath.Join(defaultConfDir, "storage")
rootCmd.AddCommand(runCmd)
rootCmd.AddCommand(setupCmd)
rootCmd.AddCommand(diagCmd)

20
pkg/datarepair/repairer/config.go
View File

@ -27,9 +27,25 @@ type Config struct {
QueueAddress string `help:"data repair queue address" default:"redis://127.0.0.1:6378?db=1&password=abc123"`
MaxRepair int `help:"maximum segments that can be repaired concurrently" default:"100"`
Interval time.Duration `help:"how frequently checker should audit segments" default:"3600s"`
OverlayAddr string `help:"Address to contact overlay server through"`
PointerDBAddr string `help:"Address to contact pointerdb server through"`
MaxBufferMem int `help:"maximum buffer memory (in bytes) to be allocated for read buffers" default:"0x400000"`
APIKey string `help:"repairer-specific pointerdb access credential"`
miniogw.NodeSelectionConfig
miniogw.ClientConfig
miniogw.RSConfig
// TODO: this is a huge bug that these are required here. these values should
// all come from the pointer for each repair. these need to be removed from the
// config
MinThreshold int `help:"TODO: remove" default:"29"`
RepairThreshold int `help:"TODO: remove" default:"35"`
SuccessThreshold int `help:"TODO: remove" default:"80"`
MaxThreshold int `help:"TODO: remove" default:"95"`
ErasureShareSize int `help:"TODO: remove" default:"1024"`
// TODO: the repairer shouldn't need to worry about inlining, as it is only
// repairing non-inlined things.
MaxInlineSize int `help:"TODO: remove" default:"4096"`
}
// Run runs the repairer with configured values

8
pkg/kademlia/config.go
View File

@ -23,13 +23,9 @@ var (
mon = monkit.Package()
)
const (
defaultAlpha = 5
)
var (
flagBucketSize = flag.Int("kademlia-bucket-size", 20, "Size of each Kademlia bucket")
flagReplacementCacheSize = flag.Int("kademlia-replacement-cache-size", 5, "Size of Kademlia replacement cache")
flagBucketSize = flag.Int("kademlia.bucket-size", 20, "Size of each Kademlia bucket")
flagReplacementCacheSize = flag.Int("kademlia.replacement-cache-size", 5, "Size of Kademlia replacement cache")
)
//CtxKey Used as kademlia key

4
pkg/kademlia/kademlia_test.go
View File

@ -26,6 +26,10 @@ import (
"storj.io/storj/pkg/storj"
)
const (
defaultAlpha = 5
)
func TestNewKademlia(t *testing.T) {
rootdir, cleanup := mktempdir(t, "kademlia")
defer cleanup()

2
pkg/provider/certificate_authority.go
View File

@ -45,7 +45,7 @@ type CASetupConfig struct {
ParentKeyPath string `help:"path to the parent authority's private key"`
CertPath string `help:"path to the certificate chain for this identity" default:"$CONFDIR/ca.cert"`
KeyPath string `help:"path to the private key for this identity" default:"$CONFDIR/ca.key"`
Difficulty uint64 `help:"minimum difficulty for identity generation" default:"12"`
Difficulty uint64 `help:"minimum difficulty for identity generation" default:"15"`
Timeout string `help:"timeout for CA generation; golang duration string (0 no timeout)" default:"5m"`
Overwrite bool `help:"if true, existing CA certs AND keys will overwritten" default:"false"`
Concurrency uint `help:"number of concurrent workers for certificate authority generation" default:"4"`