Make kademlia use less file-descriptors (#498)
This commit is contained in:
parent
21026b35f5
commit
03bd93bba7
@ -5,7 +5,6 @@ package kademlia
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
@ -20,7 +19,9 @@ import (
|
||||
"storj.io/storj/pkg/node"
|
||||
"storj.io/storj/pkg/pb"
|
||||
"storj.io/storj/pkg/provider"
|
||||
"storj.io/storj/pkg/utils"
|
||||
"storj.io/storj/storage"
|
||||
"storj.io/storj/storage/boltdb"
|
||||
)
|
||||
|
||||
// NodeErr is the class for all errors pertaining to node operations
|
||||
@ -59,14 +60,22 @@ func NewKademlia(id dht.NodeID, bootstrapNodes []pb.Node, address string, identi
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
bucketIdentifier := id.String()[:5] // need a way to differentiate between nodes if running more than one simultaneously
|
||||
rt, err := NewRoutingTable(&self, &RoutingOptions{
|
||||
kpath: filepath.Join(path, fmt.Sprintf("kbucket_%s.db", bucketIdentifier)),
|
||||
npath: filepath.Join(path, fmt.Sprintf("nbucket_%s.db", bucketIdentifier)),
|
||||
dbpath := filepath.Join(path, fmt.Sprintf("kademlia_%s.db", bucketIdentifier))
|
||||
|
||||
dbs, err := boltdb.NewShared(dbpath, KademliaBucket, NodeBucket)
|
||||
if err != nil {
|
||||
return nil, BootstrapErr.Wrap(err)
|
||||
}
|
||||
kdb, ndb := dbs[0], dbs[1]
|
||||
|
||||
rt, err := NewRoutingTable(&self, kdb, ndb, &RoutingOptions{
|
||||
idLength: kadconfig.DefaultIDLength,
|
||||
bucketSize: kadconfig.DefaultBucketSize,
|
||||
rcBucketSize: kadconfig.DefaultReplacementCacheSize,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, BootstrapErr.Wrap(err)
|
||||
}
|
||||
@ -101,8 +110,10 @@ func NewKademlia(id dht.NodeID, bootstrapNodes []pb.Node, address string, identi
|
||||
|
||||
// Disconnect safely closes connections to the Kademlia network
|
||||
func (k *Kademlia) Disconnect() error {
|
||||
// TODO(coyle)
|
||||
return errors.New("TODO Disconnect")
|
||||
return utils.CombineErrors(
|
||||
k.routingTable.Close(),
|
||||
// TODO: close connections
|
||||
)
|
||||
}
|
||||
|
||||
// GetNodes returns all nodes from a starting node up to a maximum limit
|
||||
|
@ -16,7 +16,6 @@ import (
|
||||
"storj.io/storj/pkg/pb"
|
||||
"storj.io/storj/pkg/utils"
|
||||
"storj.io/storj/storage"
|
||||
"storj.io/storj/storage/boltdb"
|
||||
"storj.io/storj/storage/storelogger"
|
||||
)
|
||||
|
||||
@ -45,32 +44,20 @@ type RoutingTable struct {
|
||||
|
||||
//RoutingOptions for configuring RoutingTable
|
||||
type RoutingOptions struct {
|
||||
kpath string
|
||||
npath string
|
||||
idLength int //TODO (JJ): add checks for > 0
|
||||
bucketSize int
|
||||
rcBucketSize int
|
||||
}
|
||||
|
||||
// NewRoutingTable returns a newly configured instance of a RoutingTable
|
||||
func NewRoutingTable(localNode *pb.Node, options *RoutingOptions) (*RoutingTable, error) {
|
||||
kdb, err := boltdb.New(options.kpath, KademliaBucket)
|
||||
if err != nil {
|
||||
return nil, RoutingErr.New("could not create kadBucketDB: %s", err)
|
||||
}
|
||||
|
||||
ndb, err := boltdb.New(options.npath, NodeBucket)
|
||||
if err != nil {
|
||||
return nil, RoutingErr.New("could not create nodeBucketDB: %s", err)
|
||||
}
|
||||
rp := make(map[string][]*pb.Node)
|
||||
func NewRoutingTable(localNode *pb.Node, kdb, ndb storage.KeyValueStore, options *RoutingOptions) (*RoutingTable, error) {
|
||||
rt := &RoutingTable{
|
||||
self: localNode,
|
||||
kadBucketDB: storelogger.New(zap.L(), kdb),
|
||||
nodeBucketDB: storelogger.New(zap.L(), ndb),
|
||||
transport: &defaultTransport,
|
||||
mutex: &sync.Mutex{},
|
||||
replacementCache: rp,
|
||||
replacementCache: make(map[string][]*pb.Node),
|
||||
idLength: options.idLength,
|
||||
bucketSize: options.bucketSize,
|
||||
rcBucketSize: options.rcBucketSize,
|
||||
@ -84,9 +71,10 @@ func NewRoutingTable(localNode *pb.Node, options *RoutingOptions) (*RoutingTable
|
||||
|
||||
// Close closes underlying databases
|
||||
func (rt *RoutingTable) Close() error {
|
||||
kerr := rt.kadBucketDB.Close()
|
||||
nerr := rt.nodeBucketDB.Close()
|
||||
return utils.CombineErrors(kerr, nerr)
|
||||
return utils.CombineErrors(
|
||||
rt.kadBucketDB.Close(),
|
||||
rt.nodeBucketDB.Close(),
|
||||
)
|
||||
}
|
||||
|
||||
// Local returns the local nodes ID
|
||||
|
@ -4,9 +4,6 @@
|
||||
package kademlia
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -16,41 +13,31 @@ import (
|
||||
|
||||
"storj.io/storj/pkg/pb"
|
||||
"storj.io/storj/storage"
|
||||
"storj.io/storj/storage/teststore"
|
||||
)
|
||||
|
||||
func tempdir(t testing.TB) (dir string, cleanup func()) {
|
||||
dir, err := ioutil.TempDir("", "storj-kademlia")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dir, func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func createRoutingTable(t *testing.T, localNodeID []byte) (*RoutingTable, func()) {
|
||||
tempdir, cleanup := tempdir(t)
|
||||
|
||||
if localNodeID == nil {
|
||||
localNodeID = []byte("AA")
|
||||
}
|
||||
localNode := &pb.Node{Id: string(localNodeID)}
|
||||
|
||||
options := &RoutingOptions{
|
||||
kpath: filepath.Join(tempdir, "Kadbucket"),
|
||||
npath: filepath.Join(tempdir, "Nodebucket"),
|
||||
idLength: 16,
|
||||
bucketSize: 6,
|
||||
rcBucketSize: 2,
|
||||
}
|
||||
rt, err := NewRoutingTable(localNode, options)
|
||||
rt, err := NewRoutingTable(localNode,
|
||||
teststore.New(),
|
||||
teststore.New(),
|
||||
options,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return rt, func() {
|
||||
err := rt.Close()
|
||||
cleanup()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
@ -5,6 +5,7 @@ package boltdb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/boltdb/bolt"
|
||||
@ -18,6 +19,8 @@ type Client struct {
|
||||
db *bolt.DB
|
||||
Path string
|
||||
Bucket []byte
|
||||
|
||||
referenceCount *int32
|
||||
}
|
||||
|
||||
const (
|
||||
@ -45,13 +48,57 @@ func New(path, bucket string) (*Client, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
refCount := new(int32)
|
||||
*refCount = 1
|
||||
|
||||
return &Client{
|
||||
db: db,
|
||||
Path: path,
|
||||
Bucket: []byte(bucket),
|
||||
db: db,
|
||||
referenceCount: refCount,
|
||||
Path: path,
|
||||
Bucket: []byte(bucket),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewShared instantiates a new BoltDB with multiple buckets
|
||||
func NewShared(path string, buckets ...string) ([]*Client, error) {
|
||||
db, err := bolt.Open(path, fileMode, &bolt.Options{Timeout: defaultTimeout})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.Update(func(tx *bolt.Tx) error {
|
||||
for _, bucket := range buckets {
|
||||
_, err := tx.CreateBucketIfNotExists([]byte(bucket))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return err
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if closeErr := db.Close(); closeErr != nil {
|
||||
return nil, utils.CombineErrors(err, closeErr)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
refCount := new(int32)
|
||||
*refCount = int32(len(buckets))
|
||||
|
||||
clients := []*Client{}
|
||||
for _, bucket := range buckets {
|
||||
clients = append(clients, &Client{
|
||||
db: db,
|
||||
referenceCount: refCount,
|
||||
Path: path,
|
||||
Bucket: []byte(bucket),
|
||||
})
|
||||
}
|
||||
|
||||
return clients, nil
|
||||
}
|
||||
|
||||
func (client *Client) update(fn func(*bolt.Bucket) error) error {
|
||||
return client.db.Update(func(tx *bolt.Tx) error {
|
||||
return fn(tx.Bucket(client.Bucket))
|
||||
@ -108,7 +155,10 @@ func (client *Client) ReverseList(first storage.Key, limit int) (storage.Keys, e
|
||||
|
||||
// Close closes a BoltDB client
|
||||
func (client *Client) Close() error {
|
||||
return client.db.Close()
|
||||
if atomic.AddInt32(client.referenceCount, -1) == 0 {
|
||||
return client.db.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAll finds all values for the provided keys up to 100 keys
|
||||
|
@ -53,3 +53,28 @@ func BenchmarkSuite(b *testing.B) {
|
||||
|
||||
testsuite.RunBenchmarks(b, store)
|
||||
}
|
||||
|
||||
func TestSuiteShared(t *testing.T) {
|
||||
tempdir, err := ioutil.TempDir("", "storj-bolt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(tempdir) }()
|
||||
|
||||
dbname := filepath.Join(tempdir, "bolt.db")
|
||||
stores, err := NewShared(dbname, "alpha", "beta")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create db: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
for _, store := range stores {
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatalf("failed to close db: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
for _, store := range stores {
|
||||
testsuite.RunTests(t, store)
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user