Make kademlia use less file-descriptors (#498)
This commit is contained in:
parent
21026b35f5
commit
03bd93bba7
@ -5,7 +5,6 @@ package kademlia
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
"os"
|
"os"
|
||||||
@ -20,7 +19,9 @@ import (
|
|||||||
"storj.io/storj/pkg/node"
|
"storj.io/storj/pkg/node"
|
||||||
"storj.io/storj/pkg/pb"
|
"storj.io/storj/pkg/pb"
|
||||||
"storj.io/storj/pkg/provider"
|
"storj.io/storj/pkg/provider"
|
||||||
|
"storj.io/storj/pkg/utils"
|
||||||
"storj.io/storj/storage"
|
"storj.io/storj/storage"
|
||||||
|
"storj.io/storj/storage/boltdb"
|
||||||
)
|
)
|
||||||
|
|
||||||
// NodeErr is the class for all errors pertaining to node operations
|
// NodeErr is the class for all errors pertaining to node operations
|
||||||
@ -59,14 +60,22 @@ func NewKademlia(id dht.NodeID, bootstrapNodes []pb.Node, address string, identi
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
bucketIdentifier := id.String()[:5] // need a way to differentiate between nodes if running more than one simultaneously
|
bucketIdentifier := id.String()[:5] // need a way to differentiate between nodes if running more than one simultaneously
|
||||||
rt, err := NewRoutingTable(&self, &RoutingOptions{
|
dbpath := filepath.Join(path, fmt.Sprintf("kademlia_%s.db", bucketIdentifier))
|
||||||
kpath: filepath.Join(path, fmt.Sprintf("kbucket_%s.db", bucketIdentifier)),
|
|
||||||
npath: filepath.Join(path, fmt.Sprintf("nbucket_%s.db", bucketIdentifier)),
|
dbs, err := boltdb.NewShared(dbpath, KademliaBucket, NodeBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, BootstrapErr.Wrap(err)
|
||||||
|
}
|
||||||
|
kdb, ndb := dbs[0], dbs[1]
|
||||||
|
|
||||||
|
rt, err := NewRoutingTable(&self, kdb, ndb, &RoutingOptions{
|
||||||
idLength: kadconfig.DefaultIDLength,
|
idLength: kadconfig.DefaultIDLength,
|
||||||
bucketSize: kadconfig.DefaultBucketSize,
|
bucketSize: kadconfig.DefaultBucketSize,
|
||||||
rcBucketSize: kadconfig.DefaultReplacementCacheSize,
|
rcBucketSize: kadconfig.DefaultReplacementCacheSize,
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, BootstrapErr.Wrap(err)
|
return nil, BootstrapErr.Wrap(err)
|
||||||
}
|
}
|
||||||
@ -101,8 +110,10 @@ func NewKademlia(id dht.NodeID, bootstrapNodes []pb.Node, address string, identi
|
|||||||
|
|
||||||
// Disconnect safely closes connections to the Kademlia network
|
// Disconnect safely closes connections to the Kademlia network
|
||||||
func (k *Kademlia) Disconnect() error {
|
func (k *Kademlia) Disconnect() error {
|
||||||
// TODO(coyle)
|
return utils.CombineErrors(
|
||||||
return errors.New("TODO Disconnect")
|
k.routingTable.Close(),
|
||||||
|
// TODO: close connections
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetNodes returns all nodes from a starting node up to a maximum limit
|
// GetNodes returns all nodes from a starting node up to a maximum limit
|
||||||
|
@ -16,7 +16,6 @@ import (
|
|||||||
"storj.io/storj/pkg/pb"
|
"storj.io/storj/pkg/pb"
|
||||||
"storj.io/storj/pkg/utils"
|
"storj.io/storj/pkg/utils"
|
||||||
"storj.io/storj/storage"
|
"storj.io/storj/storage"
|
||||||
"storj.io/storj/storage/boltdb"
|
|
||||||
"storj.io/storj/storage/storelogger"
|
"storj.io/storj/storage/storelogger"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -45,32 +44,20 @@ type RoutingTable struct {
|
|||||||
|
|
||||||
//RoutingOptions for configuring RoutingTable
|
//RoutingOptions for configuring RoutingTable
|
||||||
type RoutingOptions struct {
|
type RoutingOptions struct {
|
||||||
kpath string
|
|
||||||
npath string
|
|
||||||
idLength int //TODO (JJ): add checks for > 0
|
idLength int //TODO (JJ): add checks for > 0
|
||||||
bucketSize int
|
bucketSize int
|
||||||
rcBucketSize int
|
rcBucketSize int
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewRoutingTable returns a newly configured instance of a RoutingTable
|
// NewRoutingTable returns a newly configured instance of a RoutingTable
|
||||||
func NewRoutingTable(localNode *pb.Node, options *RoutingOptions) (*RoutingTable, error) {
|
func NewRoutingTable(localNode *pb.Node, kdb, ndb storage.KeyValueStore, options *RoutingOptions) (*RoutingTable, error) {
|
||||||
kdb, err := boltdb.New(options.kpath, KademliaBucket)
|
|
||||||
if err != nil {
|
|
||||||
return nil, RoutingErr.New("could not create kadBucketDB: %s", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ndb, err := boltdb.New(options.npath, NodeBucket)
|
|
||||||
if err != nil {
|
|
||||||
return nil, RoutingErr.New("could not create nodeBucketDB: %s", err)
|
|
||||||
}
|
|
||||||
rp := make(map[string][]*pb.Node)
|
|
||||||
rt := &RoutingTable{
|
rt := &RoutingTable{
|
||||||
self: localNode,
|
self: localNode,
|
||||||
kadBucketDB: storelogger.New(zap.L(), kdb),
|
kadBucketDB: storelogger.New(zap.L(), kdb),
|
||||||
nodeBucketDB: storelogger.New(zap.L(), ndb),
|
nodeBucketDB: storelogger.New(zap.L(), ndb),
|
||||||
transport: &defaultTransport,
|
transport: &defaultTransport,
|
||||||
mutex: &sync.Mutex{},
|
mutex: &sync.Mutex{},
|
||||||
replacementCache: rp,
|
replacementCache: make(map[string][]*pb.Node),
|
||||||
idLength: options.idLength,
|
idLength: options.idLength,
|
||||||
bucketSize: options.bucketSize,
|
bucketSize: options.bucketSize,
|
||||||
rcBucketSize: options.rcBucketSize,
|
rcBucketSize: options.rcBucketSize,
|
||||||
@ -84,9 +71,10 @@ func NewRoutingTable(localNode *pb.Node, options *RoutingOptions) (*RoutingTable
|
|||||||
|
|
||||||
// Close closes underlying databases
|
// Close closes underlying databases
|
||||||
func (rt *RoutingTable) Close() error {
|
func (rt *RoutingTable) Close() error {
|
||||||
kerr := rt.kadBucketDB.Close()
|
return utils.CombineErrors(
|
||||||
nerr := rt.nodeBucketDB.Close()
|
rt.kadBucketDB.Close(),
|
||||||
return utils.CombineErrors(kerr, nerr)
|
rt.nodeBucketDB.Close(),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Local returns the local nodes ID
|
// Local returns the local nodes ID
|
||||||
|
@ -4,9 +4,6 @@
|
|||||||
package kademlia
|
package kademlia
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"io/ioutil"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -16,41 +13,31 @@ import (
|
|||||||
|
|
||||||
"storj.io/storj/pkg/pb"
|
"storj.io/storj/pkg/pb"
|
||||||
"storj.io/storj/storage"
|
"storj.io/storj/storage"
|
||||||
|
"storj.io/storj/storage/teststore"
|
||||||
)
|
)
|
||||||
|
|
||||||
func tempdir(t testing.TB) (dir string, cleanup func()) {
|
|
||||||
dir, err := ioutil.TempDir("", "storj-kademlia")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return dir, func() {
|
|
||||||
if err := os.RemoveAll(dir); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func createRoutingTable(t *testing.T, localNodeID []byte) (*RoutingTable, func()) {
|
func createRoutingTable(t *testing.T, localNodeID []byte) (*RoutingTable, func()) {
|
||||||
tempdir, cleanup := tempdir(t)
|
|
||||||
|
|
||||||
if localNodeID == nil {
|
if localNodeID == nil {
|
||||||
localNodeID = []byte("AA")
|
localNodeID = []byte("AA")
|
||||||
}
|
}
|
||||||
localNode := &pb.Node{Id: string(localNodeID)}
|
localNode := &pb.Node{Id: string(localNodeID)}
|
||||||
|
|
||||||
options := &RoutingOptions{
|
options := &RoutingOptions{
|
||||||
kpath: filepath.Join(tempdir, "Kadbucket"),
|
|
||||||
npath: filepath.Join(tempdir, "Nodebucket"),
|
|
||||||
idLength: 16,
|
idLength: 16,
|
||||||
bucketSize: 6,
|
bucketSize: 6,
|
||||||
rcBucketSize: 2,
|
rcBucketSize: 2,
|
||||||
}
|
}
|
||||||
rt, err := NewRoutingTable(localNode, options)
|
rt, err := NewRoutingTable(localNode,
|
||||||
|
teststore.New(),
|
||||||
|
teststore.New(),
|
||||||
|
options,
|
||||||
|
)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
return rt, func() {
|
return rt, func() {
|
||||||
err := rt.Close()
|
err := rt.Close()
|
||||||
cleanup()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
@ -5,6 +5,7 @@ package boltdb
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/boltdb/bolt"
|
"github.com/boltdb/bolt"
|
||||||
@ -18,6 +19,8 @@ type Client struct {
|
|||||||
db *bolt.DB
|
db *bolt.DB
|
||||||
Path string
|
Path string
|
||||||
Bucket []byte
|
Bucket []byte
|
||||||
|
|
||||||
|
referenceCount *int32
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -45,13 +48,57 @@ func New(path, bucket string) (*Client, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
refCount := new(int32)
|
||||||
|
*refCount = 1
|
||||||
|
|
||||||
return &Client{
|
return &Client{
|
||||||
db: db,
|
db: db,
|
||||||
Path: path,
|
referenceCount: refCount,
|
||||||
Bucket: []byte(bucket),
|
Path: path,
|
||||||
|
Bucket: []byte(bucket),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewShared instantiates a new BoltDB with multiple buckets
|
||||||
|
func NewShared(path string, buckets ...string) ([]*Client, error) {
|
||||||
|
db, err := bolt.Open(path, fileMode, &bolt.Options{Timeout: defaultTimeout})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = db.Update(func(tx *bolt.Tx) error {
|
||||||
|
for _, bucket := range buckets {
|
||||||
|
_, err := tx.CreateBucketIfNotExists([]byte(bucket))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
if closeErr := db.Close(); closeErr != nil {
|
||||||
|
return nil, utils.CombineErrors(err, closeErr)
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
refCount := new(int32)
|
||||||
|
*refCount = int32(len(buckets))
|
||||||
|
|
||||||
|
clients := []*Client{}
|
||||||
|
for _, bucket := range buckets {
|
||||||
|
clients = append(clients, &Client{
|
||||||
|
db: db,
|
||||||
|
referenceCount: refCount,
|
||||||
|
Path: path,
|
||||||
|
Bucket: []byte(bucket),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return clients, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (client *Client) update(fn func(*bolt.Bucket) error) error {
|
func (client *Client) update(fn func(*bolt.Bucket) error) error {
|
||||||
return client.db.Update(func(tx *bolt.Tx) error {
|
return client.db.Update(func(tx *bolt.Tx) error {
|
||||||
return fn(tx.Bucket(client.Bucket))
|
return fn(tx.Bucket(client.Bucket))
|
||||||
@ -108,7 +155,10 @@ func (client *Client) ReverseList(first storage.Key, limit int) (storage.Keys, e
|
|||||||
|
|
||||||
// Close closes a BoltDB client
|
// Close closes a BoltDB client
|
||||||
func (client *Client) Close() error {
|
func (client *Client) Close() error {
|
||||||
return client.db.Close()
|
if atomic.AddInt32(client.referenceCount, -1) == 0 {
|
||||||
|
return client.db.Close()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetAll finds all values for the provided keys up to 100 keys
|
// GetAll finds all values for the provided keys up to 100 keys
|
||||||
|
@ -53,3 +53,28 @@ func BenchmarkSuite(b *testing.B) {
|
|||||||
|
|
||||||
testsuite.RunBenchmarks(b, store)
|
testsuite.RunBenchmarks(b, store)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSuiteShared(t *testing.T) {
|
||||||
|
tempdir, err := ioutil.TempDir("", "storj-bolt")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = os.RemoveAll(tempdir) }()
|
||||||
|
|
||||||
|
dbname := filepath.Join(tempdir, "bolt.db")
|
||||||
|
stores, err := NewShared(dbname, "alpha", "beta")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create db: %v", err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
for _, store := range stores {
|
||||||
|
if err := store.Close(); err != nil {
|
||||||
|
t.Fatalf("failed to close db: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
for _, store := range stores {
|
||||||
|
testsuite.RunTests(t, store)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user