Make kademlia use less file-descriptors (#498)

This commit is contained in:
Egon Elbre 2018-10-18 19:20:23 +03:00 committed by GitHub
parent 21026b35f5
commit 03bd93bba7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 110 additions and 49 deletions

View File

@ -5,7 +5,6 @@ package kademlia
import (
"context"
"errors"
"fmt"
"net"
"os"
@ -20,7 +19,9 @@ import (
"storj.io/storj/pkg/node"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/provider"
"storj.io/storj/pkg/utils"
"storj.io/storj/storage"
"storj.io/storj/storage/boltdb"
)
// NodeErr is the class for all errors pertaining to node operations
@ -59,14 +60,22 @@ func NewKademlia(id dht.NodeID, bootstrapNodes []pb.Node, address string, identi
return nil, err
}
}
bucketIdentifier := id.String()[:5] // need a way to differentiate between nodes if running more than one simultaneously
rt, err := NewRoutingTable(&self, &RoutingOptions{
kpath: filepath.Join(path, fmt.Sprintf("kbucket_%s.db", bucketIdentifier)),
npath: filepath.Join(path, fmt.Sprintf("nbucket_%s.db", bucketIdentifier)),
dbpath := filepath.Join(path, fmt.Sprintf("kademlia_%s.db", bucketIdentifier))
dbs, err := boltdb.NewShared(dbpath, KademliaBucket, NodeBucket)
if err != nil {
return nil, BootstrapErr.Wrap(err)
}
kdb, ndb := dbs[0], dbs[1]
rt, err := NewRoutingTable(&self, kdb, ndb, &RoutingOptions{
idLength: kadconfig.DefaultIDLength,
bucketSize: kadconfig.DefaultBucketSize,
rcBucketSize: kadconfig.DefaultReplacementCacheSize,
})
if err != nil {
return nil, BootstrapErr.Wrap(err)
}
@ -101,8 +110,10 @@ func NewKademlia(id dht.NodeID, bootstrapNodes []pb.Node, address string, identi
// Disconnect safely closes connections to the Kademlia network
func (k *Kademlia) Disconnect() error {
// TODO(coyle)
return errors.New("TODO Disconnect")
return utils.CombineErrors(
k.routingTable.Close(),
// TODO: close connections
)
}
// GetNodes returns all nodes from a starting node up to a maximum limit

View File

@ -16,7 +16,6 @@ import (
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/utils"
"storj.io/storj/storage"
"storj.io/storj/storage/boltdb"
"storj.io/storj/storage/storelogger"
)
@ -45,32 +44,20 @@ type RoutingTable struct {
//RoutingOptions for configuring RoutingTable
type RoutingOptions struct {
kpath string
npath string
idLength int //TODO (JJ): add checks for > 0
bucketSize int
rcBucketSize int
}
// NewRoutingTable returns a newly configured instance of a RoutingTable
func NewRoutingTable(localNode *pb.Node, options *RoutingOptions) (*RoutingTable, error) {
kdb, err := boltdb.New(options.kpath, KademliaBucket)
if err != nil {
return nil, RoutingErr.New("could not create kadBucketDB: %s", err)
}
ndb, err := boltdb.New(options.npath, NodeBucket)
if err != nil {
return nil, RoutingErr.New("could not create nodeBucketDB: %s", err)
}
rp := make(map[string][]*pb.Node)
func NewRoutingTable(localNode *pb.Node, kdb, ndb storage.KeyValueStore, options *RoutingOptions) (*RoutingTable, error) {
rt := &RoutingTable{
self: localNode,
kadBucketDB: storelogger.New(zap.L(), kdb),
nodeBucketDB: storelogger.New(zap.L(), ndb),
transport: &defaultTransport,
mutex: &sync.Mutex{},
replacementCache: rp,
replacementCache: make(map[string][]*pb.Node),
idLength: options.idLength,
bucketSize: options.bucketSize,
rcBucketSize: options.rcBucketSize,
@ -84,9 +71,10 @@ func NewRoutingTable(localNode *pb.Node, options *RoutingOptions) (*RoutingTable
// Close closes underlying databases
func (rt *RoutingTable) Close() error {
kerr := rt.kadBucketDB.Close()
nerr := rt.nodeBucketDB.Close()
return utils.CombineErrors(kerr, nerr)
return utils.CombineErrors(
rt.kadBucketDB.Close(),
rt.nodeBucketDB.Close(),
)
}
// Local returns the local nodes ID

View File

@ -4,9 +4,6 @@
package kademlia
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"time"
@ -16,41 +13,31 @@ import (
"storj.io/storj/pkg/pb"
"storj.io/storj/storage"
"storj.io/storj/storage/teststore"
)
func tempdir(t testing.TB) (dir string, cleanup func()) {
dir, err := ioutil.TempDir("", "storj-kademlia")
if err != nil {
t.Fatal(err)
}
return dir, func() {
if err := os.RemoveAll(dir); err != nil {
t.Fatal(err)
}
}
}
func createRoutingTable(t *testing.T, localNodeID []byte) (*RoutingTable, func()) {
tempdir, cleanup := tempdir(t)
if localNodeID == nil {
localNodeID = []byte("AA")
}
localNode := &pb.Node{Id: string(localNodeID)}
options := &RoutingOptions{
kpath: filepath.Join(tempdir, "Kadbucket"),
npath: filepath.Join(tempdir, "Nodebucket"),
idLength: 16,
bucketSize: 6,
rcBucketSize: 2,
}
rt, err := NewRoutingTable(localNode, options)
rt, err := NewRoutingTable(localNode,
teststore.New(),
teststore.New(),
options,
)
if err != nil {
t.Fatal(err)
}
return rt, func() {
err := rt.Close()
cleanup()
if err != nil {
t.Fatal(err)
}

View File

@ -5,6 +5,7 @@ package boltdb
import (
"bytes"
"sync/atomic"
"time"
"github.com/boltdb/bolt"
@ -18,6 +19,8 @@ type Client struct {
db *bolt.DB
Path string
Bucket []byte
referenceCount *int32
}
const (
@ -45,13 +48,57 @@ func New(path, bucket string) (*Client, error) {
return nil, err
}
refCount := new(int32)
*refCount = 1
return &Client{
db: db,
Path: path,
Bucket: []byte(bucket),
db: db,
referenceCount: refCount,
Path: path,
Bucket: []byte(bucket),
}, nil
}
// NewShared instantiates a new BoltDB with multiple buckets
func NewShared(path string, buckets ...string) ([]*Client, error) {
db, err := bolt.Open(path, fileMode, &bolt.Options{Timeout: defaultTimeout})
if err != nil {
return nil, err
}
err = db.Update(func(tx *bolt.Tx) error {
for _, bucket := range buckets {
_, err := tx.CreateBucketIfNotExists([]byte(bucket))
if err != nil {
return err
}
}
return err
})
if err != nil {
if closeErr := db.Close(); closeErr != nil {
return nil, utils.CombineErrors(err, closeErr)
}
return nil, err
}
refCount := new(int32)
*refCount = int32(len(buckets))
clients := []*Client{}
for _, bucket := range buckets {
clients = append(clients, &Client{
db: db,
referenceCount: refCount,
Path: path,
Bucket: []byte(bucket),
})
}
return clients, nil
}
func (client *Client) update(fn func(*bolt.Bucket) error) error {
return client.db.Update(func(tx *bolt.Tx) error {
return fn(tx.Bucket(client.Bucket))
@ -108,7 +155,10 @@ func (client *Client) ReverseList(first storage.Key, limit int) (storage.Keys, e
// Close closes a BoltDB client
func (client *Client) Close() error {
return client.db.Close()
if atomic.AddInt32(client.referenceCount, -1) == 0 {
return client.db.Close()
}
return nil
}
// GetAll finds all values for the provided keys up to 100 keys

View File

@ -53,3 +53,28 @@ func BenchmarkSuite(b *testing.B) {
testsuite.RunBenchmarks(b, store)
}
func TestSuiteShared(t *testing.T) {
tempdir, err := ioutil.TempDir("", "storj-bolt")
if err != nil {
t.Fatal(err)
}
defer func() { _ = os.RemoveAll(tempdir) }()
dbname := filepath.Join(tempdir, "bolt.db")
stores, err := NewShared(dbname, "alpha", "beta")
if err != nil {
t.Fatalf("failed to create db: %v", err)
}
defer func() {
for _, store := range stores {
if err := store.Close(); err != nil {
t.Fatalf("failed to close db: %v", err)
}
}
}()
for _, store := range stores {
testsuite.RunTests(t, store)
}
}