Make kademlia use less file-descriptors (#498)

pkg/kademlia/kademlia.go

@@ -5,7 +5,6 @@ package kademlia
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"net"
 	"os"
@@ -20,7 +19,9 @@ import (
 	"storj.io/storj/pkg/node"
 	"storj.io/storj/pkg/pb"
 	"storj.io/storj/pkg/provider"
+	"storj.io/storj/pkg/utils"
 	"storj.io/storj/storage"
+	"storj.io/storj/storage/boltdb"
 )
 
 // NodeErr is the class for all errors pertaining to node operations
@@ -59,14 +60,22 @@ func NewKademlia(id dht.NodeID, bootstrapNodes []pb.Node, address string, identi
 			return nil, err
 		}
 	}
+
 	bucketIdentifier := id.String()[:5] // need a way to differentiate between nodes if running more than one simultaneously
-	rt, err := NewRoutingTable(&self, &RoutingOptions{
+	dbpath := filepath.Join(path, fmt.Sprintf("kademlia_%s.db", bucketIdentifier))
-		kpath:        filepath.Join(path, fmt.Sprintf("kbucket_%s.db", bucketIdentifier)),
+
-		npath:        filepath.Join(path, fmt.Sprintf("nbucket_%s.db", bucketIdentifier)),
+	dbs, err := boltdb.NewShared(dbpath, KademliaBucket, NodeBucket)
+	if err != nil {
+		return nil, BootstrapErr.Wrap(err)
+	}
+	kdb, ndb := dbs[0], dbs[1]
+
+	rt, err := NewRoutingTable(&self, kdb, ndb, &RoutingOptions{
 		idLength:     kadconfig.DefaultIDLength,
 		bucketSize:   kadconfig.DefaultBucketSize,
 		rcBucketSize: kadconfig.DefaultReplacementCacheSize,
 	})
+
 	if err != nil {
 		return nil, BootstrapErr.Wrap(err)
 	}
@@ -101,8 +110,10 @@ func NewKademlia(id dht.NodeID, bootstrapNodes []pb.Node, address string, identi
 
 // Disconnect safely closes connections to the Kademlia network
 func (k *Kademlia) Disconnect() error {
-	// TODO(coyle)
+	return utils.CombineErrors(
-	return errors.New("TODO Disconnect")
+		k.routingTable.Close(),
+		// TODO: close connections
+	)
 }
 
 // GetNodes returns all nodes from a starting node up to a maximum limit

pkg/kademlia/routing.go

@@ -16,7 +16,6 @@ import (
 	"storj.io/storj/pkg/pb"
 	"storj.io/storj/pkg/utils"
 	"storj.io/storj/storage"
-	"storj.io/storj/storage/boltdb"
 	"storj.io/storj/storage/storelogger"
 )
 
@@ -45,32 +44,20 @@ type RoutingTable struct {
 
 //RoutingOptions for configuring RoutingTable
 type RoutingOptions struct {
-	kpath        string
-	npath        string
 	idLength     int //TODO (JJ): add checks for > 0
 	bucketSize   int
 	rcBucketSize int
 }
 
 // NewRoutingTable returns a newly configured instance of a RoutingTable
-func NewRoutingTable(localNode *pb.Node, options *RoutingOptions) (*RoutingTable, error) {
+func NewRoutingTable(localNode *pb.Node, kdb, ndb storage.KeyValueStore, options *RoutingOptions) (*RoutingTable, error) {
-	kdb, err := boltdb.New(options.kpath, KademliaBucket)
-	if err != nil {
-		return nil, RoutingErr.New("could not create kadBucketDB: %s", err)
-	}
-
-	ndb, err := boltdb.New(options.npath, NodeBucket)
-	if err != nil {
-		return nil, RoutingErr.New("could not create nodeBucketDB: %s", err)
-	}
-	rp := make(map[string][]*pb.Node)
 	rt := &RoutingTable{
 		self:             localNode,
 		kadBucketDB:      storelogger.New(zap.L(), kdb),
 		nodeBucketDB:     storelogger.New(zap.L(), ndb),
 		transport:        &defaultTransport,
 		mutex:            &sync.Mutex{},
-		replacementCache: rp,
+		replacementCache: make(map[string][]*pb.Node),
 		idLength:         options.idLength,
 		bucketSize:       options.bucketSize,
 		rcBucketSize:     options.rcBucketSize,
@@ -84,9 +71,10 @@ func NewRoutingTable(localNode *pb.Node, options *RoutingOptions) (*RoutingTable
 
 // Close closes underlying databases
 func (rt *RoutingTable) Close() error {
-	kerr := rt.kadBucketDB.Close()
+	return utils.CombineErrors(
-	nerr := rt.nodeBucketDB.Close()
+		rt.kadBucketDB.Close(),
-	return utils.CombineErrors(kerr, nerr)
+		rt.nodeBucketDB.Close(),
+	)
 }
 
 // Local returns the local nodes ID

pkg/kademlia/routing_helpers_test.go

@@ -4,9 +4,6 @@
 package kademlia
 
 import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -16,41 +13,31 @@ import (
 
 	"storj.io/storj/pkg/pb"
 	"storj.io/storj/storage"
+	"storj.io/storj/storage/teststore"
 )
 
-func tempdir(t testing.TB) (dir string, cleanup func()) {
-	dir, err := ioutil.TempDir("", "storj-kademlia")
-	if err != nil {
-		t.Fatal(err)
-	}
-	return dir, func() {
-		if err := os.RemoveAll(dir); err != nil {
-			t.Fatal(err)
-		}
-	}
-}
-
 func createRoutingTable(t *testing.T, localNodeID []byte) (*RoutingTable, func()) {
-	tempdir, cleanup := tempdir(t)
-
 	if localNodeID == nil {
 		localNodeID = []byte("AA")
 	}
 	localNode := &pb.Node{Id: string(localNodeID)}
+
 	options := &RoutingOptions{
-		kpath:        filepath.Join(tempdir, "Kadbucket"),
-		npath:        filepath.Join(tempdir, "Nodebucket"),
 		idLength:     16,
 		bucketSize:   6,
 		rcBucketSize: 2,
 	}
-	rt, err := NewRoutingTable(localNode, options)
+	rt, err := NewRoutingTable(localNode,
+		teststore.New(),
+		teststore.New(),
+		options,
+	)
+
 	if err != nil {
 		t.Fatal(err)
 	}
 	return rt, func() {
 		err := rt.Close()
-		cleanup()
 		if err != nil {
 			t.Fatal(err)
 		}

storage/boltdb/client.go

@@ -5,6 +5,7 @@ package boltdb
 
 import (
 	"bytes"
+	"sync/atomic"
 	"time"
 
 	"github.com/boltdb/bolt"
@@ -18,6 +19,8 @@ type Client struct {
 	db     *bolt.DB
 	Path   string
 	Bucket []byte
+
+	referenceCount *int32
 }
 
 const (
@@ -45,13 +48,57 @@ func New(path, bucket string) (*Client, error) {
 		return nil, err
 	}
 
+	refCount := new(int32)
+	*refCount = 1
+
 	return &Client{
 		db:             db,
+		referenceCount: refCount,
 		Path:           path,
 		Bucket:         []byte(bucket),
 	}, nil
 }
 
+// NewShared instantiates a new BoltDB with multiple buckets
+func NewShared(path string, buckets ...string) ([]*Client, error) {
+	db, err := bolt.Open(path, fileMode, &bolt.Options{Timeout: defaultTimeout})
+	if err != nil {
+		return nil, err
+	}
+
+	err = db.Update(func(tx *bolt.Tx) error {
+		for _, bucket := range buckets {
+			_, err := tx.CreateBucketIfNotExists([]byte(bucket))
+			if err != nil {
+				return err
+			}
+		}
+		return err
+	})
+
+	if err != nil {
+		if closeErr := db.Close(); closeErr != nil {
+			return nil, utils.CombineErrors(err, closeErr)
+		}
+		return nil, err
+	}
+
+	refCount := new(int32)
+	*refCount = int32(len(buckets))
+
+	clients := []*Client{}
+	for _, bucket := range buckets {
+		clients = append(clients, &Client{
+			db:             db,
+			referenceCount: refCount,
+			Path:           path,
+			Bucket:         []byte(bucket),
+		})
+	}
+
+	return clients, nil
+}
+
 func (client *Client) update(fn func(*bolt.Bucket) error) error {
 	return client.db.Update(func(tx *bolt.Tx) error {
 		return fn(tx.Bucket(client.Bucket))
@@ -108,7 +155,10 @@ func (client *Client) ReverseList(first storage.Key, limit int) (storage.Keys, e
 
 // Close closes a BoltDB client
 func (client *Client) Close() error {
+	if atomic.AddInt32(client.referenceCount, -1) == 0 {
 		return client.db.Close()
+	}
+	return nil
 }
 
 // GetAll finds all values for the provided keys up to 100 keys

storage/boltdb/client_test.go

@@ -53,3 +53,28 @@ func BenchmarkSuite(b *testing.B) {
 
 	testsuite.RunBenchmarks(b, store)
 }
+
+func TestSuiteShared(t *testing.T) {
+	tempdir, err := ioutil.TempDir("", "storj-bolt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.RemoveAll(tempdir) }()
+
+	dbname := filepath.Join(tempdir, "bolt.db")
+	stores, err := NewShared(dbname, "alpha", "beta")
+	if err != nil {
+		t.Fatalf("failed to create db: %v", err)
+	}
+	defer func() {
+		for _, store := range stores {
+			if err := store.Close(); err != nil {
+				t.Fatalf("failed to close db: %v", err)
+			}
+		}
+	}()
+
+	for _, store := range stores {
+		testsuite.RunTests(t, store)
+	}
+}