storj/pkg/overlay/cache.go

163 lines
3.8 KiB
Go
Raw Normal View History

2018-04-18 17:55:28 +01:00
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package overlay
2018-04-18 16:34:15 +01:00
import (
"context"
2018-04-18 16:34:15 +01:00
"github.com/gogo/protobuf/proto"
"github.com/zeebo/errs"
"go.uber.org/zap"
2018-04-18 16:34:15 +01:00
"storj.io/storj/pkg/dht"
"storj.io/storj/pkg/kademlia"
2018-04-18 16:34:15 +01:00
"storj.io/storj/protos/overlay"
"storj.io/storj/storage"
"storj.io/storj/storage/boltdb"
"storj.io/storj/storage/redis"
2018-04-18 16:34:15 +01:00
)
// ErrNodeNotFound error standardization
var ErrNodeNotFound = errs.New("Node not found")
// OverlayError creates class of errors for stack traces
var OverlayError = errs.Class("Overlay Error")
Cache (#67) * add reference to dht to overlay client struct * wip * wip * Implement FindNode * get nodes * WIP * Merge in Dennis kademlia code, get it working with our code * ping and moar * WIP trying to get cache working with kademlia * WIP more wiring up * WIP * Update service cli commands * WIP * added GetNodes * added nodes to Kbucket * default transport changed to TCP * GetBuckets interface changed * filling in more routing * timestamp methods * removed store * Added initial network overlay explorer page * Updating and building with dockerfile * Working on adding bootstrap node code * WIP merging in dennis' code * WIP * connects cache to pkg/kademlia implementation * WIP redis cache * testing * Add bootstrap network function for CLI usage * cleanup * call bootstrap on init network * Add BootstrapNetwork function to interface * Merge in dennis kad code * WIP updates to redis/overlay client interface * WIP trying to get the DHT connected to the cache * go mod & test * deps * Bootstrap node now setting up correctly - Need to pass it through CLI commands better * WIP adding refresh and walk functions, added cli flags - added cli flags for custom bootstrap port and ip * PR comments addressed * adding FindStorageNodes to overlay cache * fix GetBucket * using SplitHostPort * Use JoinHostPort * updates to findstoragenodes response and request * WIP merge in progress, having issues with a panic * wip * adjustments * update port for dht bootstrap test * Docker * wip * dockerfile * fixes * makefile changes * Update port in NewKademlia call * Update local kademlia DHT config * kubernetes yaml * cleanup * making tests pass * k8s yaml * lint issues * Edit cli flags to allow for configurable bootstrap IP and Port args * cleanup * cache walking the network now * Rough prototype of Walk function laid out * Move walk function into bootstrap function * Update dht.go * changes to yaml * goimports
2018-06-05 22:06:37 +01:00
// Cache is used to store overlay data in Redis
type Cache struct {
DB storage.KeyValueStore
DHT dht.DHT
2018-04-18 16:34:15 +01:00
}
// NewRedisOverlayCache returns a pointer to a new Cache instance with an initalized connection to Redis.
func NewRedisOverlayCache(address, password string, db int, DHT dht.DHT) (*Cache, error) {
rc, err := redis.NewClient(address, password, db)
2018-04-18 16:34:15 +01:00
if err != nil {
return nil, err
}
return &Cache{
Cache (#67) * add reference to dht to overlay client struct * wip * wip * Implement FindNode * get nodes * WIP * Merge in Dennis kademlia code, get it working with our code * ping and moar * WIP trying to get cache working with kademlia * WIP more wiring up * WIP * Update service cli commands * WIP * added GetNodes * added nodes to Kbucket * default transport changed to TCP * GetBuckets interface changed * filling in more routing * timestamp methods * removed store * Added initial network overlay explorer page * Updating and building with dockerfile * Working on adding bootstrap node code * WIP merging in dennis' code * WIP * connects cache to pkg/kademlia implementation * WIP redis cache * testing * Add bootstrap network function for CLI usage * cleanup * call bootstrap on init network * Add BootstrapNetwork function to interface * Merge in dennis kad code * WIP updates to redis/overlay client interface * WIP trying to get the DHT connected to the cache * go mod & test * deps * Bootstrap node now setting up correctly - Need to pass it through CLI commands better * WIP adding refresh and walk functions, added cli flags - added cli flags for custom bootstrap port and ip * PR comments addressed * adding FindStorageNodes to overlay cache * fix GetBucket * using SplitHostPort * Use JoinHostPort * updates to findstoragenodes response and request * WIP merge in progress, having issues with a panic * wip * adjustments * update port for dht bootstrap test * Docker * wip * dockerfile * fixes * makefile changes * Update port in NewKademlia call * Update local kademlia DHT config * kubernetes yaml * cleanup * making tests pass * k8s yaml * lint issues * Edit cli flags to allow for configurable bootstrap IP and Port args * cleanup * cache walking the network now * Rough prototype of Walk function laid out * Move walk function into bootstrap function * Update dht.go * changes to yaml * goimports
2018-06-05 22:06:37 +01:00
DB: rc,
DHT: DHT,
2018-04-18 17:55:28 +01:00
}, nil
2018-04-18 16:34:15 +01:00
}
// NewBoltOverlayCache returns a pointer to a new Cache instance with an initalized connection to a Bolt db.
func NewBoltOverlayCache(dbPath string, DHT dht.DHT) (*Cache, error) {
bc, err := boltdb.NewClient(nil, dbPath, boltdb.OverlayBucket)
if err != nil {
return nil, err
}
return &Cache{
DB: bc,
DHT: DHT,
}, nil
}
2018-04-18 16:34:15 +01:00
// Get looks up the provided nodeID from the redis cache
func (o *Cache) Get(ctx context.Context, key string) (*overlay.NodeAddress, error) {
b, err := o.DB.Get([]byte(key))
2018-04-18 16:34:15 +01:00
if err != nil {
return nil, err
}
adds netstate pagination (#95) * adds netstate rpc server pagination, mocks pagination in test/util.go * updates ns client example, combines ns client and server test to netstate_test, adds pagination to bolt client * better organizes netstate test calls * wip breaking netstate test into smaller tests * wip modularizing netstate tests * adds some test panics * wip netstate test attempts * testing bug in netstate TestDeleteAuth * wip fixes global variable problem, still issues with list * wip fixes get request params and args * fixes bug in path when using MakePointers helper fn * updates mockdb list func, adds test, changes Limit to int * fixes merge conflicts * fixes broken tests from merge * remove unnecessary PointerEntry struct * removes error when Get returns nil value from boltdb * breaks boltdb client tests into smaller tests * renames AssertNoErr test helper to HandleErr * adds StartingKey and Limit parameters to redis list func, adds beginning of redis tests * adds helper func for mockdb List function * if no starting key provided for netstate List, the first value in storage will be used * adds basic pagination for redis List function, adds tests * adds list limit to call in overlay/server.go * streamlines/fixes some nits from review * removes use of obsolete EncryptedUnencryptedSize * uses MockKeyValueStore instead of redis instance in redis client test * changes test to expect nil returned for getting missing key * remove error from `KeyValueStore#Get` * fix bolt test * Merge pull request #1 from bryanchriswhite/nat-pagination remove error from `KeyValueStore#Get` * adds Get returning error back to KeyValueStore interface and affected clients * trying to appease travis: returns errors in Get calls in overlay/cache and cache_test * handles redis get error when no key found
2018-06-29 21:06:25 +01:00
if b.IsZero() {
// TODO: log? return an error?
return nil, nil
}
2018-04-18 16:34:15 +01:00
na := &overlay.NodeAddress{}
Cache (#67) * add reference to dht to overlay client struct * wip * wip * Implement FindNode * get nodes * WIP * Merge in Dennis kademlia code, get it working with our code * ping and moar * WIP trying to get cache working with kademlia * WIP more wiring up * WIP * Update service cli commands * WIP * added GetNodes * added nodes to Kbucket * default transport changed to TCP * GetBuckets interface changed * filling in more routing * timestamp methods * removed store * Added initial network overlay explorer page * Updating and building with dockerfile * Working on adding bootstrap node code * WIP merging in dennis' code * WIP * connects cache to pkg/kademlia implementation * WIP redis cache * testing * Add bootstrap network function for CLI usage * cleanup * call bootstrap on init network * Add BootstrapNetwork function to interface * Merge in dennis kad code * WIP updates to redis/overlay client interface * WIP trying to get the DHT connected to the cache * go mod & test * deps * Bootstrap node now setting up correctly - Need to pass it through CLI commands better * WIP adding refresh and walk functions, added cli flags - added cli flags for custom bootstrap port and ip * PR comments addressed * adding FindStorageNodes to overlay cache * fix GetBucket * using SplitHostPort * Use JoinHostPort * updates to findstoragenodes response and request * WIP merge in progress, having issues with a panic * wip * adjustments * update port for dht bootstrap test * Docker * wip * dockerfile * fixes * makefile changes * Update port in NewKademlia call * Update local kademlia DHT config * kubernetes yaml * cleanup * making tests pass * k8s yaml * lint issues * Edit cli flags to allow for configurable bootstrap IP and Port args * cleanup * cache walking the network now * Rough prototype of Walk function laid out * Move walk function into bootstrap function * Update dht.go * changes to yaml * goimports
2018-06-05 22:06:37 +01:00
if err := proto.Unmarshal(b, na); err != nil {
return nil, err
}
2018-04-18 16:34:15 +01:00
Cache (#67) * add reference to dht to overlay client struct * wip * wip * Implement FindNode * get nodes * WIP * Merge in Dennis kademlia code, get it working with our code * ping and moar * WIP trying to get cache working with kademlia * WIP more wiring up * WIP * Update service cli commands * WIP * added GetNodes * added nodes to Kbucket * default transport changed to TCP * GetBuckets interface changed * filling in more routing * timestamp methods * removed store * Added initial network overlay explorer page * Updating and building with dockerfile * Working on adding bootstrap node code * WIP merging in dennis' code * WIP * connects cache to pkg/kademlia implementation * WIP redis cache * testing * Add bootstrap network function for CLI usage * cleanup * call bootstrap on init network * Add BootstrapNetwork function to interface * Merge in dennis kad code * WIP updates to redis/overlay client interface * WIP trying to get the DHT connected to the cache * go mod & test * deps * Bootstrap node now setting up correctly - Need to pass it through CLI commands better * WIP adding refresh and walk functions, added cli flags - added cli flags for custom bootstrap port and ip * PR comments addressed * adding FindStorageNodes to overlay cache * fix GetBucket * using SplitHostPort * Use JoinHostPort * updates to findstoragenodes response and request * WIP merge in progress, having issues with a panic * wip * adjustments * update port for dht bootstrap test * Docker * wip * dockerfile * fixes * makefile changes * Update port in NewKademlia call * Update local kademlia DHT config * kubernetes yaml * cleanup * making tests pass * k8s yaml * lint issues * Edit cli flags to allow for configurable bootstrap IP and Port args * cleanup * cache walking the network now * Rough prototype of Walk function laid out * Move walk function into bootstrap function * Update dht.go * changes to yaml * goimports
2018-06-05 22:06:37 +01:00
return na, nil
2018-04-18 16:34:15 +01:00
}
// Put adds a nodeID to the redis cache with a binary representation of proto defined NodeAddress
func (o *Cache) Put(nodeID string, value overlay.NodeAddress) error {
2018-04-18 16:34:15 +01:00
data, err := proto.Marshal(&value)
if err != nil {
return err
}
return o.DB.Put([]byte(nodeID), []byte(data))
2018-04-18 16:34:15 +01:00
}
// Bootstrap walks the initialized network and populates the cache
func (o *Cache) Bootstrap(ctx context.Context) error {
Cache (#67) * add reference to dht to overlay client struct * wip * wip * Implement FindNode * get nodes * WIP * Merge in Dennis kademlia code, get it working with our code * ping and moar * WIP trying to get cache working with kademlia * WIP more wiring up * WIP * Update service cli commands * WIP * added GetNodes * added nodes to Kbucket * default transport changed to TCP * GetBuckets interface changed * filling in more routing * timestamp methods * removed store * Added initial network overlay explorer page * Updating and building with dockerfile * Working on adding bootstrap node code * WIP merging in dennis' code * WIP * connects cache to pkg/kademlia implementation * WIP redis cache * testing * Add bootstrap network function for CLI usage * cleanup * call bootstrap on init network * Add BootstrapNetwork function to interface * Merge in dennis kad code * WIP updates to redis/overlay client interface * WIP trying to get the DHT connected to the cache * go mod & test * deps * Bootstrap node now setting up correctly - Need to pass it through CLI commands better * WIP adding refresh and walk functions, added cli flags - added cli flags for custom bootstrap port and ip * PR comments addressed * adding FindStorageNodes to overlay cache * fix GetBucket * using SplitHostPort * Use JoinHostPort * updates to findstoragenodes response and request * WIP merge in progress, having issues with a panic * wip * adjustments * update port for dht bootstrap test * Docker * wip * dockerfile * fixes * makefile changes * Update port in NewKademlia call * Update local kademlia DHT config * kubernetes yaml * cleanup * making tests pass * k8s yaml * lint issues * Edit cli flags to allow for configurable bootstrap IP and Port args * cleanup * cache walking the network now * Rough prototype of Walk function laid out * Move walk function into bootstrap function * Update dht.go * changes to yaml * goimports
2018-06-05 22:06:37 +01:00
nodes, err := o.DHT.GetNodes(ctx, "0", 1280)
if err != nil {
zap.Error(OverlayError.New("Error getting nodes from DHT", err))
}
Cache (#67) * add reference to dht to overlay client struct * wip * wip * Implement FindNode * get nodes * WIP * Merge in Dennis kademlia code, get it working with our code * ping and moar * WIP trying to get cache working with kademlia * WIP more wiring up * WIP * Update service cli commands * WIP * added GetNodes * added nodes to Kbucket * default transport changed to TCP * GetBuckets interface changed * filling in more routing * timestamp methods * removed store * Added initial network overlay explorer page * Updating and building with dockerfile * Working on adding bootstrap node code * WIP merging in dennis' code * WIP * connects cache to pkg/kademlia implementation * WIP redis cache * testing * Add bootstrap network function for CLI usage * cleanup * call bootstrap on init network * Add BootstrapNetwork function to interface * Merge in dennis kad code * WIP updates to redis/overlay client interface * WIP trying to get the DHT connected to the cache * go mod & test * deps * Bootstrap node now setting up correctly - Need to pass it through CLI commands better * WIP adding refresh and walk functions, added cli flags - added cli flags for custom bootstrap port and ip * PR comments addressed * adding FindStorageNodes to overlay cache * fix GetBucket * using SplitHostPort * Use JoinHostPort * updates to findstoragenodes response and request * WIP merge in progress, having issues with a panic * wip * adjustments * update port for dht bootstrap test * Docker * wip * dockerfile * fixes * makefile changes * Update port in NewKademlia call * Update local kademlia DHT config * kubernetes yaml * cleanup * making tests pass * k8s yaml * lint issues * Edit cli flags to allow for configurable bootstrap IP and Port args * cleanup * cache walking the network now * Rough prototype of Walk function laid out * Move walk function into bootstrap function * Update dht.go * changes to yaml * goimports
2018-06-05 22:06:37 +01:00
for _, v := range nodes {
found, err := o.DHT.FindNode(ctx, kademlia.StringToNodeID(v.Id))
Cache (#67) * add reference to dht to overlay client struct * wip * wip * Implement FindNode * get nodes * WIP * Merge in Dennis kademlia code, get it working with our code * ping and moar * WIP trying to get cache working with kademlia * WIP more wiring up * WIP * Update service cli commands * WIP * added GetNodes * added nodes to Kbucket * default transport changed to TCP * GetBuckets interface changed * filling in more routing * timestamp methods * removed store * Added initial network overlay explorer page * Updating and building with dockerfile * Working on adding bootstrap node code * WIP merging in dennis' code * WIP * connects cache to pkg/kademlia implementation * WIP redis cache * testing * Add bootstrap network function for CLI usage * cleanup * call bootstrap on init network * Add BootstrapNetwork function to interface * Merge in dennis kad code * WIP updates to redis/overlay client interface * WIP trying to get the DHT connected to the cache * go mod & test * deps * Bootstrap node now setting up correctly - Need to pass it through CLI commands better * WIP adding refresh and walk functions, added cli flags - added cli flags for custom bootstrap port and ip * PR comments addressed * adding FindStorageNodes to overlay cache * fix GetBucket * using SplitHostPort * Use JoinHostPort * updates to findstoragenodes response and request * WIP merge in progress, having issues with a panic * wip * adjustments * update port for dht bootstrap test * Docker * wip * dockerfile * fixes * makefile changes * Update port in NewKademlia call * Update local kademlia DHT config * kubernetes yaml * cleanup * making tests pass * k8s yaml * lint issues * Edit cli flags to allow for configurable bootstrap IP and Port args * cleanup * cache walking the network now * Rough prototype of Walk function laid out * Move walk function into bootstrap function * Update dht.go * changes to yaml * goimports
2018-06-05 22:06:37 +01:00
if err != nil {
zap.Error(ErrNodeNotFound)
Cache (#67) * add reference to dht to overlay client struct * wip * wip * Implement FindNode * get nodes * WIP * Merge in Dennis kademlia code, get it working with our code * ping and moar * WIP trying to get cache working with kademlia * WIP more wiring up * WIP * Update service cli commands * WIP * added GetNodes * added nodes to Kbucket * default transport changed to TCP * GetBuckets interface changed * filling in more routing * timestamp methods * removed store * Added initial network overlay explorer page * Updating and building with dockerfile * Working on adding bootstrap node code * WIP merging in dennis' code * WIP * connects cache to pkg/kademlia implementation * WIP redis cache * testing * Add bootstrap network function for CLI usage * cleanup * call bootstrap on init network * Add BootstrapNetwork function to interface * Merge in dennis kad code * WIP updates to redis/overlay client interface * WIP trying to get the DHT connected to the cache * go mod & test * deps * Bootstrap node now setting up correctly - Need to pass it through CLI commands better * WIP adding refresh and walk functions, added cli flags - added cli flags for custom bootstrap port and ip * PR comments addressed * adding FindStorageNodes to overlay cache * fix GetBucket * using SplitHostPort * Use JoinHostPort * updates to findstoragenodes response and request * WIP merge in progress, having issues with a panic * wip * adjustments * update port for dht bootstrap test * Docker * wip * dockerfile * fixes * makefile changes * Update port in NewKademlia call * Update local kademlia DHT config * kubernetes yaml * cleanup * making tests pass * k8s yaml * lint issues * Edit cli flags to allow for configurable bootstrap IP and Port args * cleanup * cache walking the network now * Rough prototype of Walk function laid out * Move walk function into bootstrap function * Update dht.go * changes to yaml * goimports
2018-06-05 22:06:37 +01:00
}
addr, err := proto.Marshal(found.Address)
if err != nil {
return err
}
if err := o.DB.Put([]byte(found.Id), addr); err != nil {
return err
}
Cache (#67) * add reference to dht to overlay client struct * wip * wip * Implement FindNode * get nodes * WIP * Merge in Dennis kademlia code, get it working with our code * ping and moar * WIP trying to get cache working with kademlia * WIP more wiring up * WIP * Update service cli commands * WIP * added GetNodes * added nodes to Kbucket * default transport changed to TCP * GetBuckets interface changed * filling in more routing * timestamp methods * removed store * Added initial network overlay explorer page * Updating and building with dockerfile * Working on adding bootstrap node code * WIP merging in dennis' code * WIP * connects cache to pkg/kademlia implementation * WIP redis cache * testing * Add bootstrap network function for CLI usage * cleanup * call bootstrap on init network * Add BootstrapNetwork function to interface * Merge in dennis kad code * WIP updates to redis/overlay client interface * WIP trying to get the DHT connected to the cache * go mod & test * deps * Bootstrap node now setting up correctly - Need to pass it through CLI commands better * WIP adding refresh and walk functions, added cli flags - added cli flags for custom bootstrap port and ip * PR comments addressed * adding FindStorageNodes to overlay cache * fix GetBucket * using SplitHostPort * Use JoinHostPort * updates to findstoragenodes response and request * WIP merge in progress, having issues with a panic * wip * adjustments * update port for dht bootstrap test * Docker * wip * dockerfile * fixes * makefile changes * Update port in NewKademlia call * Update local kademlia DHT config * kubernetes yaml * cleanup * making tests pass * k8s yaml * lint issues * Edit cli flags to allow for configurable bootstrap IP and Port args * cleanup * cache walking the network now * Rough prototype of Walk function laid out * Move walk function into bootstrap function * Update dht.go * changes to yaml * goimports
2018-06-05 22:06:37 +01:00
}
// called after kademlia is bootstrapped
// needs to take RoutingTable and start to persist it into the cache
// take bootstrap node
// get their route table
// loop through nodes in RT and get THEIR route table
// keep going forever and ever
// Other Possibilities: Randomly generate node ID's to ask for?
_, err = o.DHT.GetRoutingTable(ctx)
return err
}
// Refresh walks the network looking for new nodes and pings existing nodes to eliminate stale addresses
func (o *Cache) Refresh(ctx context.Context) error {
Cache (#67) * add reference to dht to overlay client struct * wip * wip * Implement FindNode * get nodes * WIP * Merge in Dennis kademlia code, get it working with our code * ping and moar * WIP trying to get cache working with kademlia * WIP more wiring up * WIP * Update service cli commands * WIP * added GetNodes * added nodes to Kbucket * default transport changed to TCP * GetBuckets interface changed * filling in more routing * timestamp methods * removed store * Added initial network overlay explorer page * Updating and building with dockerfile * Working on adding bootstrap node code * WIP merging in dennis' code * WIP * connects cache to pkg/kademlia implementation * WIP redis cache * testing * Add bootstrap network function for CLI usage * cleanup * call bootstrap on init network * Add BootstrapNetwork function to interface * Merge in dennis kad code * WIP updates to redis/overlay client interface * WIP trying to get the DHT connected to the cache * go mod & test * deps * Bootstrap node now setting up correctly - Need to pass it through CLI commands better * WIP adding refresh and walk functions, added cli flags - added cli flags for custom bootstrap port and ip * PR comments addressed * adding FindStorageNodes to overlay cache * fix GetBucket * using SplitHostPort * Use JoinHostPort * updates to findstoragenodes response and request * WIP merge in progress, having issues with a panic * wip * adjustments * update port for dht bootstrap test * Docker * wip * dockerfile * fixes * makefile changes * Update port in NewKademlia call * Update local kademlia DHT config * kubernetes yaml * cleanup * making tests pass * k8s yaml * lint issues * Edit cli flags to allow for configurable bootstrap IP and Port args * cleanup * cache walking the network now * Rough prototype of Walk function laid out * Move walk function into bootstrap function * Update dht.go * changes to yaml * goimports
2018-06-05 22:06:37 +01:00
// iterate over all nodes
// compare responses to find new nodes
// listen for responses from existing nodes
// if no response from existing, then mark it as offline for time period
// if responds, it refreshes in DHT
_, rtErr := o.DHT.GetRoutingTable(ctx)
if rtErr != nil {
return rtErr
}
_, err := o.DHT.GetNodes(ctx, "0", 128)
if err != nil {
return err
}
return nil
}
// Walk iterates over buckets to traverse the network
func (o *Cache) Walk(ctx context.Context) error {
Cache (#67) * add reference to dht to overlay client struct * wip * wip * Implement FindNode * get nodes * WIP * Merge in Dennis kademlia code, get it working with our code * ping and moar * WIP trying to get cache working with kademlia * WIP more wiring up * WIP * Update service cli commands * WIP * added GetNodes * added nodes to Kbucket * default transport changed to TCP * GetBuckets interface changed * filling in more routing * timestamp methods * removed store * Added initial network overlay explorer page * Updating and building with dockerfile * Working on adding bootstrap node code * WIP merging in dennis' code * WIP * connects cache to pkg/kademlia implementation * WIP redis cache * testing * Add bootstrap network function for CLI usage * cleanup * call bootstrap on init network * Add BootstrapNetwork function to interface * Merge in dennis kad code * WIP updates to redis/overlay client interface * WIP trying to get the DHT connected to the cache * go mod & test * deps * Bootstrap node now setting up correctly - Need to pass it through CLI commands better * WIP adding refresh and walk functions, added cli flags - added cli flags for custom bootstrap port and ip * PR comments addressed * adding FindStorageNodes to overlay cache * fix GetBucket * using SplitHostPort * Use JoinHostPort * updates to findstoragenodes response and request * WIP merge in progress, having issues with a panic * wip * adjustments * update port for dht bootstrap test * Docker * wip * dockerfile * fixes * makefile changes * Update port in NewKademlia call * Update local kademlia DHT config * kubernetes yaml * cleanup * making tests pass * k8s yaml * lint issues * Edit cli flags to allow for configurable bootstrap IP and Port args * cleanup * cache walking the network now * Rough prototype of Walk function laid out * Move walk function into bootstrap function * Update dht.go * changes to yaml * goimports
2018-06-05 22:06:37 +01:00
nodes, err := o.DHT.GetNodes(ctx, "0", 128)
if err != nil {
return err
}
for _, v := range nodes {
_, err := o.DHT.FindNode(ctx, kademlia.StringToNodeID(v.Id))
if err != nil {
zap.Error(ErrNodeNotFound)
return err
Cache (#67) * add reference to dht to overlay client struct * wip * wip * Implement FindNode * get nodes * WIP * Merge in Dennis kademlia code, get it working with our code * ping and moar * WIP trying to get cache working with kademlia * WIP more wiring up * WIP * Update service cli commands * WIP * added GetNodes * added nodes to Kbucket * default transport changed to TCP * GetBuckets interface changed * filling in more routing * timestamp methods * removed store * Added initial network overlay explorer page * Updating and building with dockerfile * Working on adding bootstrap node code * WIP merging in dennis' code * WIP * connects cache to pkg/kademlia implementation * WIP redis cache * testing * Add bootstrap network function for CLI usage * cleanup * call bootstrap on init network * Add BootstrapNetwork function to interface * Merge in dennis kad code * WIP updates to redis/overlay client interface * WIP trying to get the DHT connected to the cache * go mod & test * deps * Bootstrap node now setting up correctly - Need to pass it through CLI commands better * WIP adding refresh and walk functions, added cli flags - added cli flags for custom bootstrap port and ip * PR comments addressed * adding FindStorageNodes to overlay cache * fix GetBucket * using SplitHostPort * Use JoinHostPort * updates to findstoragenodes response and request * WIP merge in progress, having issues with a panic * wip * adjustments * update port for dht bootstrap test * Docker * wip * dockerfile * fixes * makefile changes * Update port in NewKademlia call * Update local kademlia DHT config * kubernetes yaml * cleanup * making tests pass * k8s yaml * lint issues * Edit cli flags to allow for configurable bootstrap IP and Port args * cleanup * cache walking the network now * Rough prototype of Walk function laid out * Move walk function into bootstrap function * Update dht.go * changes to yaml * goimports
2018-06-05 22:06:37 +01:00
}
}
return nil
}