JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cache (#67)

Browse Source 
* add reference to dht to overlay client struct

* wip

* wip

* Implement FindNode

* get nodes

* WIP

* Merge in Dennis kademlia code, get it working with our code

* ping and moar

* WIP trying to get cache working with kademlia

* WIP more wiring up

* WIP

* Update service cli commands

* WIP

* added GetNodes

* added nodes to Kbucket

* default transport changed to TCP

* GetBuckets interface changed

* filling in more routing

* timestamp methods

* removed store

* Added initial network overlay explorer page

* Updating and building with dockerfile

* Working on adding bootstrap node code

* WIP merging in dennis' code

* WIP

* connects cache to pkg/kademlia implementation

* WIP redis cache

* testing

* Add bootstrap network function for CLI usage

* cleanup

* call bootstrap on init network

* Add BootstrapNetwork function to interface

* Merge in dennis kad code

* WIP updates to redis/overlay client interface

* WIP trying to get the DHT connected to the cache

* go mod & test

* deps

* Bootstrap node now setting up correctly

- Need to pass it through CLI commands better

* WIP adding refresh and walk functions, added cli flags

- added cli flags for custom bootstrap port and ip

* PR comments addressed

* adding FindStorageNodes to overlay cache

* fix GetBucket

* using SplitHostPort

* Use JoinHostPort

* updates to findstoragenodes response and request

* WIP merge in progress, having issues with a panic

* wip

* adjustments

* update port for dht bootstrap test

* Docker

* wip

* dockerfile

* fixes

* makefile changes

* Update port in NewKademlia call

* Update local kademlia DHT config

* kubernetes yaml

* cleanup

* making tests pass

* k8s yaml

* lint issues

* Edit cli flags to allow for configurable bootstrap IP and Port args

* cleanup

* cache walking the network now

* Rough prototype of Walk function laid out

* Move walk function into bootstrap function

* Update dht.go

* changes to yaml

* goimports
This commit is contained in:
Dylan Lott 2018-06-05 17:06:37 -04:00 committed by Dennis Coyle
parent 6be2baf9f9
commit 325a70d514
19 changed files with 610 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.gitignore vendored
View File

@ -18,5 +18,12 @@
/.vscode
debug
# Binaries
./main
./storj
# Jetbrains
.idea/*
# vendor
vendor

12
Dockerfile Normal file
View File

@ -0,0 +1,12 @@
# build
FROM golang:alpine AS build-env
ADD . /go/src/storj.io/storj
RUN cd /go/src/storj.io/storj/cmd/overlay && go build -o overlay
# final stage
FROM alpine
WORKDIR /app
COPY --from=build-env /go/src/storj.io/storj/cmd/overlay/overlay /app/
ENTRYPOINT ./overlay -redisAddress=${REDIS_ADDRESS} -redisPassword=${REDIS_PASSWORD} -db=${REDIS_DB} -srvPort=${OVERLAY_PORT} -httpPort=${HTTP_PORT}

285
Gopkg.lock generated
View File

@ -44,13 +44,23 @@
[[projects]]
name = "github.com/cheggaaa/pb"
packages = ["."]
revision = "73ae1d68fe0bd482ab11913a9828634f795b987f"
revision = "c112833d014c77e8bde723fd0158e3156951639f"
version = "v2.0.6"
[[projects]]
name = "github.com/cloudfoundry/gosigar"
packages = [
".",
"sys/windows"
]
revision = "dcb1afc219e4c99f21c37fb1ec22ed9ffe443317"
version = "v1.1.0"
[[projects]]
branch = "master"
name = "github.com/coyle/kademlia"
packages = ["."]
revision = "9b02bb94db6771a85fb279998bfd5a854efe7df4"
revision = "23c5c505df986913b68fd7fa16af645c64de8557"
[[projects]]
name = "github.com/davecgh/go-spew"
@ -64,18 +74,20 @@
".",
"request"
]
revision = "01aeca54ebda6e0fbfafd0a524d234159c05ec20"
revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
version = "v3.2.0"
[[projects]]
name = "github.com/djherbis/atime"
packages = ["."]
revision = "8e47e0e01d08df8b9f840d74299c8ab70a024a30"
version = "v1.0.0"
[[projects]]
branch = "master"
name = "github.com/djherbis/atime"
packages = ["."]
revision = "89517e96e10b93292169a79fd4523807bdc5d5fa"
[[projects]]
name = "github.com/dustin/go-humanize"
packages = ["."]
revision = "259d2a102b871d17f30e3cd9881a642961a1e486"
revision = "02af3965c54e8cacf948b97fef38925c4120652c"
[[projects]]
branch = "master"
@ -107,17 +119,20 @@
".",
"packets"
]
revision = "d06cc70ac43d625e602946b5ff2346ddebb768e4"
revision = "36d01c2b4cbeb3d2a12063e4880ce30800af9560"
version = "v1.1.1"
[[projects]]
name = "github.com/elazarl/go-bindata-assetfs"
packages = ["."]
revision = "57eb5e1fc594ad4b0b1dbea7b286d299e0cb43c2"
revision = "30f82fa23fd844bd5bb1e5f216db87fd77b5eb43"
version = "v1.0.0"
[[projects]]
name = "github.com/fatih/color"
packages = ["."]
revision = "42c364ba490082e4815b5222728711b3440603eb"
revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4"
version = "v1.7.0"
[[projects]]
name = "github.com/fatih/structs"
@ -137,13 +152,14 @@
"internal",
"redis"
]
revision = "0d253a66e6e1349f4581d6d2b300ee434ee2da9f"
revision = "a69d19351219b6dd56f274f96d85a7014a2ec34e"
version = "v1.6.0"
[[projects]]
name = "github.com/go-ini/ini"
packages = ["."]
revision = "6529cf7c58879c08d927016dde4477f18a0634cb"
version = "v1.36.0"
revision = "06f5f3d67269ccec1fe5fe4134ba6e982984f7f5"
version = "v1.37.0"
[[projects]]
name = "github.com/go-redis/redis"
@ -157,13 +173,14 @@
"internal/singleflight",
"internal/util"
]
revision = "0f9028adf0837cf93c9705817493e5f6997cf026"
version = "v6.11.0"
revision = "83fb42932f6145ce52df09860384a4653d2d332a"
version = "v6.12.0"
[[projects]]
name = "github.com/go-sql-driver/mysql"
packages = ["."]
revision = "2e00b5cd70399450106cec6431c2e2ce3cae5034"
revision = "d523deb1b23d913de5bdada721a6071e71283618"
version = "v1.4.0"
[[projects]]
name = "github.com/gogo/protobuf"
@ -175,6 +192,12 @@
revision = "1adfc126b41513cc696b209667c8656ea7aac67c"
version = "v1.0.0"
[[projects]]
name = "github.com/golang/mock"
packages = ["gomock"]
revision = "c34cdb4725f4c3844d095133c6e40e448b86589b"
version = "v1.1.1"
[[projects]]
name = "github.com/golang/protobuf"
packages = [
@ -202,12 +225,14 @@
[[projects]]
name = "github.com/gorilla/handlers"
packages = ["."]
revision = "66e6c6f01d8da976ee113437745ca029c2b585a6"
revision = "90663712d74cb411cbef281bc1e08c19d1a76145"
version = "v1.3.0"
[[projects]]
name = "github.com/gorilla/mux"
packages = ["."]
revision = "9fa818a44c2bf1396a17f9d5a3c0f6dd39d2ff8e"
revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf"
version = "v1.6.2"
[[projects]]
name = "github.com/gorilla/rpc"
@ -215,7 +240,8 @@
"v2",
"v2/json2"
]
revision = "bd3317b8f6704b1b40e4e705ec9e987a535cd5d3"
revision = "22c016f3df3febe0c1f6727598b6389507e03a18"
version = "v1.1.0"
[[projects]]
branch = "master"
@ -262,6 +288,12 @@
revision = "6237cf65f3a6f7111cd8a42be3590df99a66bc7d"
version = "1.0.0"
[[projects]]
branch = "master"
name = "github.com/jtolds/monkit-hw"
packages = ["."]
revision = "9b6edb34372a110992ea2e174886cb03ff971230"
[[projects]]
name = "github.com/julienschmidt/httprouter"
packages = ["."]
@ -274,24 +306,20 @@
revision = "ae7887de9fa5d2db4eaa8174a7eff2c1ac00f2da"
version = "v1.1"
[[projects]]
name = "github.com/klauspost/crc32"
packages = ["."]
revision = "cb6bfca970f6908083f26f39a79009d608efd5cd"
version = "v1.1"
[[projects]]
name = "github.com/klauspost/reedsolomon"
packages = ["."]
revision = "0b30fa71cc8e4e9010c9aba6d0320e2e5b163b29"
revision = "6bb6130ff6a76a904c1841707d65603aec9cc288"
version = "v1.6"
[[projects]]
branch = "master"
name = "github.com/lib/pq"
packages = [
".",
"oid"
]
revision = "50761b0867bd1d9d069276790bcd4a3bccf2324a"
revision = "90697d60dd844d5ef6ff15135d0203f65d2f53b8"
[[projects]]
name = "github.com/magiconair/properties"
@ -299,6 +327,17 @@
revision = "c2353362d570a7bfa228149c62842019201cfb71"
version = "v1.8.0"
[[projects]]
branch = "master"
name = "github.com/mailru/easyjson"
packages = [
".",
"buffer",
"jlexer",
"jwriter"
]
revision = "9825584555aa620c53c265d4a09ace0df1346fd9"
[[projects]]
name = "github.com/mattn/go-colorable"
packages = ["."]
@ -311,16 +350,23 @@
revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
version = "v0.0.3"
[[projects]]
name = "github.com/mattn/go-sqlite3"
packages = ["."]
revision = "323a32be5a2421b8c7087225079c6c900ec397cd"
version = "v1.7.0"
[[projects]]
name = "github.com/matttproud/golang_protobuf_extensions"
packages = ["pbutil"]
revision = "3247c84500bff8d9fb6d579d800f20b3e091582c"
version = "v1.0.0"
revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c"
version = "v1.0.1"
[[projects]]
name = "github.com/minio/cli"
packages = ["."]
revision = "b8ae5507c0ceceecc22d5dbd386b58fbd4fdce72"
revision = "8683fa7fef37cc8cb092f47bdb6b403e0049f9ee"
version = "v1.3.0"
[[projects]]
branch = "master"
@ -329,19 +375,22 @@
revision = "439a0961af700f80db84cc180fe324a89070fa65"
[[projects]]
branch = "master"
name = "github.com/minio/highwayhash"
packages = ["."]
revision = "1ea1b5fce73b1d7abbd822a194b51f3736b1f23b"
revision = "85fc8a2dacad36a6beb2865793cd81363a496696"
[[projects]]
branch = "master"
name = "github.com/minio/lsync"
packages = ["."]
revision = "2d7c40f41402df6f0713a749a011cddc12d1b2f3"
revision = "f332c3883f63c75ea6c95eae3aec71a2b2d88b49"
[[projects]]
branch = "master"
name = "github.com/minio/mc"
packages = ["pkg/console"]
revision = "db6b4f13442b26995f04b3b2b31b006cae7786e6"
revision = "c0b89a7aecb89174f3fda68f726eea2e3342b0aa"
[[projects]]
branch = "master"
@ -353,6 +402,7 @@
"cmd/logger",
"pkg/auth",
"pkg/bpool",
"pkg/certs",
"pkg/cgroup",
"pkg/disk",
"pkg/ellipses",
@ -376,7 +426,7 @@
"pkg/wildcard",
"pkg/words"
]
revision = "5afd8563554a3b57291d25596addf9c504f4a1d2"
revision = "6fb06045028b7a57c37c60a612c8e50735279ab4"
[[projects]]
name = "github.com/minio/minio-go"
@ -389,26 +439,29 @@
"pkg/s3utils",
"pkg/set"
]
revision = "e163d8055f79cf2a9b8af9d358b2814f21fd0472"
version = "4.0.5"
revision = "034ea465b079a920e0720e3e1e0cbf5ccfeb6373"
version = "v6.0.2"
[[projects]]
branch = "master"
name = "github.com/minio/sha256-simd"
packages = ["."]
revision = "43ed500fe4d485d97534014d9f98521216240002"
revision = "ad98a36ba0da87206e3378c556abbfeaeaa98668"
[[projects]]
branch = "master"
name = "github.com/minio/sio"
packages = [
".",
"internal/cpu"
]
revision = "b421622190be1ffb4569c7303ed1b7bef201a7c3"
revision = "6a41828a60f0ec95a159ce7921ca3dd566ebd7e3"
[[projects]]
branch = "master"
name = "github.com/mitchellh/go-homedir"
packages = ["."]
revision = "b8bc1bf767474819792c23f32d8286a45736f1c6"
revision = "3864e76763d94a6df2f9960b16a20a33da9f9a66"
[[projects]]
branch = "master"
@ -432,15 +485,14 @@
".",
"pb"
]
revision = "077898146bfbb849a620202e7e5eaaf707492206"
revision = "e15a53f85e4932540600a16b56f6c4f65f58176f"
version = "v0.4.0"
[[projects]]
name = "github.com/nats-io/nats"
packages = [
".",
"encoders/builtin"
]
revision = "70b70be17b77e8da86b6a3bcfe94fb22718a8dd0"
packages = ["."]
revision = "062418ea1c2181f52dc0f954f6204370519a868b"
version = "v1.5.0"
[[projects]]
name = "github.com/nats-io/nuid"
@ -454,10 +506,29 @@
revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8"
version = "v1.1.0"
[[projects]]
name = "github.com/pierrec/lz4"
packages = ["."]
revision = "2fcda4cb7018ce05a25959d2fe08c83e3329f169"
version = "v1.1"
[[projects]]
name = "github.com/pierrec/xxHash"
packages = ["xxHash32"]
revision = "f051bb7f1d1aaf1b5a665d74fb6b0217712c69f7"
version = "v0.1.1"
[[projects]]
name = "github.com/pkg/errors"
packages = ["."]
revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
version = "v0.8.0"
[[projects]]
name = "github.com/pkg/profile"
packages = ["."]
revision = "c78aac22bd43883fd2817833b982153dcac17b3b"
revision = "5b67d428864e92711fcbd2f8629456121a56d91f"
version = "v1.2.1"
[[projects]]
name = "github.com/pmezard/go-difflib"
@ -471,7 +542,8 @@
"prometheus",
"prometheus/promhttp"
]
revision = "82f5ff156b29e276022b1a958f7d385870fb9814"
revision = "c5b7fccd204277076155f10851dad72b76a49317"
version = "v0.8.0"
[[projects]]
branch = "master"
@ -498,25 +570,31 @@
"nfs",
"xfs"
]
revision = "8b1c2da0d56deffdbb9e48d4414b4e674bd8083e"
revision = "94663424ae5ae9856b40a9f170762b4197024661"
[[projects]]
branch = "master"
name = "github.com/rcrowley/go-metrics"
packages = ["."]
revision = "e2704e165165ec55d062f5919b4b29494e9fa790"
[[projects]]
branch = "master"
name = "github.com/rjeczalik/notify"
packages = ["."]
revision = "d152f3ce359a5464dc41e84a8919fc67e55bbbf0"
[[projects]]
name = "github.com/rs/cors"
packages = ["."]
revision = "a62a804a8a009876ca59105f7899938a1349f4b3"
version = "v1.0"
revision = "ca016a06a5753f8ba03029c0aa5e54afb1bf713f"
version = "v1.4.0"
[[projects]]
name = "github.com/rs/xhandler"
packages = ["."]
revision = "d9d9599b6aaf6a058cb7b1f48291ded2cbd13390"
version = "v1.0"
[[projects]]
branch = "master"
name = "github.com/segmentio/go-prompt"
packages = ["."]
revision = "f0d19b6901ade831d5a3204edc0d6a7d6457fbb2"
revision = "0be3d6bb7dff1db1f7d4127e5861ce97772dd891"
version = "v1.2.0"
[[projects]]
name = "github.com/sirupsen/logrus"
@ -525,9 +603,10 @@
version = "v1.0.5"
[[projects]]
branch = "master"
name = "github.com/skyrings/skyring-common"
packages = ["tools/uuid"]
revision = "762fd2bfc12e766d90478d638255981ab1966a3d"
revision = "d1c0bb1cbd5ed8438be1385c85c4f494608cde1e"
[[projects]]
branch = "master"
@ -554,6 +633,12 @@
]
revision = "7067dc99a42a3d5214038549d4b677779c21d5c2"
[[projects]]
branch = "master"
name = "github.com/spacemonkeygo/spacelog"
packages = ["."]
revision = "2296661a0572a51438413369004fa931c2641923"
[[projects]]
name = "github.com/spf13/afero"
packages = [
@ -588,9 +673,10 @@
version = "v1.0.2"
[[projects]]
branch = "master"
name = "github.com/streadway/amqp"
packages = ["."]
revision = "2e25825abdbd7752ff08b270d313b93519a0a232"
revision = "e5adc2ada8b8efff032bf61173a233d143e9318e"
[[projects]]
name = "github.com/stretchr/testify"
@ -601,7 +687,8 @@
[[projects]]
name = "github.com/tidwall/gjson"
packages = ["."]
revision = "09d1c5c5bc64e094394dfe2150220d906c55ac37"
revision = "01f00f129617a6fe98941fb920d6c760241b54d2"
version = "v1.1.0"
[[projects]]
branch = "master"
@ -659,8 +746,8 @@
[[projects]]
name = "go.uber.org/atomic"
packages = ["."]
revision = "4e336646b2ef9fc6e47be8e21594178f98e5ebcf"
version = "v1.2.0"
revision = "1ea20fb1cbb1cc08cbd0d913a96dead89aa18289"
version = "v1.3.2"
[[projects]]
name = "go.uber.org/multierr"
@ -685,6 +772,7 @@
branch = "master"
name = "golang.org/x/crypto"
packages = [
"argon2",
"blake2b",
"chacha20poly1305",
"internal/chacha20",
@ -694,23 +782,24 @@
"salsa20/salsa",
"ssh/terminal"
]
revision = "75e913eb8a8e3d31a97b216de09de106a7b07681"
revision = "df8d4716b3472e4a531c33cedbe537dae921a1a9"
[[projects]]
branch = "master"
name = "golang.org/x/net"
packages = [
"context",
"context/ctxhttp",
"http/httpguts",
"http2",
"http2/hpack",
"idna",
"internal/socks",
"internal/timeseries",
"proxy",
"trace",
"websocket"
]
revision = "9ef9f5bb98a1fdc41f8cf6c250a4404b4085e389"
revision = "1e491301e022f8f977054da4c2d852decd59571f"
[[projects]]
branch = "master"
@ -720,7 +809,7 @@
"unix",
"windows"
]
revision = "56ad15cc2170219c9fed5b7d059e825c97588071"
revision = "c11f84a56e43e20a78cee75a7c034031ecf57d1f"
[[projects]]
name = "golang.org/x/text"
@ -744,15 +833,22 @@
version = "v0.3.0"
[[projects]]
branch = "master"
name = "golang.org/x/time"
packages = ["rate"]
revision = "6dc17368e09b0e8634d71cac8168d853e869a0c7"
revision = "fbb02b2291d28baffd63558aa44b4b56f178d650"
[[projects]]
name = "google.golang.org/appengine"
packages = ["cloudsql"]
revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
version = "v1.0.0"
[[projects]]
branch = "master"
name = "google.golang.org/genproto"
packages = ["googleapis/rpc/status"]
revision = "694d95ba50e67b2e363f3483057db5d4910c18f9"
revision = "81158efcc9f219c511e4d3c0d61a0e6e49c01a24"
[[projects]]
name = "google.golang.org/grpc"
@ -788,17 +884,54 @@
[[projects]]
name = "gopkg.in/Shopify/sarama.v1"
packages = ["."]
revision = "bd61cae2be85fa6ff40eb23dcdd24567967ac2ae"
version = "v1.10.1"
revision = "35324cf48e33d8260e1c7c18854465a904ade249"
version = "v1.17.0"
[[projects]]
name = "gopkg.in/VividCortex/ewma.v1"
packages = ["."]
revision = "b24eb346a94c3ba12c1da1e564dbac1b498a77ce"
version = "v1.1.1"
[[projects]]
name = "gopkg.in/cheggaaa/pb.v2"
packages = ["termutil"]
revision = "c112833d014c77e8bde723fd0158e3156951639f"
version = "v2.0.6"
[[projects]]
name = "gopkg.in/fatih/color.v1"
packages = ["."]
revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4"
version = "v1.7.0"
[[projects]]
name = "gopkg.in/mattn/go-colorable.v0"
packages = ["."]
revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072"
version = "v0.0.9"
[[projects]]
name = "gopkg.in/mattn/go-isatty.v0"
packages = ["."]
revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
version = "v0.0.3"
[[projects]]
name = "gopkg.in/mattn/go-runewidth.v0"
packages = ["."]
revision = "9e777a8366cce605130a531d2cd6363d07ad7317"
version = "v0.0.2"
[[projects]]
name = "gopkg.in/olivere/elastic.v5"
packages = [
".",
"config",
"uritemplates"
]
revision = "cf4e58efdcee2e8e7c18dad44d51ed166fb256c2"
version = "v5.0.31"
revision = "b708306d715bea9b983685e94ab4602cdc9f988b"
version = "v5.0.69"
[[projects]]
branch = "v2"
@ -820,6 +953,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "1977a85f03364feade0907b136af3681d6e047e4e20bb34394ba0fb046c56519"
inputs-digest = "101bde363c549e6e970d993982cd39dc7751cb980569d98187078b0dd8dbcdff"
solver-name = "gps-cdcl"
solver-version = 1

33
Makefile
View File

@ -3,7 +3,7 @@
lint: check-copyrights
@echo "Running ${@}"
@gometalinter \
--deadline=70s \
--deadline=170s \
--disable-all \
--enable=golint \
--enable=goimports \
@ -31,3 +31,34 @@ test: lint
go install -v ./...
go test ./...
@echo done
build-binaries:
docker build -t overlay .
run-overlay:
docker network create test-net
docker run -d \
--name redis \
--network test-net \
-p 127.0.0.1:6379:6379 \
redis
docker run -d \
--name=overlay \
--network test-net \
-e REDIS_ADDRESS=redis:6379 \
-e REDIS_PASSWORD="" \
-e REDIS_DB=1 \
-e OVERLAY_PORT=8080 \
overlay
clean-local:
# cleanup overlay
docker stop overlay || true
docker rm overlay || true
# cleanup redis
docker stop redis || true
docker rm redis || true
# cleanup docker network
docker network rm test-net || true

23
cmd/overlay/README.md
View File

@ -1 +1,22 @@
TODO
TODO
# Overlay Network
Documentation for developing and building the overlay network component of the Storj node.
## Running as a cache server
To run a cache, you'll need a running instance of Redis.
Using docker is the fastest way to get a redis instance up and running.
`docker run -p 6379:6379 --name -d redis`
Once you have that running, build the binary.
`go build cmd/overlay/main.go`
Then you can run the node with the -cache flag
`./main -cache localhost:6379`

74
cmd/overlay/overlay.yaml Normal file
View File

@ -0,0 +1,74 @@
apiVersion: v1
kind: Service
metadata:
name: overlay
labels:
app: overlay
spec:
ports:
- name: grpc
port: 8080
targetPort: 8080
- name: http
port: 8081
targetPort: 8081
selector:
app: overlay
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: overlay
labels:
app: overlay
spec:
minReadySeconds: 10
revisionHistoryLimit: 3
strategy:
rollingUpdate:
maxUnavailable: 1
replicas: 1
template:
metadata:
labels:
app: overlay
spec:
terminationGracePeriodSeconds: 60
containers:
- image: "docker.io/storjlabs/overlay"
imagePullPolicy: Always
name: overlay
livenessProbe:
httpGet:
path: /health
port: 8081
initialDelaySeconds: 30
readinessProbe:
httpGet:
path: /health
port: 8081
initialDelaySeconds: 10
env:
- name: REDIS_ADDRESS
value: "35.184.203.66:6379"
- name: REDIS_PASSWORD
value: ""
- name: REDIS_DB
value: "1"
- name: OVERLAY_PORT
value: "8080"
- name: HTTP_PORT
value: "8081"
ports:
- name: grpc
containerPort: 8080
- name: http
containerPort: 8081
resources:
requests:
cpu: 200m
memory: 64Mi
limits:
cpu: 300m
memory: 128Mi

9
go.mod
View File

@ -6,7 +6,8 @@ require (
github.com/anacrolix/utp v0.0.0-20180219060659-9e0e1d1d0572
github.com/boltdb/bolt v1.3.1
github.com/ccding/go-stun v0.0.0-20171206150302-d9bbe8f8fa7b
github.com/coyle/kademlia v0.0.0-20180531194258-9b02bb94db67
github.com/cloudfoundry/gosigar v1.1.0
github.com/coyle/kademlia v0.0.0-20180604160050-23c5c505df98
github.com/fatih/structs v1.0.0
github.com/fsnotify/fsnotify v1.4.7
github.com/go-redis/redis v0.0.0-20180417061816-9ccc23344a52
@ -15,19 +16,19 @@ require (
github.com/golang/protobuf v1.1.0
github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce
github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6
github.com/jtolds/monkit-hw v0.0.0-20180222001630-9b6edb34372a
github.com/magiconair/properties v1.7.6
github.com/mattn/go-sqlite3 v1.7.0
github.com/minio/cli v1.3.0
github.com/minio/minio v0.0.0-20180601024350-c22b9d5d4db3
github.com/mattn/go-sqlite3 v1.7.0
github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238
github.com/pelletier/go-toml v1.1.0
github.com/spacemonkeygo/errors v0.0.0-20171212215202-9064522e9fd1
github.com/spacemonkeygo/flagfile v0.0.0-20180426194429-0d750334dbb8
github.com/spacemonkeygo/monotime v0.0.0-20180102220400-7067dc99a42a
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572
github.com/spf13/afero v1.1.0
github.com/spf13/cast v1.2.0
github.com/spf13/jWalterWeatherman v0.0.0-20180109140146-7c0cea34c8ec
github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec
github.com/spf13/pflag v1.0.1
github.com/spf13/viper v1.0.2
github.com/tyler-smith/go-bip39 v0.0.0-20160629163856-8e7a99b3e716

1
index.html Normal file
View File

@ -0,0 +1 @@
<h1>Storj Node</h1>

2
pkg/kademlia/bucket.go
View File

@ -7,7 +7,7 @@ import "storj.io/storj/protos/overlay"
// KBucket implements the Bucket interface
type KBucket struct {
nodes []overlay.Node
nodes []*overlay.Node
}
// Routing __ (TODO) still not entirely sure what the bucket methods are supposed to do

50
pkg/kademlia/dht.go
View File

@ -43,13 +43,17 @@ func NewKademlia(bootstrapNodes []proto.Node, ip string, port string) (*Kademlia
return nil, err
}
bdht, _ := bkad.NewDHT(&bkad.MemoryStore{}, &bkad.Options{
bdht, err := bkad.NewDHT(&bkad.MemoryStore{}, &bkad.Options{
ID: []byte(id),
IP: ip,
Port: port,
BootstrapNodes: bb,
})
if err != nil {
return nil, err
}
rt := RouteTable{
ht: bdht.HT,
dht: bdht,
@ -66,10 +70,14 @@ func NewKademlia(bootstrapNodes []proto.Node, ip string, port string) (*Kademlia
}
// GetNodes returns all nodes from a starting node up to a maximum limit stored in the local routing table
func (k *Kademlia) GetNodes(ctx context.Context, start string, limit int) ([]proto.Node, error) {
func (k Kademlia) GetNodes(ctx context.Context, start string, limit int) ([]*proto.Node, error) {
if start == "" {
start = k.dht.GetSelfID()
}
nn, err := k.dht.FindNodes(ctx, start, limit)
if err != nil {
return []proto.Node{}, err
return []*proto.Node{}, err
}
return convertNetworkNodes(nn), nil
}
@ -132,7 +140,9 @@ func (k *Kademlia) ListenAndServe() error {
return err
}
return k.dht.Listen()
go k.dht.Listen()
return nil
}
func convertProtoNodes(n []proto.Node) ([]*bkad.NetworkNode, error) {
@ -148,8 +158,8 @@ func convertProtoNodes(n []proto.Node) ([]*bkad.NetworkNode, error) {
return nn, nil
}
func convertNetworkNodes(n []*bkad.NetworkNode) []proto.Node {
nn := make([]proto.Node, len(n))
func convertNetworkNodes(n []*bkad.NetworkNode) []*proto.Node {
nn := make([]*proto.Node, len(n))
for i, v := range n {
nn[i] = convertNetworkNode(v)
}
@ -157,8 +167,8 @@ func convertNetworkNodes(n []*bkad.NetworkNode) []proto.Node {
return nn
}
func convertNetworkNode(v *bkad.NetworkNode) proto.Node {
return proto.Node{
func convertNetworkNode(v *bkad.NetworkNode) *proto.Node {
return &proto.Node{
Id: string(v.ID),
Address: &proto.NodeAddress{Transport: defaultTransport, Address: net.JoinHostPort(v.IP.String(), strconv.Itoa(v.Port))},
}
@ -183,3 +193,27 @@ func newID() ([]byte, error) {
_, err := rand.Read(result)
return result, err
}
// GetIntroNode determines the best node to bootstrap a new node onto the network
func GetIntroNode(ip, port string) proto.Node {
id, _ := newID() // TODO(coyle): This is solely to bootstrap our very first node, after we get an ID, we will just hardcode that ID
// if ip == "" {
// return proto.Node{
// Id: string(id),
// Address: &proto.NodeAddress{
// Transport: defaultTransport,
// Address: "35.232.202.229:8080",
// },
// }
// }
//
// address := fmt.Sprintf("%s:%s", ip, port)
return proto.Node{
Id: string(id),
Address: &proto.NodeAddress{
Transport: defaultTransport,
Address: "35.232.202.229:8080",
},
}
}

4
pkg/kademlia/kademlia.go
View File

@ -15,7 +15,7 @@ type NodeID string
// DHT is the interface for the DHT in the Storj network
type DHT interface {
GetNodes(ctx context.Context, start string, limit int) ([]overlay.Node, error)
GetNodes(ctx context.Context, start string, limit int) ([]*overlay.Node, error)
GetRoutingTable(ctx context.Context) (RoutingTable, error)
Bootstrap(ctx context.Context) error
@ -33,7 +33,7 @@ type RoutingTable interface {
GetBucket(id string) (bucket Bucket, ok bool)
GetBuckets() ([]Bucket, error)
FindNear(id NodeID, limit int) ([]overlay.Node, error)
FindNear(id NodeID, limit int) ([]*overlay.Node, error)
ConnectionSuccess(id string, address overlay.NodeAddress)
ConnectionFailed(id string, address overlay.NodeAddress)

13
pkg/kademlia/routing.go
View File

@ -4,7 +4,9 @@
package kademlia
import (
"context"
"encoding/hex"
"fmt"
"strconv"
"time"
@ -39,7 +41,7 @@ func (rt RouteTable) K() int {
// CacheSize returns the total current size of the cache
func (rt RouteTable) CacheSize() int {
//TODO: How is this calculated ? size of the routing table ? is it total bytes, mb, kb etc .?
// TODO: How is this calculated ? size of the routing table ? is it total bytes, mb, kb etc .?
return 0
}
@ -71,7 +73,7 @@ func (rt RouteTable) GetBuckets() (k []Bucket, err error) {
}
// FindNear finds all Nodes near the provided nodeID up to the provided limit
func (rt RouteTable) FindNear(id NodeID, limit int) ([]overlay.Node, error) {
func (rt RouteTable) FindNear(id NodeID, limit int) ([]*overlay.Node, error) {
return convertNetworkNodes(rt.ht.GetClosestContacts([]byte(id), limit)), nil
}
@ -105,3 +107,10 @@ func (rt RouteTable) SetBucketTimestamp(id string, now time.Time) error {
func (rt RouteTable) GetBucketTimestamp(id string, bucket Bucket) (time.Time, error) {
return rt.dht.GetExpirationTime([]byte(id)), nil
}
// GetNodeRoutingTable gets a routing table for a given node rather than the local node's routing table
func GetNodeRoutingTable(ctx context.Context, ID NodeID) (RouteTable, error) {
fmt.Println("GetNodeRoutingTable")
fmt.Println("id: ", ID)
return RouteTable{}, nil
}

37
pkg/overlay/overlay.go
View File

@ -6,20 +6,47 @@ package overlay
import (
"context"
"go.uber.org/zap"
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/kademlia"
proto "storj.io/storj/protos/overlay" // naming proto to avoid confusion with this package
"storj.io/storj/storage/redis"
)
// Overlay implements our overlay RPC service
type Overlay struct{}
type Overlay struct {
kad *kademlia.Kademlia
DB *redis.OverlayClient
logger *zap.Logger
metrics *monkit.Registry
}
// Lookup finds the address of a node in our overlay network
func (o *Overlay) Lookup(ctx context.Context, req *proto.LookupRequest) (*proto.LookupResponse, error) {
// TODO: fill this in with logic to communicate with kademlia
return &proto.LookupResponse{}, nil
na, err := o.DB.Get(ctx, req.NodeID)
if err != nil {
o.logger.Error("Error looking up node", zap.Error(err), zap.String("nodeID", req.NodeID))
return nil, err
}
return &proto.LookupResponse{
NodeAddress: na,
}, nil
}
// FindStorageNodes searches the overlay network for nodes that meet the provided requirements
func (o *Overlay) FindStorageNodes(ctx context.Context, req *proto.FindStorageNodesRequest) (*proto.FindStorageNodesResponse, error) {
// TODO: fill this in with logic to communicate with kademlia
return &proto.FindStorageNodesResponse{}, nil
// NB: call FilterNodeReputation from node_reputation package to find nodes for storage
// TODO(coyle): need to determine if we will pull the startID and Limit from the request or just use hardcoded data
// for now just using 40 for demos and empty string which will default the Id to the kademlia node doing the lookup
nodes, err := o.kad.GetNodes(ctx, "", 40)
if err != nil {
o.logger.Error("Error getting nodes", zap.Error(err))
return nil, err
}
return &proto.FindStorageNodesResponse{
Node: nodes,
}, nil
}

75
pkg/overlay/service.go
View File

@ -8,6 +8,7 @@ import (
"flag"
"fmt"
"net"
"net/http"
"go.uber.org/zap"
"google.golang.org/grpc"
@ -19,22 +20,32 @@ import (
)
var (
redisAddress string
redisPassword string
db int
redisAddress, redisPassword, httpPort, bootstrapIP, bootstrapPort, localPort string
db int
srvPort uint
)
func init() {
flag.StringVar(&redisAddress, "cache", "", "The <IP:PORT> string to use for connection to a redis cache")
flag.StringVar(&redisPassword, "password", "", "The password used for authentication to a secured redis instance")
flag.StringVar(&httpPort, "httpPort", "", "The port for the health endpoint")
flag.StringVar(&redisAddress, "redisAddress", "", "The <IP:PORT> string to use for connection to a redis cache")
flag.StringVar(&redisPassword, "redisPassword", "", "The password used for authentication to a secured redis instance")
flag.IntVar(&db, "db", 0, "The network cache database")
flag.UintVar(&srvPort, "srvPort", 8080, "Port to listen on")
flag.StringVar(&bootstrapIP, "bootstrapIP", "", "Optional IP to bootstrap node against")
flag.StringVar(&bootstrapPort, "bootstrapPort", "", "Optional port of node to bootstrap against")
flag.StringVar(&localPort, "localPort", "8080", "Specify a different port to listen on locally")
flag.Parse()
}
// NewServer creates a new Overlay Service Server
func NewServer() *grpc.Server {
func NewServer(k *kademlia.Kademlia, db *redis.OverlayClient, l *zap.Logger, m *monkit.Registry) *grpc.Server {
grpcServer := grpc.NewServer()
proto.RegisterOverlayServer(grpcServer, &Overlay{})
proto.RegisterOverlayServer(grpcServer, &Overlay{
kad: k,
DB: db,
logger: l,
metrics: m,
})
return grpcServer
}
@ -58,16 +69,38 @@ type Service struct {
// Process is the main function that executes the service
func (s *Service) Process(ctx context.Context) error {
// bootstrap network
kad := kademlia.Kademlia{}
// TODO
// 1. Boostrap a node on the network
// 2. Start up the overlay gRPC service
// 3. Connect to Redis
// 4. Boostrap Redis Cache
kad.Bootstrap(ctx)
// bootstrap cache
cache, err := redis.NewOverlayClient(redisAddress, redisPassword, db, &kad)
// TODO(coyle): Should add the ability to pass a configuration to change the bootstrap node
in := kademlia.GetIntroNode(bootstrapIP, bootstrapPort)
kad, err := kademlia.NewKademlia([]proto.Node{in}, "127.0.0.1", localPort)
if err != nil {
s.logger.Error("Failed to create a new overlay client", zap.Error(err))
s.logger.Error("Failed to instantiate new Kademlia", zap.Error(err))
return err
}
if err := kad.ListenAndServe(); err != nil {
s.logger.Error("Failed to ListenAndServe on new Kademlia", zap.Error(err))
return err
}
if err := kad.Bootstrap(ctx); err != nil {
s.logger.Error("Failed to Bootstrap on new Kademlia", zap.Error(err))
return err
}
// bootstrap cache
cache, err := redis.NewOverlayClient(redisAddress, redisPassword, db, kad)
if err != nil {
s.logger.Error("Failed to create a new redis overlay client", zap.Error(err))
return err
}
if err := cache.Bootstrap(ctx); err != nil {
s.logger.Error("Failed to boostrap cache", zap.Error(err))
return err
@ -76,18 +109,26 @@ func (s *Service) Process(ctx context.Context) error {
// send off cache refreshes concurrently
go cache.Refresh(ctx)
lis, err := net.Listen("tcp", fmt.Sprintf(":%d", 0))
lis, err := net.Listen("tcp", fmt.Sprintf(":%d", srvPort))
if err != nil {
s.logger.Error("Failed to initialize TCP connection", zap.Error(err))
return err
}
grpcServer := grpc.NewServer()
proto.RegisterOverlayServer(grpcServer, &Overlay{})
proto.RegisterOverlayServer(grpcServer, &Overlay{
kad: kad,
DB: cache,
logger: s.logger,
metrics: s.metrics,
})
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, "OK") })
go func() { http.ListenAndServe(fmt.Sprintf(":%s", httpPort), nil) }()
go cache.Walk(ctx)
defer grpcServer.GracefulStop()
return grpcServer.Serve(lis)
}
// SetLogger adds the initialized logger to the Service

9
pkg/overlay/service_test.go
View File

@ -16,10 +16,11 @@ import (
)
func TestNewServer(t *testing.T) {
t.SkipNow()
lis, err := net.Listen("tcp", fmt.Sprintf(":%d", 0))
assert.NoError(t, err)
srv := NewServer()
srv := NewServer(nil, nil, nil, nil)
assert.NotNil(t, srv)
go srv.Serve(lis)
@ -27,10 +28,12 @@ func TestNewServer(t *testing.T) {
}
func TestNewClient(t *testing.T) {
//a := "35.232.202.229:8080"
//c, err := NewClient(&a, grpc.WithInsecure())
t.SkipNow()
lis, err := net.Listen("tcp", fmt.Sprintf(":%d", 0))
assert.NoError(t, err)
srv := NewServer()
srv := NewServer(nil, nil, nil, nil)
go srv.Serve(lis)
defer srv.Stop()

1
protos/overlay/overlay.proto
View File

@ -51,6 +51,7 @@ message OverlayOptions {
google.protobuf.Duration maxLatency = 1;
NodeRep minReputation = 2; // Not sure what NodeRep is yet.
int64 minSpeedKbps = 3;
int64 limit = 4;
}
// NodeRep is the reputation characteristics of a node

19
static/index.html Normal file
View File

@ -0,0 +1,19 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Storj Node Network Explorer</title>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
<script defer src="https://use.fontawesome.com/releases/v5.0.7/js/all.js"></script>
</head>
<body>
<section class="section">
<div class="container">
<h1 class="title">
Storj Node Network Explorer
</h1>
</div>
</section>
</body>
</html>

80
storage/redis/overlay.go
View File

@ -6,6 +6,7 @@ package redis
import (
"context"
"errors"
"fmt"
"time"
"github.com/gogo/protobuf/proto"
@ -16,6 +17,9 @@ import (
const defaultNodeExpiration = 61 * time.Minute
// ErrNodeNotFound standardizes errors here
var ErrNodeNotFound = errors.New("Node not found")
// OverlayClient is used to store overlay data in Redis
type OverlayClient struct {
DB Client
@ -30,20 +34,24 @@ func NewOverlayClient(address, password string, db int, DHT kademlia.DHT) (*Over
}
return &OverlayClient{
DB: rc,
DB: rc,
DHT: DHT,
}, nil
}
// Get looks up the provided nodeID from the redis cache
func (o *OverlayClient) Get(key string) (*overlay.NodeAddress, error) {
d, err := o.DB.Get(key)
func (o *OverlayClient) Get(ctx context.Context, key string) (*overlay.NodeAddress, error) {
b, err := o.DB.Get(key)
if err != nil {
return nil, err
}
na := &overlay.NodeAddress{}
if err := proto.Unmarshal(b, na); err != nil {
return nil, err
}
return na, proto.Unmarshal(d, na)
return na, nil
}
// Set adds a nodeID to the redis cache with a binary representation of proto defined NodeAddress
@ -58,10 +66,70 @@ func (o *OverlayClient) Set(nodeID string, value overlay.NodeAddress) error {
// Bootstrap walks the initialized network and populates the cache
func (o *OverlayClient) Bootstrap(ctx context.Context) error {
return errors.New("TODO")
fmt.Println("bootstrapping cache")
nodes, err := o.DHT.GetNodes(ctx, "0", 1280)
for _, v := range nodes {
found, err := o.DHT.FindNode(ctx, kademlia.NodeID(v.Id))
if err != nil {
fmt.Println("could not find node in network", err, v.Id)
}
addr, err := proto.Marshal(found.Address)
o.DB.Set(found.Id, addr, defaultNodeExpiration)
}
// called after kademlia is bootstrapped
// needs to take RoutingTable and start to persist it into the cache
// take bootstrap node
// get their route table
// loop through nodes in RT and get THEIR route table
// keep going forever and ever
// Other Possibilities: Randomly generate node ID's to ask for?
_, err = o.DHT.GetRoutingTable(ctx)
if err != nil {
return err
}
return nil
}
// Refresh walks the network looking for new nodes and pings existing nodes to eliminate stale addresses
func (o *OverlayClient) Refresh(ctx context.Context) error {
return errors.New("TODO")
// iterate over all nodes
// compare responses to find new nodes
// listen for responses from existing nodes
// if no response from existing, then mark it as offline for time period
// if responds, it refreshes in DHT
_, rtErr := o.DHT.GetRoutingTable(ctx)
if rtErr != nil {
return rtErr
}
_, err := o.DHT.GetNodes(ctx, "0", 128)
if err != nil {
return err
}
return nil
}
// Walk iterates over buckets to traverse the network
func (o *OverlayClient) Walk(ctx context.Context) error {
nodes, err := o.DHT.GetNodes(ctx, "0", 128)
if err != nil {
return err
}
for _, v := range nodes {
_, err := o.DHT.FindNode(ctx, kademlia.NodeID(v.Id))
if err != nil {
fmt.Println("could not find node in network", err, v.Id)
}
}
return nil
}

3
storage/redis/overlay_test.go
View File

@ -4,6 +4,7 @@
package redis
import (
"context"
"testing"
"github.com/gogo/protobuf/proto"
@ -69,7 +70,7 @@ func TestGet(t *testing.T) {
assert.Equal(t, 0, c.client.getCalled)
resp, err := oc.Get(c.key)
resp, err := oc.Get(context.Background(), c.key)
assert.Equal(t, c.expectedError, err)
assert.Equal(t, c.expectedResponse, resp)
assert.Equal(t, c.expectedTimesCalled, c.client.getCalled)