From 325a70d514203836984ddf02d796c65c344ee1fc Mon Sep 17 00:00:00 2001 From: Dylan Lott Date: Tue, 5 Jun 2018 17:06:37 -0400 Subject: [PATCH] Cache (#67) * add reference to dht to overlay client struct * wip * wip * Implement FindNode * get nodes * WIP * Merge in Dennis kademlia code, get it working with our code * ping and moar * WIP trying to get cache working with kademlia * WIP more wiring up * WIP * Update service cli commands * WIP * added GetNodes * added nodes to Kbucket * default transport changed to TCP * GetBuckets interface changed * filling in more routing * timestamp methods * removed store * Added initial network overlay explorer page * Updating and building with dockerfile * Working on adding bootstrap node code * WIP merging in dennis' code * WIP * connects cache to pkg/kademlia implementation * WIP redis cache * testing * Add bootstrap network function for CLI usage * cleanup * call bootstrap on init network * Add BootstrapNetwork function to interface * Merge in dennis kad code * WIP updates to redis/overlay client interface * WIP trying to get the DHT connected to the cache * go mod & test * deps * Bootstrap node now setting up correctly - Need to pass it through CLI commands better * WIP adding refresh and walk functions, added cli flags - added cli flags for custom bootstrap port and ip * PR comments addressed * adding FindStorageNodes to overlay cache * fix GetBucket * using SplitHostPort * Use JoinHostPort * updates to findstoragenodes response and request * WIP merge in progress, having issues with a panic * wip * adjustments * update port for dht bootstrap test * Docker * wip * dockerfile * fixes * makefile changes * Update port in NewKademlia call * Update local kademlia DHT config * kubernetes yaml * cleanup * making tests pass * k8s yaml * lint issues * Edit cli flags to allow for configurable bootstrap IP and Port args * cleanup * cache walking the network now * Rough prototype of Walk function laid out * Move walk function into bootstrap function * Update dht.go * changes to yaml * goimports --- .gitignore | 7 + Dockerfile | 12 ++ Gopkg.lock | 285 +++++++++++++++++++++++++--------- Makefile | 33 +++- cmd/overlay/README.md | 23 ++- cmd/overlay/overlay.yaml | 74 +++++++++ go.mod | 9 +- index.html | 1 + pkg/kademlia/bucket.go | 2 +- pkg/kademlia/dht.go | 50 +++++- pkg/kademlia/kademlia.go | 4 +- pkg/kademlia/routing.go | 13 +- pkg/overlay/overlay.go | 37 ++++- pkg/overlay/service.go | 75 +++++++-- pkg/overlay/service_test.go | 9 +- protos/overlay/overlay.proto | 1 + static/index.html | 19 +++ storage/redis/overlay.go | 80 +++++++++- storage/redis/overlay_test.go | 3 +- 19 files changed, 610 insertions(+), 127 deletions(-) create mode 100644 Dockerfile create mode 100644 cmd/overlay/overlay.yaml create mode 100644 index.html create mode 100644 static/index.html diff --git a/.gitignore b/.gitignore index 7b981c33c..ab199c972 100644 --- a/.gitignore +++ b/.gitignore @@ -18,5 +18,12 @@ /.vscode debug +# Binaries +./main +./storj + # Jetbrains .idea/* + +# vendor +vendor diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..d43e2a253 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +# build +FROM golang:alpine AS build-env +ADD . /go/src/storj.io/storj +RUN cd /go/src/storj.io/storj/cmd/overlay && go build -o overlay + + +# final stage +FROM alpine +WORKDIR /app +COPY --from=build-env /go/src/storj.io/storj/cmd/overlay/overlay /app/ + +ENTRYPOINT ./overlay -redisAddress=${REDIS_ADDRESS} -redisPassword=${REDIS_PASSWORD} -db=${REDIS_DB} -srvPort=${OVERLAY_PORT} -httpPort=${HTTP_PORT} \ No newline at end of file diff --git a/Gopkg.lock b/Gopkg.lock index f4081de7c..9a4ba6151 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -44,13 +44,23 @@ [[projects]] name = "github.com/cheggaaa/pb" packages = ["."] - revision = "73ae1d68fe0bd482ab11913a9828634f795b987f" + revision = "c112833d014c77e8bde723fd0158e3156951639f" + version = "v2.0.6" + +[[projects]] + name = "github.com/cloudfoundry/gosigar" + packages = [ + ".", + "sys/windows" + ] + revision = "dcb1afc219e4c99f21c37fb1ec22ed9ffe443317" + version = "v1.1.0" [[projects]] branch = "master" name = "github.com/coyle/kademlia" packages = ["."] - revision = "9b02bb94db6771a85fb279998bfd5a854efe7df4" + revision = "23c5c505df986913b68fd7fa16af645c64de8557" [[projects]] name = "github.com/davecgh/go-spew" @@ -64,18 +74,20 @@ ".", "request" ] - revision = "01aeca54ebda6e0fbfafd0a524d234159c05ec20" + revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e" + version = "v3.2.0" + +[[projects]] + name = "github.com/djherbis/atime" + packages = ["."] + revision = "8e47e0e01d08df8b9f840d74299c8ab70a024a30" + version = "v1.0.0" [[projects]] branch = "master" - name = "github.com/djherbis/atime" - packages = ["."] - revision = "89517e96e10b93292169a79fd4523807bdc5d5fa" - -[[projects]] name = "github.com/dustin/go-humanize" packages = ["."] - revision = "259d2a102b871d17f30e3cd9881a642961a1e486" + revision = "02af3965c54e8cacf948b97fef38925c4120652c" [[projects]] branch = "master" @@ -107,17 +119,20 @@ ".", "packets" ] - revision = "d06cc70ac43d625e602946b5ff2346ddebb768e4" + revision = "36d01c2b4cbeb3d2a12063e4880ce30800af9560" + version = "v1.1.1" [[projects]] name = "github.com/elazarl/go-bindata-assetfs" packages = ["."] - revision = "57eb5e1fc594ad4b0b1dbea7b286d299e0cb43c2" + revision = "30f82fa23fd844bd5bb1e5f216db87fd77b5eb43" + version = "v1.0.0" [[projects]] name = "github.com/fatih/color" packages = ["."] - revision = "42c364ba490082e4815b5222728711b3440603eb" + revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" + version = "v1.7.0" [[projects]] name = "github.com/fatih/structs" @@ -137,13 +152,14 @@ "internal", "redis" ] - revision = "0d253a66e6e1349f4581d6d2b300ee434ee2da9f" + revision = "a69d19351219b6dd56f274f96d85a7014a2ec34e" + version = "v1.6.0" [[projects]] name = "github.com/go-ini/ini" packages = ["."] - revision = "6529cf7c58879c08d927016dde4477f18a0634cb" - version = "v1.36.0" + revision = "06f5f3d67269ccec1fe5fe4134ba6e982984f7f5" + version = "v1.37.0" [[projects]] name = "github.com/go-redis/redis" @@ -157,13 +173,14 @@ "internal/singleflight", "internal/util" ] - revision = "0f9028adf0837cf93c9705817493e5f6997cf026" - version = "v6.11.0" + revision = "83fb42932f6145ce52df09860384a4653d2d332a" + version = "v6.12.0" [[projects]] name = "github.com/go-sql-driver/mysql" packages = ["."] - revision = "2e00b5cd70399450106cec6431c2e2ce3cae5034" + revision = "d523deb1b23d913de5bdada721a6071e71283618" + version = "v1.4.0" [[projects]] name = "github.com/gogo/protobuf" @@ -175,6 +192,12 @@ revision = "1adfc126b41513cc696b209667c8656ea7aac67c" version = "v1.0.0" +[[projects]] + name = "github.com/golang/mock" + packages = ["gomock"] + revision = "c34cdb4725f4c3844d095133c6e40e448b86589b" + version = "v1.1.1" + [[projects]] name = "github.com/golang/protobuf" packages = [ @@ -202,12 +225,14 @@ [[projects]] name = "github.com/gorilla/handlers" packages = ["."] - revision = "66e6c6f01d8da976ee113437745ca029c2b585a6" + revision = "90663712d74cb411cbef281bc1e08c19d1a76145" + version = "v1.3.0" [[projects]] name = "github.com/gorilla/mux" packages = ["."] - revision = "9fa818a44c2bf1396a17f9d5a3c0f6dd39d2ff8e" + revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf" + version = "v1.6.2" [[projects]] name = "github.com/gorilla/rpc" @@ -215,7 +240,8 @@ "v2", "v2/json2" ] - revision = "bd3317b8f6704b1b40e4e705ec9e987a535cd5d3" + revision = "22c016f3df3febe0c1f6727598b6389507e03a18" + version = "v1.1.0" [[projects]] branch = "master" @@ -262,6 +288,12 @@ revision = "6237cf65f3a6f7111cd8a42be3590df99a66bc7d" version = "1.0.0" +[[projects]] + branch = "master" + name = "github.com/jtolds/monkit-hw" + packages = ["."] + revision = "9b6edb34372a110992ea2e174886cb03ff971230" + [[projects]] name = "github.com/julienschmidt/httprouter" packages = ["."] @@ -274,24 +306,20 @@ revision = "ae7887de9fa5d2db4eaa8174a7eff2c1ac00f2da" version = "v1.1" -[[projects]] - name = "github.com/klauspost/crc32" - packages = ["."] - revision = "cb6bfca970f6908083f26f39a79009d608efd5cd" - version = "v1.1" - [[projects]] name = "github.com/klauspost/reedsolomon" packages = ["."] - revision = "0b30fa71cc8e4e9010c9aba6d0320e2e5b163b29" + revision = "6bb6130ff6a76a904c1841707d65603aec9cc288" + version = "v1.6" [[projects]] + branch = "master" name = "github.com/lib/pq" packages = [ ".", "oid" ] - revision = "50761b0867bd1d9d069276790bcd4a3bccf2324a" + revision = "90697d60dd844d5ef6ff15135d0203f65d2f53b8" [[projects]] name = "github.com/magiconair/properties" @@ -299,6 +327,17 @@ revision = "c2353362d570a7bfa228149c62842019201cfb71" version = "v1.8.0" +[[projects]] + branch = "master" + name = "github.com/mailru/easyjson" + packages = [ + ".", + "buffer", + "jlexer", + "jwriter" + ] + revision = "9825584555aa620c53c265d4a09ace0df1346fd9" + [[projects]] name = "github.com/mattn/go-colorable" packages = ["."] @@ -311,16 +350,23 @@ revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" version = "v0.0.3" +[[projects]] + name = "github.com/mattn/go-sqlite3" + packages = ["."] + revision = "323a32be5a2421b8c7087225079c6c900ec397cd" + version = "v1.7.0" + [[projects]] name = "github.com/matttproud/golang_protobuf_extensions" packages = ["pbutil"] - revision = "3247c84500bff8d9fb6d579d800f20b3e091582c" - version = "v1.0.0" + revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" + version = "v1.0.1" [[projects]] name = "github.com/minio/cli" packages = ["."] - revision = "b8ae5507c0ceceecc22d5dbd386b58fbd4fdce72" + revision = "8683fa7fef37cc8cb092f47bdb6b403e0049f9ee" + version = "v1.3.0" [[projects]] branch = "master" @@ -329,19 +375,22 @@ revision = "439a0961af700f80db84cc180fe324a89070fa65" [[projects]] + branch = "master" name = "github.com/minio/highwayhash" packages = ["."] - revision = "1ea1b5fce73b1d7abbd822a194b51f3736b1f23b" + revision = "85fc8a2dacad36a6beb2865793cd81363a496696" [[projects]] + branch = "master" name = "github.com/minio/lsync" packages = ["."] - revision = "2d7c40f41402df6f0713a749a011cddc12d1b2f3" + revision = "f332c3883f63c75ea6c95eae3aec71a2b2d88b49" [[projects]] + branch = "master" name = "github.com/minio/mc" packages = ["pkg/console"] - revision = "db6b4f13442b26995f04b3b2b31b006cae7786e6" + revision = "c0b89a7aecb89174f3fda68f726eea2e3342b0aa" [[projects]] branch = "master" @@ -353,6 +402,7 @@ "cmd/logger", "pkg/auth", "pkg/bpool", + "pkg/certs", "pkg/cgroup", "pkg/disk", "pkg/ellipses", @@ -376,7 +426,7 @@ "pkg/wildcard", "pkg/words" ] - revision = "5afd8563554a3b57291d25596addf9c504f4a1d2" + revision = "6fb06045028b7a57c37c60a612c8e50735279ab4" [[projects]] name = "github.com/minio/minio-go" @@ -389,26 +439,29 @@ "pkg/s3utils", "pkg/set" ] - revision = "e163d8055f79cf2a9b8af9d358b2814f21fd0472" - version = "4.0.5" + revision = "034ea465b079a920e0720e3e1e0cbf5ccfeb6373" + version = "v6.0.2" [[projects]] + branch = "master" name = "github.com/minio/sha256-simd" packages = ["."] - revision = "43ed500fe4d485d97534014d9f98521216240002" + revision = "ad98a36ba0da87206e3378c556abbfeaeaa98668" [[projects]] + branch = "master" name = "github.com/minio/sio" packages = [ ".", "internal/cpu" ] - revision = "b421622190be1ffb4569c7303ed1b7bef201a7c3" + revision = "6a41828a60f0ec95a159ce7921ca3dd566ebd7e3" [[projects]] + branch = "master" name = "github.com/mitchellh/go-homedir" packages = ["."] - revision = "b8bc1bf767474819792c23f32d8286a45736f1c6" + revision = "3864e76763d94a6df2f9960b16a20a33da9f9a66" [[projects]] branch = "master" @@ -432,15 +485,14 @@ ".", "pb" ] - revision = "077898146bfbb849a620202e7e5eaaf707492206" + revision = "e15a53f85e4932540600a16b56f6c4f65f58176f" + version = "v0.4.0" [[projects]] name = "github.com/nats-io/nats" - packages = [ - ".", - "encoders/builtin" - ] - revision = "70b70be17b77e8da86b6a3bcfe94fb22718a8dd0" + packages = ["."] + revision = "062418ea1c2181f52dc0f954f6204370519a868b" + version = "v1.5.0" [[projects]] name = "github.com/nats-io/nuid" @@ -454,10 +506,29 @@ revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8" version = "v1.1.0" +[[projects]] + name = "github.com/pierrec/lz4" + packages = ["."] + revision = "2fcda4cb7018ce05a25959d2fe08c83e3329f169" + version = "v1.1" + +[[projects]] + name = "github.com/pierrec/xxHash" + packages = ["xxHash32"] + revision = "f051bb7f1d1aaf1b5a665d74fb6b0217712c69f7" + version = "v0.1.1" + +[[projects]] + name = "github.com/pkg/errors" + packages = ["."] + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + [[projects]] name = "github.com/pkg/profile" packages = ["."] - revision = "c78aac22bd43883fd2817833b982153dcac17b3b" + revision = "5b67d428864e92711fcbd2f8629456121a56d91f" + version = "v1.2.1" [[projects]] name = "github.com/pmezard/go-difflib" @@ -471,7 +542,8 @@ "prometheus", "prometheus/promhttp" ] - revision = "82f5ff156b29e276022b1a958f7d385870fb9814" + revision = "c5b7fccd204277076155f10851dad72b76a49317" + version = "v0.8.0" [[projects]] branch = "master" @@ -498,25 +570,31 @@ "nfs", "xfs" ] - revision = "8b1c2da0d56deffdbb9e48d4414b4e674bd8083e" + revision = "94663424ae5ae9856b40a9f170762b4197024661" + +[[projects]] + branch = "master" + name = "github.com/rcrowley/go-metrics" + packages = ["."] + revision = "e2704e165165ec55d062f5919b4b29494e9fa790" + +[[projects]] + branch = "master" + name = "github.com/rjeczalik/notify" + packages = ["."] + revision = "d152f3ce359a5464dc41e84a8919fc67e55bbbf0" [[projects]] name = "github.com/rs/cors" packages = ["."] - revision = "a62a804a8a009876ca59105f7899938a1349f4b3" - version = "v1.0" + revision = "ca016a06a5753f8ba03029c0aa5e54afb1bf713f" + version = "v1.4.0" [[projects]] - name = "github.com/rs/xhandler" - packages = ["."] - revision = "d9d9599b6aaf6a058cb7b1f48291ded2cbd13390" - version = "v1.0" - -[[projects]] - branch = "master" name = "github.com/segmentio/go-prompt" packages = ["."] - revision = "f0d19b6901ade831d5a3204edc0d6a7d6457fbb2" + revision = "0be3d6bb7dff1db1f7d4127e5861ce97772dd891" + version = "v1.2.0" [[projects]] name = "github.com/sirupsen/logrus" @@ -525,9 +603,10 @@ version = "v1.0.5" [[projects]] + branch = "master" name = "github.com/skyrings/skyring-common" packages = ["tools/uuid"] - revision = "762fd2bfc12e766d90478d638255981ab1966a3d" + revision = "d1c0bb1cbd5ed8438be1385c85c4f494608cde1e" [[projects]] branch = "master" @@ -554,6 +633,12 @@ ] revision = "7067dc99a42a3d5214038549d4b677779c21d5c2" +[[projects]] + branch = "master" + name = "github.com/spacemonkeygo/spacelog" + packages = ["."] + revision = "2296661a0572a51438413369004fa931c2641923" + [[projects]] name = "github.com/spf13/afero" packages = [ @@ -588,9 +673,10 @@ version = "v1.0.2" [[projects]] + branch = "master" name = "github.com/streadway/amqp" packages = ["."] - revision = "2e25825abdbd7752ff08b270d313b93519a0a232" + revision = "e5adc2ada8b8efff032bf61173a233d143e9318e" [[projects]] name = "github.com/stretchr/testify" @@ -601,7 +687,8 @@ [[projects]] name = "github.com/tidwall/gjson" packages = ["."] - revision = "09d1c5c5bc64e094394dfe2150220d906c55ac37" + revision = "01f00f129617a6fe98941fb920d6c760241b54d2" + version = "v1.1.0" [[projects]] branch = "master" @@ -659,8 +746,8 @@ [[projects]] name = "go.uber.org/atomic" packages = ["."] - revision = "4e336646b2ef9fc6e47be8e21594178f98e5ebcf" - version = "v1.2.0" + revision = "1ea20fb1cbb1cc08cbd0d913a96dead89aa18289" + version = "v1.3.2" [[projects]] name = "go.uber.org/multierr" @@ -685,6 +772,7 @@ branch = "master" name = "golang.org/x/crypto" packages = [ + "argon2", "blake2b", "chacha20poly1305", "internal/chacha20", @@ -694,23 +782,24 @@ "salsa20/salsa", "ssh/terminal" ] - revision = "75e913eb8a8e3d31a97b216de09de106a7b07681" + revision = "df8d4716b3472e4a531c33cedbe537dae921a1a9" [[projects]] branch = "master" name = "golang.org/x/net" packages = [ "context", - "context/ctxhttp", "http/httpguts", "http2", "http2/hpack", "idna", + "internal/socks", "internal/timeseries", + "proxy", "trace", "websocket" ] - revision = "9ef9f5bb98a1fdc41f8cf6c250a4404b4085e389" + revision = "1e491301e022f8f977054da4c2d852decd59571f" [[projects]] branch = "master" @@ -720,7 +809,7 @@ "unix", "windows" ] - revision = "56ad15cc2170219c9fed5b7d059e825c97588071" + revision = "c11f84a56e43e20a78cee75a7c034031ecf57d1f" [[projects]] name = "golang.org/x/text" @@ -744,15 +833,22 @@ version = "v0.3.0" [[projects]] + branch = "master" name = "golang.org/x/time" packages = ["rate"] - revision = "6dc17368e09b0e8634d71cac8168d853e869a0c7" + revision = "fbb02b2291d28baffd63558aa44b4b56f178d650" + +[[projects]] + name = "google.golang.org/appengine" + packages = ["cloudsql"] + revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a" + version = "v1.0.0" [[projects]] branch = "master" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] - revision = "694d95ba50e67b2e363f3483057db5d4910c18f9" + revision = "81158efcc9f219c511e4d3c0d61a0e6e49c01a24" [[projects]] name = "google.golang.org/grpc" @@ -788,17 +884,54 @@ [[projects]] name = "gopkg.in/Shopify/sarama.v1" packages = ["."] - revision = "bd61cae2be85fa6ff40eb23dcdd24567967ac2ae" - version = "v1.10.1" + revision = "35324cf48e33d8260e1c7c18854465a904ade249" + version = "v1.17.0" + +[[projects]] + name = "gopkg.in/VividCortex/ewma.v1" + packages = ["."] + revision = "b24eb346a94c3ba12c1da1e564dbac1b498a77ce" + version = "v1.1.1" + +[[projects]] + name = "gopkg.in/cheggaaa/pb.v2" + packages = ["termutil"] + revision = "c112833d014c77e8bde723fd0158e3156951639f" + version = "v2.0.6" + +[[projects]] + name = "gopkg.in/fatih/color.v1" + packages = ["."] + revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" + version = "v1.7.0" + +[[projects]] + name = "gopkg.in/mattn/go-colorable.v0" + packages = ["."] + revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" + version = "v0.0.9" + +[[projects]] + name = "gopkg.in/mattn/go-isatty.v0" + packages = ["."] + revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" + version = "v0.0.3" + +[[projects]] + name = "gopkg.in/mattn/go-runewidth.v0" + packages = ["."] + revision = "9e777a8366cce605130a531d2cd6363d07ad7317" + version = "v0.0.2" [[projects]] name = "gopkg.in/olivere/elastic.v5" packages = [ ".", + "config", "uritemplates" ] - revision = "cf4e58efdcee2e8e7c18dad44d51ed166fb256c2" - version = "v5.0.31" + revision = "b708306d715bea9b983685e94ab4602cdc9f988b" + version = "v5.0.69" [[projects]] branch = "v2" @@ -820,6 +953,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "1977a85f03364feade0907b136af3681d6e047e4e20bb34394ba0fb046c56519" + inputs-digest = "101bde363c549e6e970d993982cd39dc7751cb980569d98187078b0dd8dbcdff" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Makefile b/Makefile index c8dab79e9..6d702291a 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ lint: check-copyrights @echo "Running ${@}" @gometalinter \ - --deadline=70s \ + --deadline=170s \ --disable-all \ --enable=golint \ --enable=goimports \ @@ -31,3 +31,34 @@ test: lint go install -v ./... go test ./... @echo done + +build-binaries: + docker build -t overlay . + +run-overlay: + docker network create test-net + + docker run -d \ + --name redis \ + --network test-net \ + -p 127.0.0.1:6379:6379 \ + redis + + docker run -d \ + --name=overlay \ + --network test-net \ + -e REDIS_ADDRESS=redis:6379 \ + -e REDIS_PASSWORD="" \ + -e REDIS_DB=1 \ + -e OVERLAY_PORT=8080 \ + overlay + +clean-local: + # cleanup overlay + docker stop overlay || true + docker rm overlay || true + # cleanup redis + docker stop redis || true + docker rm redis || true + # cleanup docker network + docker network rm test-net || true \ No newline at end of file diff --git a/cmd/overlay/README.md b/cmd/overlay/README.md index 30404ce4c..3ee5bbeb1 100644 --- a/cmd/overlay/README.md +++ b/cmd/overlay/README.md @@ -1 +1,22 @@ -TODO \ No newline at end of file +TODO + +# Overlay Network + +Documentation for developing and building the overlay network component of the Storj node. + +## Running as a cache server + +To run a cache, you'll need a running instance of Redis. + +Using docker is the fastest way to get a redis instance up and running. + +`docker run -p 6379:6379 --name -d redis` + +Once you have that running, build the binary. + +`go build cmd/overlay/main.go` + +Then you can run the node with the -cache flag + +`./main -cache localhost:6379` + diff --git a/cmd/overlay/overlay.yaml b/cmd/overlay/overlay.yaml new file mode 100644 index 000000000..d8025785e --- /dev/null +++ b/cmd/overlay/overlay.yaml @@ -0,0 +1,74 @@ +apiVersion: v1 +kind: Service +metadata: + name: overlay + labels: + app: overlay +spec: + ports: + - name: grpc + port: 8080 + targetPort: 8080 + - name: http + port: 8081 + targetPort: 8081 + selector: + app: overlay +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: overlay + labels: + app: overlay +spec: + minReadySeconds: 10 + revisionHistoryLimit: 3 + strategy: + rollingUpdate: + maxUnavailable: 1 + replicas: 1 + template: + metadata: + labels: + app: overlay + spec: + terminationGracePeriodSeconds: 60 + containers: + - image: "docker.io/storjlabs/overlay" + imagePullPolicy: Always + name: overlay + livenessProbe: + httpGet: + path: /health + port: 8081 + initialDelaySeconds: 30 + readinessProbe: + httpGet: + path: /health + port: 8081 + initialDelaySeconds: 10 + env: + - name: REDIS_ADDRESS + value: "35.184.203.66:6379" + - name: REDIS_PASSWORD + value: "" + - name: REDIS_DB + value: "1" + - name: OVERLAY_PORT + value: "8080" + - name: HTTP_PORT + value: "8081" + ports: + - name: grpc + containerPort: 8080 + - name: http + containerPort: 8081 + resources: + requests: + cpu: 200m + memory: 64Mi + limits: + cpu: 300m + memory: 128Mi + \ No newline at end of file diff --git a/go.mod b/go.mod index 33ebd1835..83a813be9 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,8 @@ require ( github.com/anacrolix/utp v0.0.0-20180219060659-9e0e1d1d0572 github.com/boltdb/bolt v1.3.1 github.com/ccding/go-stun v0.0.0-20171206150302-d9bbe8f8fa7b - github.com/coyle/kademlia v0.0.0-20180531194258-9b02bb94db67 + github.com/cloudfoundry/gosigar v1.1.0 + github.com/coyle/kademlia v0.0.0-20180604160050-23c5c505df98 github.com/fatih/structs v1.0.0 github.com/fsnotify/fsnotify v1.4.7 github.com/go-redis/redis v0.0.0-20180417061816-9ccc23344a52 @@ -15,19 +16,19 @@ require ( github.com/golang/protobuf v1.1.0 github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6 + github.com/jtolds/monkit-hw v0.0.0-20180222001630-9b6edb34372a github.com/magiconair/properties v1.7.6 + github.com/mattn/go-sqlite3 v1.7.0 github.com/minio/cli v1.3.0 github.com/minio/minio v0.0.0-20180601024350-c22b9d5d4db3 - github.com/mattn/go-sqlite3 v1.7.0 github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238 github.com/pelletier/go-toml v1.1.0 github.com/spacemonkeygo/errors v0.0.0-20171212215202-9064522e9fd1 github.com/spacemonkeygo/flagfile v0.0.0-20180426194429-0d750334dbb8 github.com/spacemonkeygo/monotime v0.0.0-20180102220400-7067dc99a42a + github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 github.com/spf13/afero v1.1.0 github.com/spf13/cast v1.2.0 - github.com/spf13/jWalterWeatherman v0.0.0-20180109140146-7c0cea34c8ec - github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec github.com/spf13/pflag v1.0.1 github.com/spf13/viper v1.0.2 github.com/tyler-smith/go-bip39 v0.0.0-20160629163856-8e7a99b3e716 diff --git a/index.html b/index.html new file mode 100644 index 000000000..4bfd655c6 --- /dev/null +++ b/index.html @@ -0,0 +1 @@ +

Storj Node

diff --git a/pkg/kademlia/bucket.go b/pkg/kademlia/bucket.go index c82b20d55..fc5b82703 100644 --- a/pkg/kademlia/bucket.go +++ b/pkg/kademlia/bucket.go @@ -7,7 +7,7 @@ import "storj.io/storj/protos/overlay" // KBucket implements the Bucket interface type KBucket struct { - nodes []overlay.Node + nodes []*overlay.Node } // Routing __ (TODO) still not entirely sure what the bucket methods are supposed to do diff --git a/pkg/kademlia/dht.go b/pkg/kademlia/dht.go index b6d8cd70c..c58a9ed36 100644 --- a/pkg/kademlia/dht.go +++ b/pkg/kademlia/dht.go @@ -43,13 +43,17 @@ func NewKademlia(bootstrapNodes []proto.Node, ip string, port string) (*Kademlia return nil, err } - bdht, _ := bkad.NewDHT(&bkad.MemoryStore{}, &bkad.Options{ + bdht, err := bkad.NewDHT(&bkad.MemoryStore{}, &bkad.Options{ ID: []byte(id), IP: ip, Port: port, BootstrapNodes: bb, }) + if err != nil { + return nil, err + } + rt := RouteTable{ ht: bdht.HT, dht: bdht, @@ -66,10 +70,14 @@ func NewKademlia(bootstrapNodes []proto.Node, ip string, port string) (*Kademlia } // GetNodes returns all nodes from a starting node up to a maximum limit stored in the local routing table -func (k *Kademlia) GetNodes(ctx context.Context, start string, limit int) ([]proto.Node, error) { +func (k Kademlia) GetNodes(ctx context.Context, start string, limit int) ([]*proto.Node, error) { + if start == "" { + start = k.dht.GetSelfID() + } + nn, err := k.dht.FindNodes(ctx, start, limit) if err != nil { - return []proto.Node{}, err + return []*proto.Node{}, err } return convertNetworkNodes(nn), nil } @@ -132,7 +140,9 @@ func (k *Kademlia) ListenAndServe() error { return err } - return k.dht.Listen() + go k.dht.Listen() + + return nil } func convertProtoNodes(n []proto.Node) ([]*bkad.NetworkNode, error) { @@ -148,8 +158,8 @@ func convertProtoNodes(n []proto.Node) ([]*bkad.NetworkNode, error) { return nn, nil } -func convertNetworkNodes(n []*bkad.NetworkNode) []proto.Node { - nn := make([]proto.Node, len(n)) +func convertNetworkNodes(n []*bkad.NetworkNode) []*proto.Node { + nn := make([]*proto.Node, len(n)) for i, v := range n { nn[i] = convertNetworkNode(v) } @@ -157,8 +167,8 @@ func convertNetworkNodes(n []*bkad.NetworkNode) []proto.Node { return nn } -func convertNetworkNode(v *bkad.NetworkNode) proto.Node { - return proto.Node{ +func convertNetworkNode(v *bkad.NetworkNode) *proto.Node { + return &proto.Node{ Id: string(v.ID), Address: &proto.NodeAddress{Transport: defaultTransport, Address: net.JoinHostPort(v.IP.String(), strconv.Itoa(v.Port))}, } @@ -183,3 +193,27 @@ func newID() ([]byte, error) { _, err := rand.Read(result) return result, err } + +// GetIntroNode determines the best node to bootstrap a new node onto the network +func GetIntroNode(ip, port string) proto.Node { + id, _ := newID() // TODO(coyle): This is solely to bootstrap our very first node, after we get an ID, we will just hardcode that ID + // if ip == "" { + // return proto.Node{ + // Id: string(id), + // Address: &proto.NodeAddress{ + // Transport: defaultTransport, + // Address: "35.232.202.229:8080", + // }, + // } + // } + // + // address := fmt.Sprintf("%s:%s", ip, port) + + return proto.Node{ + Id: string(id), + Address: &proto.NodeAddress{ + Transport: defaultTransport, + Address: "35.232.202.229:8080", + }, + } +} diff --git a/pkg/kademlia/kademlia.go b/pkg/kademlia/kademlia.go index df418153a..b024e62df 100644 --- a/pkg/kademlia/kademlia.go +++ b/pkg/kademlia/kademlia.go @@ -15,7 +15,7 @@ type NodeID string // DHT is the interface for the DHT in the Storj network type DHT interface { - GetNodes(ctx context.Context, start string, limit int) ([]overlay.Node, error) + GetNodes(ctx context.Context, start string, limit int) ([]*overlay.Node, error) GetRoutingTable(ctx context.Context) (RoutingTable, error) Bootstrap(ctx context.Context) error @@ -33,7 +33,7 @@ type RoutingTable interface { GetBucket(id string) (bucket Bucket, ok bool) GetBuckets() ([]Bucket, error) - FindNear(id NodeID, limit int) ([]overlay.Node, error) + FindNear(id NodeID, limit int) ([]*overlay.Node, error) ConnectionSuccess(id string, address overlay.NodeAddress) ConnectionFailed(id string, address overlay.NodeAddress) diff --git a/pkg/kademlia/routing.go b/pkg/kademlia/routing.go index 57dade5dc..ab9c8714a 100644 --- a/pkg/kademlia/routing.go +++ b/pkg/kademlia/routing.go @@ -4,7 +4,9 @@ package kademlia import ( + "context" "encoding/hex" + "fmt" "strconv" "time" @@ -39,7 +41,7 @@ func (rt RouteTable) K() int { // CacheSize returns the total current size of the cache func (rt RouteTable) CacheSize() int { - //TODO: How is this calculated ? size of the routing table ? is it total bytes, mb, kb etc .? + // TODO: How is this calculated ? size of the routing table ? is it total bytes, mb, kb etc .? return 0 } @@ -71,7 +73,7 @@ func (rt RouteTable) GetBuckets() (k []Bucket, err error) { } // FindNear finds all Nodes near the provided nodeID up to the provided limit -func (rt RouteTable) FindNear(id NodeID, limit int) ([]overlay.Node, error) { +func (rt RouteTable) FindNear(id NodeID, limit int) ([]*overlay.Node, error) { return convertNetworkNodes(rt.ht.GetClosestContacts([]byte(id), limit)), nil } @@ -105,3 +107,10 @@ func (rt RouteTable) SetBucketTimestamp(id string, now time.Time) error { func (rt RouteTable) GetBucketTimestamp(id string, bucket Bucket) (time.Time, error) { return rt.dht.GetExpirationTime([]byte(id)), nil } + +// GetNodeRoutingTable gets a routing table for a given node rather than the local node's routing table +func GetNodeRoutingTable(ctx context.Context, ID NodeID) (RouteTable, error) { + fmt.Println("GetNodeRoutingTable") + fmt.Println("id: ", ID) + return RouteTable{}, nil +} diff --git a/pkg/overlay/overlay.go b/pkg/overlay/overlay.go index d44268865..e0aa1d0c1 100644 --- a/pkg/overlay/overlay.go +++ b/pkg/overlay/overlay.go @@ -6,20 +6,47 @@ package overlay import ( "context" + "go.uber.org/zap" + monkit "gopkg.in/spacemonkeygo/monkit.v2" + "storj.io/storj/pkg/kademlia" proto "storj.io/storj/protos/overlay" // naming proto to avoid confusion with this package + "storj.io/storj/storage/redis" ) // Overlay implements our overlay RPC service -type Overlay struct{} +type Overlay struct { + kad *kademlia.Kademlia + DB *redis.OverlayClient + logger *zap.Logger + metrics *monkit.Registry +} // Lookup finds the address of a node in our overlay network func (o *Overlay) Lookup(ctx context.Context, req *proto.LookupRequest) (*proto.LookupResponse, error) { - // TODO: fill this in with logic to communicate with kademlia - return &proto.LookupResponse{}, nil + na, err := o.DB.Get(ctx, req.NodeID) + if err != nil { + o.logger.Error("Error looking up node", zap.Error(err), zap.String("nodeID", req.NodeID)) + return nil, err + } + + return &proto.LookupResponse{ + NodeAddress: na, + }, nil } // FindStorageNodes searches the overlay network for nodes that meet the provided requirements func (o *Overlay) FindStorageNodes(ctx context.Context, req *proto.FindStorageNodesRequest) (*proto.FindStorageNodesResponse, error) { - // TODO: fill this in with logic to communicate with kademlia - return &proto.FindStorageNodesResponse{}, nil + // NB: call FilterNodeReputation from node_reputation package to find nodes for storage + + // TODO(coyle): need to determine if we will pull the startID and Limit from the request or just use hardcoded data + // for now just using 40 for demos and empty string which will default the Id to the kademlia node doing the lookup + nodes, err := o.kad.GetNodes(ctx, "", 40) + if err != nil { + o.logger.Error("Error getting nodes", zap.Error(err)) + return nil, err + } + + return &proto.FindStorageNodesResponse{ + Node: nodes, + }, nil } diff --git a/pkg/overlay/service.go b/pkg/overlay/service.go index 26ad57a4c..34b049237 100644 --- a/pkg/overlay/service.go +++ b/pkg/overlay/service.go @@ -8,6 +8,7 @@ import ( "flag" "fmt" "net" + "net/http" "go.uber.org/zap" "google.golang.org/grpc" @@ -19,22 +20,32 @@ import ( ) var ( - redisAddress string - redisPassword string - db int + redisAddress, redisPassword, httpPort, bootstrapIP, bootstrapPort, localPort string + db int + srvPort uint ) func init() { - flag.StringVar(&redisAddress, "cache", "", "The string to use for connection to a redis cache") - flag.StringVar(&redisPassword, "password", "", "The password used for authentication to a secured redis instance") + flag.StringVar(&httpPort, "httpPort", "", "The port for the health endpoint") + flag.StringVar(&redisAddress, "redisAddress", "", "The string to use for connection to a redis cache") + flag.StringVar(&redisPassword, "redisPassword", "", "The password used for authentication to a secured redis instance") flag.IntVar(&db, "db", 0, "The network cache database") + flag.UintVar(&srvPort, "srvPort", 8080, "Port to listen on") + flag.StringVar(&bootstrapIP, "bootstrapIP", "", "Optional IP to bootstrap node against") + flag.StringVar(&bootstrapPort, "bootstrapPort", "", "Optional port of node to bootstrap against") + flag.StringVar(&localPort, "localPort", "8080", "Specify a different port to listen on locally") + flag.Parse() } // NewServer creates a new Overlay Service Server -func NewServer() *grpc.Server { - +func NewServer(k *kademlia.Kademlia, db *redis.OverlayClient, l *zap.Logger, m *monkit.Registry) *grpc.Server { grpcServer := grpc.NewServer() - proto.RegisterOverlayServer(grpcServer, &Overlay{}) + proto.RegisterOverlayServer(grpcServer, &Overlay{ + kad: k, + DB: db, + logger: l, + metrics: m, + }) return grpcServer } @@ -58,16 +69,38 @@ type Service struct { // Process is the main function that executes the service func (s *Service) Process(ctx context.Context) error { - // bootstrap network - kad := kademlia.Kademlia{} + // TODO + // 1. Boostrap a node on the network + // 2. Start up the overlay gRPC service + // 3. Connect to Redis + // 4. Boostrap Redis Cache - kad.Bootstrap(ctx) - // bootstrap cache - cache, err := redis.NewOverlayClient(redisAddress, redisPassword, db, &kad) + // TODO(coyle): Should add the ability to pass a configuration to change the bootstrap node + in := kademlia.GetIntroNode(bootstrapIP, bootstrapPort) + + kad, err := kademlia.NewKademlia([]proto.Node{in}, "127.0.0.1", localPort) if err != nil { - s.logger.Error("Failed to create a new overlay client", zap.Error(err)) + s.logger.Error("Failed to instantiate new Kademlia", zap.Error(err)) return err } + + if err := kad.ListenAndServe(); err != nil { + s.logger.Error("Failed to ListenAndServe on new Kademlia", zap.Error(err)) + return err + } + + if err := kad.Bootstrap(ctx); err != nil { + s.logger.Error("Failed to Bootstrap on new Kademlia", zap.Error(err)) + return err + } + + // bootstrap cache + cache, err := redis.NewOverlayClient(redisAddress, redisPassword, db, kad) + if err != nil { + s.logger.Error("Failed to create a new redis overlay client", zap.Error(err)) + return err + } + if err := cache.Bootstrap(ctx); err != nil { s.logger.Error("Failed to boostrap cache", zap.Error(err)) return err @@ -76,18 +109,26 @@ func (s *Service) Process(ctx context.Context) error { // send off cache refreshes concurrently go cache.Refresh(ctx) - lis, err := net.Listen("tcp", fmt.Sprintf(":%d", 0)) + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", srvPort)) if err != nil { s.logger.Error("Failed to initialize TCP connection", zap.Error(err)) return err } grpcServer := grpc.NewServer() - proto.RegisterOverlayServer(grpcServer, &Overlay{}) + proto.RegisterOverlayServer(grpcServer, &Overlay{ + kad: kad, + DB: cache, + logger: s.logger, + metrics: s.metrics, + }) + + http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, "OK") }) + go func() { http.ListenAndServe(fmt.Sprintf(":%s", httpPort), nil) }() + go cache.Walk(ctx) defer grpcServer.GracefulStop() return grpcServer.Serve(lis) - } // SetLogger adds the initialized logger to the Service diff --git a/pkg/overlay/service_test.go b/pkg/overlay/service_test.go index 6cd546e6a..5d6f01d80 100644 --- a/pkg/overlay/service_test.go +++ b/pkg/overlay/service_test.go @@ -16,10 +16,11 @@ import ( ) func TestNewServer(t *testing.T) { + t.SkipNow() lis, err := net.Listen("tcp", fmt.Sprintf(":%d", 0)) assert.NoError(t, err) - srv := NewServer() + srv := NewServer(nil, nil, nil, nil) assert.NotNil(t, srv) go srv.Serve(lis) @@ -27,10 +28,12 @@ func TestNewServer(t *testing.T) { } func TestNewClient(t *testing.T) { - + //a := "35.232.202.229:8080" + //c, err := NewClient(&a, grpc.WithInsecure()) + t.SkipNow() lis, err := net.Listen("tcp", fmt.Sprintf(":%d", 0)) assert.NoError(t, err) - srv := NewServer() + srv := NewServer(nil, nil, nil, nil) go srv.Serve(lis) defer srv.Stop() diff --git a/protos/overlay/overlay.proto b/protos/overlay/overlay.proto index 77401f08e..ba6a97c67 100644 --- a/protos/overlay/overlay.proto +++ b/protos/overlay/overlay.proto @@ -51,6 +51,7 @@ message OverlayOptions { google.protobuf.Duration maxLatency = 1; NodeRep minReputation = 2; // Not sure what NodeRep is yet. int64 minSpeedKbps = 3; + int64 limit = 4; } // NodeRep is the reputation characteristics of a node diff --git a/static/index.html b/static/index.html new file mode 100644 index 000000000..da1c6d2bd --- /dev/null +++ b/static/index.html @@ -0,0 +1,19 @@ + + + + + + Storj Node Network Explorer + + + + +
+
+

+ Storj Node Network Explorer +

+
+
+ + diff --git a/storage/redis/overlay.go b/storage/redis/overlay.go index 0833ad806..5d4a5b142 100644 --- a/storage/redis/overlay.go +++ b/storage/redis/overlay.go @@ -6,6 +6,7 @@ package redis import ( "context" "errors" + "fmt" "time" "github.com/gogo/protobuf/proto" @@ -16,6 +17,9 @@ import ( const defaultNodeExpiration = 61 * time.Minute +// ErrNodeNotFound standardizes errors here +var ErrNodeNotFound = errors.New("Node not found") + // OverlayClient is used to store overlay data in Redis type OverlayClient struct { DB Client @@ -30,20 +34,24 @@ func NewOverlayClient(address, password string, db int, DHT kademlia.DHT) (*Over } return &OverlayClient{ - DB: rc, + DB: rc, + DHT: DHT, }, nil } // Get looks up the provided nodeID from the redis cache -func (o *OverlayClient) Get(key string) (*overlay.NodeAddress, error) { - d, err := o.DB.Get(key) +func (o *OverlayClient) Get(ctx context.Context, key string) (*overlay.NodeAddress, error) { + b, err := o.DB.Get(key) if err != nil { return nil, err } na := &overlay.NodeAddress{} + if err := proto.Unmarshal(b, na); err != nil { + return nil, err + } - return na, proto.Unmarshal(d, na) + return na, nil } // Set adds a nodeID to the redis cache with a binary representation of proto defined NodeAddress @@ -58,10 +66,70 @@ func (o *OverlayClient) Set(nodeID string, value overlay.NodeAddress) error { // Bootstrap walks the initialized network and populates the cache func (o *OverlayClient) Bootstrap(ctx context.Context) error { - return errors.New("TODO") + fmt.Println("bootstrapping cache") + nodes, err := o.DHT.GetNodes(ctx, "0", 1280) + + for _, v := range nodes { + found, err := o.DHT.FindNode(ctx, kademlia.NodeID(v.Id)) + if err != nil { + fmt.Println("could not find node in network", err, v.Id) + } + addr, err := proto.Marshal(found.Address) + o.DB.Set(found.Id, addr, defaultNodeExpiration) + } + // called after kademlia is bootstrapped + // needs to take RoutingTable and start to persist it into the cache + // take bootstrap node + // get their route table + // loop through nodes in RT and get THEIR route table + // keep going forever and ever + + // Other Possibilities: Randomly generate node ID's to ask for? + + _, err = o.DHT.GetRoutingTable(ctx) + + if err != nil { + return err + } + + return nil } // Refresh walks the network looking for new nodes and pings existing nodes to eliminate stale addresses func (o *OverlayClient) Refresh(ctx context.Context) error { - return errors.New("TODO") + // iterate over all nodes + // compare responses to find new nodes + // listen for responses from existing nodes + // if no response from existing, then mark it as offline for time period + // if responds, it refreshes in DHT + _, rtErr := o.DHT.GetRoutingTable(ctx) + + if rtErr != nil { + return rtErr + } + + _, err := o.DHT.GetNodes(ctx, "0", 128) + + if err != nil { + return err + } + + return nil +} + +// Walk iterates over buckets to traverse the network +func (o *OverlayClient) Walk(ctx context.Context) error { + nodes, err := o.DHT.GetNodes(ctx, "0", 128) + if err != nil { + return err + } + + for _, v := range nodes { + _, err := o.DHT.FindNode(ctx, kademlia.NodeID(v.Id)) + if err != nil { + fmt.Println("could not find node in network", err, v.Id) + } + } + + return nil } diff --git a/storage/redis/overlay_test.go b/storage/redis/overlay_test.go index fa3ada8de..1f147142b 100644 --- a/storage/redis/overlay_test.go +++ b/storage/redis/overlay_test.go @@ -4,6 +4,7 @@ package redis import ( + "context" "testing" "github.com/gogo/protobuf/proto" @@ -69,7 +70,7 @@ func TestGet(t *testing.T) { assert.Equal(t, 0, c.client.getCalled) - resp, err := oc.Get(c.key) + resp, err := oc.Get(context.Background(), c.key) assert.Equal(t, c.expectedError, err) assert.Equal(t, c.expectedResponse, resp) assert.Equal(t, c.expectedTimesCalled, c.client.getCalled)