diff --git a/.travis.yml b/.travis.yml index e50a49c74..2220f8cd8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ cache: services: - redis + - postgresql before_script: # Add an IPv6 config - see the corresponding Travis issue @@ -19,7 +20,8 @@ before_script: - if [ "${TRAVIS_OS_NAME}" == "linux" ]; then sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6'; sudo sh -c 'echo "\n::1 localhost\n" >> /etc/hosts'; - fi + psql -c 'create database pointerdb' -U postgres; + fi; before_install: - source scripts/setup-gopath.sh @@ -28,7 +30,9 @@ matrix: - os: windows # allow failures on windows because it's slow include: ### tests ### - - env: MODE=tests + - env: + - MODE=tests + - STORJ_POSTGRESKV_TEST=postgres://postgres@localhost/pointerdb?sslmode=disable install: - pushd ~ - GOBIN=${GOPATH}/bin GOPATH=~/gotools go get github.com/mattn/goveralls @@ -70,3 +74,6 @@ matrix: - goveralls -coverprofile=.coverprofile -service=travis-ci fast_finish: true + +addons: + postgresql: "9.5" diff --git a/docker-compose.yaml b/docker-compose.yaml index 96f0e0ef7..2bdfb6d64 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -5,8 +5,15 @@ services: context: . dockerfile: test/Dockerfile network_mode: service:test-redis + depends_on: + - test-postgres-pointerdb test-redis: image: redis + test-postgres-pointerdb: + image: postgres + environment: + - POSTGRES_USER=pointerdb + - POSTGRES_PASSWORD=pg-secret-pass storagenode: image: storjlabs/storagenode:${VERSION} diff --git a/go.mod b/go.mod index a752ea557..b75226cfa 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 // indirect github.com/alicebob/miniredis v0.0.0-20180911162847-3657542c8629 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect - github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect github.com/boltdb/bolt v1.3.1 github.com/cheggaaa/pb v1.0.5-0.20160713104425-73ae1d68fe0b github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect @@ -20,19 +19,15 @@ require ( github.com/elazarl/go-bindata-assetfs v1.0.0 // indirect github.com/fatih/color v1.7.0 // indirect github.com/fatih/structs v1.0.0 // indirect - github.com/go-ini/ini v1.38.2 // indirect + github.com/go-bindata/go-bindata v1.0.0 github.com/go-redis/redis v6.14.1+incompatible - github.com/go-sql-driver/mysql v1.4.0 // indirect github.com/gogo/protobuf v1.1.1 + github.com/golang-migrate/migrate/v3 v3.5.2 github.com/golang/mock v1.1.1 github.com/golang/protobuf v1.2.0 - github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect github.com/gomodule/redigo v2.0.0+incompatible // indirect github.com/google/go-cmp v0.2.0 - github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c // indirect - github.com/gorilla/context v1.1.1 // indirect github.com/gorilla/handlers v1.4.0 // indirect - github.com/gorilla/mux v1.6.2 // indirect github.com/gorilla/rpc v1.1.0 // indirect github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 github.com/hashicorp/go-immutable-radix v1.0.0 // indirect @@ -41,19 +36,16 @@ require ( github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c // indirect github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6 - github.com/jtolds/gls v4.2.1+incompatible // indirect github.com/jtolds/monkit-hw v0.0.0-20180827162413-5a254051f35d github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e // indirect github.com/klauspost/reedsolomon v0.0.0-20180704173009-925cb01d6510 // indirect - github.com/kr/pretty v0.1.0 // indirect - github.com/lib/pq v0.0.0-20180523175426-90697d60dd84 // indirect + github.com/lib/pq v1.0.0 github.com/loov/hrtime v0.0.0-20180911122900-a9e82bc6c180 github.com/loov/plot v0.0.0-20180510142208-e59891ae1271 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 // indirect github.com/mattn/go-isatty v0.0.4 // indirect github.com/mattn/go-runewidth v0.0.3 // indirect github.com/mattn/go-sqlite3 v1.9.0 - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/minio/cli v1.3.0 github.com/minio/dsync v0.0.0-20180124070302-439a0961af70 // indirect github.com/minio/highwayhash v0.0.0-20180501080913-85fc8a2dacad // indirect @@ -70,7 +62,6 @@ require ( github.com/nats-io/nats v1.6.0 // indirect github.com/nats-io/nats-streaming-server v0.11.0 // indirect github.com/nats-io/nuid v1.0.0 // indirect - github.com/onsi/gomega v1.4.2 // indirect github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c // indirect github.com/pierrec/lz4 v2.0.5+incompatible // indirect github.com/pkg/profile v1.2.1 // indirect @@ -79,8 +70,6 @@ require ( github.com/rs/cors v1.5.0 // indirect github.com/shirou/gopsutil v2.17.12+incompatible github.com/skyrings/skyring-common v0.0.0-20160929130248-d1c0bb1cbd5e // indirect - github.com/smartystreets/assertions v0.0.0-20180820201707-7c9eb446e3cf // indirect - github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a // indirect github.com/spacemonkeygo/errors v0.0.0-20171212215202-9064522e9fd1 // indirect github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 @@ -104,9 +93,7 @@ require ( golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect google.golang.org/grpc v1.15.0 gopkg.in/Shopify/sarama.v1 v1.18.0 // indirect - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.25 // indirect - gopkg.in/ini.v1 v1.38.2 // indirect gopkg.in/olivere/elastic.v5 v5.0.76 // indirect gopkg.in/spacemonkeygo/monkit.v2 v2.0.0-20180827161543-6ebf5a752f9b gopkg.in/vmihailenco/msgpack.v2 v2.9.1 // indirect @@ -122,12 +109,8 @@ require ( github.com/minio/minio v0.0.0-20180508161510-54cd29b51c38 github.com/mitchellh/mapstructure v1.1.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v0.9.0-pre1.0.20180416233856-82f5ff156b29 // indirect - github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5 // indirect - github.com/prometheus/common v0.0.0-20180326160409-38c53a9f4bfc // indirect - github.com/prometheus/procfs v0.0.0-20180408092902-8b1c2da0d56d // indirect github.com/segmentio/go-prompt v1.2.1-0.20161017233205-f0d19b6901ad // indirect ) diff --git a/go.sum b/go.sum index d031933a0..4a0b1ab84 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,16 @@ cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.27.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +contrib.go.opencensus.io/exporter/stackdriver v0.6.0/go.mod h1:QeFzMJDAw8TXt5+aRaSuE8l5BwaMIOIlaVkBOPRuMuw= +git.apache.org/thrift.git v0.0.0-20180807212849-6e67faa92827/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q= +github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Shopify/toxiproxy v2.1.3+incompatible h1:awiJqUYH4q4OmoBiRccJykjd7B+w0loJi2keSna4X/M= github.com/Shopify/toxiproxy v2.1.3+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/Sirupsen/logrus v1.0.6 h1:HCAGQRk48dRVPA5Y+Yh0qdCSTzPOyU1tBJ7Q9YzotII= +github.com/Sirupsen/logrus v1.0.6/go.mod h1:rmk17hk6i8ZSAJkSDa7nOxamrG+SP4P0mm+DAvExv4U= github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f h1:5ZfJxyXo8KyX8DgGXC5B7ILL8y51fci/qYz2B4j8iLY= github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 h1:45bxf7AZMwWcqkLzDAQugVEwedisr5nRJ1r+7LYnv0U= @@ -12,8 +19,11 @@ github.com/alicebob/miniredis v0.0.0-20180911162847-3657542c8629 h1:gLoh8jzwIxdi github.com/alicebob/miniredis v0.0.0-20180911162847-3657542c8629/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/aws/aws-sdk-go v1.15.34/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/cheggaaa/pb v1.0.5-0.20160713104425-73ae1d68fe0b h1:CMRCnhHx4xVxJy+wPsS67xmi9RHGNctLMoVn9Q1Kit8= @@ -21,12 +31,33 @@ github.com/cheggaaa/pb v1.0.5-0.20160713104425-73ae1d68fe0b/go.mod h1:pQciLPpbU0 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudfoundry/gosigar v1.1.0 h1:V/dVCzhKOdIU3WRB5inQU20s4yIgL9Dxx/Mhi0SF8eM= github.com/cloudfoundry/gosigar v1.1.0/go.mod h1:3qLfc2GlfmwOx2+ZDaRGH3Y9fwQ0sQeaAleo2GV5pH0= +github.com/cockroachdb/cockroach-go v0.0.0-20180212155653-59c0560478b7/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk= +github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= +github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f/go.mod h1:8S58EK26zhXSxzv7NQFpnliaOQsmDUxvoQO3rt154Vg= +github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc= +github.com/cznic/internal v0.0.0-20180608152220-f44710a21d00/go.mod h1:olo7eAdKwJdXxb55TKGLiJ6xt1H0/tiiRCWKVLmtjY4= +github.com/cznic/lldb v1.1.0/go.mod h1:FIZVUmYUVhPwRiPzL8nD/mpFcJ/G7SSXjjXYG4uRI3A= +github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= +github.com/cznic/ql v1.2.0/go.mod h1:FbpzhyZrqr0PVlK6ury+PoW3T0ODUV22OeWIxcaOrSE= +github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= +github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= +github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc/go.mod h1:Y1SNZ4dRUOKXshKUbwUapqNncRrho4mkjQebgEHZLj8= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg= github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8= +github.com/docker/distribution v0.0.0-20180720172123-0dae0957e5fe h1:ZRQNMB7Sw5jf9g/0imDbI+vTFNk4J7qBdtFI5/zf1kg= +github.com/docker/distribution v0.0.0-20180720172123-0dae0957e5fe/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v0.0.0-20170502054910-90d35abf7b35 h1:zQuy/ry/KtSvCczhEPo+ud47S8alruA/Z9NDlz7EzVo= +github.com/docker/docker v0.0.0-20170502054910-90d35abf7b35/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/dustin/go-humanize v0.0.0-20180713052910-9f541cc9db5d h1:lDrio3iIdNb0Gw9CgH7cQF+iuB5mOOjdJ9ERNJCBgb4= github.com/dustin/go-humanize v0.0.0-20180713052910-9f541cc9db5d/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU= @@ -37,6 +68,7 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eclipse/paho.mqtt.golang v1.1.1 h1:iPJYXJLaViCshRTW/PSqImSS6HJ2Rf671WR0bXZ2GIU= github.com/eclipse/paho.mqtt.golang v1.1.1/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= +github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/go-bindata-assetfs v1.0.0 h1:G/bYguwHIzWq9ZoyUQqrjTmJbbYn3j3CKKpKinvZLFk= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= @@ -47,8 +79,12 @@ github.com/fortytw2/leaktest v1.2.0 h1:cj6GCiwJDH7l3tMHLjZDo0QqPtrXJiWSI9JgpeQKw github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsouza/fake-gcs-server v1.2.0/go.mod h1:rM69NBSmfAkTlKDhzXC41OeX9lQxQXnkimkiGWDU1Ek= github.com/garyburd/redigo v1.0.1-0.20170216214944-0d253a66e6e1 h1:YmyuMm99D7kezPc0ZVWYnaUIWfMKR81lVVXttKTnDbw= github.com/garyburd/redigo v1.0.1-0.20170216214944-0d253a66e6e1/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/go-bindata/go-bindata v1.0.0 h1:upAQEGKG3hFv00/4cU/VhzF+NpCHz+Jk3rO946wWr+A= +github.com/go-bindata/go-bindata v1.0.0/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo= +github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-ini/ini v1.38.2 h1:6Hl/z3p3iFkA0dlDfzYxuFuUGD+kaweypF6btsR2/Q4= github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= @@ -57,8 +93,11 @@ github.com/go-redis/redis v6.14.1+incompatible h1:kSJohAREGMr344uMa8PzuIg5OU6ylC github.com/go-redis/redis v6.14.1+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/gocql/gocql v0.0.0-20180913072538-864d5908455a/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0= github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang-migrate/migrate/v3 v3.5.2 h1:SUWSv6PD8Lr2TGx1lmVW7W2lRoQiVny3stM4He6jczQ= +github.com/golang-migrate/migrate/v3 v3.5.2/go.mod h1:QDa9JH6Bdwbi+1jIEpOIWnTNoYRbtaVgByErXU0844E= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= @@ -66,12 +105,17 @@ github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/martian v2.0.0-beta.2+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c h1:16eHWuMGvCjSfgRJKqIzapE78onvvTbdi1rMkU00lZw= github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= @@ -84,6 +128,7 @@ github.com/gorilla/rpc v1.1.0 h1:marKfvVP0Gpd/jHlVBKCQ8RAoUPdX7K1Nuh6l1BNh7A= github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 h1:7xsUJsB2NrdcttQPa7JLEaGzvdbk7KvfrjgHZXOQRo0= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hanwen/go-fuse v0.0.0-20181011180456-b760b55765be h1:RXF6Da5rbJlRUosxKxuy/3OrLLG77aXRrZYcjDs6aB4= github.com/hanwen/go-fuse v0.0.0-20181011180456-b760b55765be/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok= github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= @@ -108,6 +153,8 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6 h1:4zOlv2my+vf98jT1nQt4bT/yKWUImevYPJ2H344CloE= github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6/go.mod h1:r/8JmuR0qjuCiEhAolkfvdZgmPiHTnJaG0UXCSeR1Zo= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/monkit-hw v0.0.0-20180827162413-5a254051f35d h1:+TWXq24sHXNI6DVobckzgJCbCvb4bYjDZuKRR+d/Nuw= @@ -123,8 +170,9 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v0.0.0-20180523175426-90697d60dd84 h1:it29sI2IM490luSc3RAhp5WuCYnc6RtbfLVAB7nmC5M= -github.com/lib/pq v0.0.0-20180523175426-90697d60dd84/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/kshvakov/clickhouse v1.3.4/go.mod h1:DMzX7FxRymoNkVgizH0DWAL8Cur7wHLgx3MUnGwJqpE= +github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/loov/hrtime v0.0.0-20180911122900-a9e82bc6c180 h1:kLwg5eA/kaWQ/RwANTH7Gg+VdxmdjbcSWyaS/1VQGkA= github.com/loov/hrtime v0.0.0-20180911122900-a9e82bc6c180/go.mod h1:2871C3urfEJnq/bpTYjFdMOdgxVd8otLLEL6vMNy/Iw= github.com/loov/plot v0.0.0-20180510142208-e59891ae1271 h1:51ToN6N0TDtCruf681gufYuEhO9qFHQzM3RFTS/n6XE= @@ -187,6 +235,9 @@ github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= @@ -199,14 +250,15 @@ github.com/pkg/profile v1.2.1 h1:F++O52m40owAmADcojzM+9gyjmMOY/T4oYJkgFDH8RE= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.0-pre1.0.20180416233856-82f5ff156b29 h1:NWk5kfSrcwyaUbo3Ee6EECK/fXsY1sJt0U6kYnFNYmc= github.com/prometheus/client_golang v0.9.0-pre1.0.20180416233856-82f5ff156b29/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5 h1:cLL6NowurKLMfCeQy4tIeph12XNQWgANCNvdyrOYKV4= -github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/common v0.0.0-20180326160409-38c53a9f4bfc h1:tyg3EcZAmwCUe90Jzl4Qw6Af+ajuW8S9b1VFitMNOQs= -github.com/prometheus/common v0.0.0-20180326160409-38c53a9f4bfc/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/procfs v0.0.0-20180408092902-8b1c2da0d56d h1:RCcsxyRr6+/pLg6wr0cUjPovhEhSNOtPh0SOz6u3hGU= -github.com/prometheus/procfs v0.0.0-20180408092902-8b1c2da0d56d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e h1:n/3MEhJQjQxrOUCzh1Y3Re6aJUUWRp2M9+Oc3eVn/54= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 h1:agujYaXJSxSo18YNX3jzl+4G6Bstwt+kqv47GS12uL0= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/rcrowley/go-metrics v0.0.0-20180503174638-e2704e165165 h1:nkcn14uNmFEuGCb2mBZbBb24RdNRL08b/wb+xBOYpuk= github.com/rcrowley/go-metrics v0.0.0-20180503174638-e2704e165165/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rs/cors v1.5.0 h1:dgSHE6+ia18arGOTIYQKKGWLvEbGvmbNE6NfxhoNHUY= @@ -215,6 +267,8 @@ github.com/segmentio/go-prompt v1.2.1-0.20161017233205-f0d19b6901ad h1:EqOdoSJGI github.com/segmentio/go-prompt v1.2.1-0.20161017233205-f0d19b6901ad/go.mod h1:B3ehdD1xPoWDKgrQgUaGk+m8H1xb1J5TyYDfKpKNeEE= github.com/shirou/gopsutil v2.17.12+incompatible h1:FNbznluSK3DQggqiVw3wK/tFKJrKlLPBuQ+V8XkkCOc= github.com/shirou/gopsutil v2.17.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/sirupsen/logrus v1.0.6 h1:hcP1GmhGigz/O7h1WVUM5KklBp1JoNS9FggWKdj/j3s= +github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/skyrings/skyring-common v0.0.0-20160929130248-d1c0bb1cbd5e h1:jrZSSgPUDtBeJbGXqgGUeupQH8I+ZvGXfhpIahye2Bc= github.com/skyrings/skyring-common v0.0.0-20160929130248-d1c0bb1cbd5e/go.mod h1:d8hQseuYt4rJoOo21lFzYJdhMjmDqLY++ayArbgYjWI= github.com/smartystreets/assertions v0.0.0-20180820201707-7c9eb446e3cf h1:6V1qxN6Usn4jy8unvggSJz/NC790tefw8Zdy6OZS5co= @@ -262,33 +316,34 @@ github.com/zeebo/float16 v0.1.0 h1:kRqxv5og6z1emEyz5FpW0/BVHe5VfxEAw6b1ljCZlUc= github.com/zeebo/float16 v0.1.0/go.mod h1:fssGvvXu+XS8MH57cKmyrLB/cqioYeYX/2mXCN3a5wo= github.com/zeebo/incenc v0.0.0-20180505221441-0d92902eec54 h1:+cwNE5KJ3pika4HuzmDHkDlK5myo0G9Sv+eO7WWxnUQ= github.com/zeebo/incenc v0.0.0-20180505221441-0d92902eec54/go.mod h1:EI8LcOBDlSL3POyqwC1eJhOYlMBMidES+613EtmmT5w= +go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= +go.opencensus.io v0.16.0/go.mod h1:0TeCCqcQSLNZtiq/62+vUzqwnjqF5el6hjmuZaFtyNk= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4 h1:Vk3wNqEZwyGyei9yq5ekj7frek2u7HUfffJ1/opblzc= -golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941 h1:qBTHLajHecfu+xzRI9PqVDcqx7SdHj9d4B+EzSn3tAc= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/net v0.0.0-20180821023952-922f4815f713/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181003013248-f5e5bdd77824 h1:MkjFNbaZJyH98M67Q3umtwZ+EdVdrNJLqSwZp5vcv60= golang.org/x/net v0.0.0-20181003013248-f5e5bdd77824/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181005035420-146acd28ed58 h1:otZG8yDCO4LVps5+9bxOeNiCvgmOyt96J3roHTYs7oE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180821140842-3b58ed4ad339/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180824143301-4910a1d54f87 h1:GqwDwfvIpC33dK9bA1fD+JiDUNsuAiQiEkpHqUKze4o= golang.org/x/sys v0.0.0-20180824143301-4910a1d54f87/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181004145325-8469e314837c h1:SJ7JoQNVl3mC7EWkkONgBWgCno8LcABIJwFMkWBC+EY= -golang.org/x/sys v0.0.0-20181004145325-8469e314837c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181005133103-4497e2df6f9e h1:EfdBzeKbFSvOjoIqSZcfS8wp0FBLokGBEs9lz1OtSg0= golang.org/x/sys v0.0.0-20181005133103-4497e2df6f9e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= @@ -297,14 +352,23 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2I golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52 h1:JG/0uqcGdTNgq7FdU+61l5Pdmb8putNZlXb65bJBROs= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180911133044-677d2ff680c1/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/api v0.0.0-20180818000503-e21acd801f91/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20180826000528-7954115fcf34/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180912233945-5a2fd4cab2d6 h1:YN6g8chEdBTmaERxJzbJ6WKLrW3+Bf6rznOBkWNSQP0= +google.golang.org/genproto v0.0.0-20180912233945-5a2fd4cab2d6/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.15.0 h1:Az/KuahOM4NAidTEuJCv/RonAA7rYsTPkqXVjr+8OOw= google.golang.org/grpc v1.15.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= gopkg.in/Shopify/sarama.v1 v1.18.0 h1:f9aTXuIEFEjVvLG9p+kMSk01dMfFumHsySRk1okTdqU= gopkg.in/Shopify/sarama.v1 v1.18.0/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= +gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -312,6 +376,9 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25 h1:Ev7yu1/f6+d+b3pi5vPdRPc6nNtP1umSfcWiEfRqv6I= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.38.2 h1:dGcbywv4RufeGeiMycPT/plKB5FtmLKLnWKwBiLhUA4= gopkg.in/ini.v1 v1.38.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/olivere/elastic.v5 v5.0.76 h1:A6W7X4yLPQDINHiYAqIwqev+rD5hIQ4G0e1d5H//VXk= diff --git a/pkg/pointerdb/config.go b/pkg/pointerdb/config.go index 63bbc6310..9af1ec6f5 100644 --- a/pkg/pointerdb/config.go +++ b/pkg/pointerdb/config.go @@ -12,13 +12,15 @@ import ( "storj.io/storj/pkg/pb" "storj.io/storj/pkg/provider" "storj.io/storj/pkg/utils" + "storj.io/storj/storage" "storj.io/storj/storage/boltdb" + "storj.io/storj/storage/postgreskv" "storj.io/storj/storage/storelogger" ) const ( - // PointerBucket is the string representing the bucket used for `PointerEntries` - PointerBucket = "pointers" + // BoltPointerBucket is the string representing the bucket used for `PointerEntries` in BoltDB + BoltPointerBucket = "pointers" ) // Config is a configuration struct that is everything you need to start a @@ -30,25 +32,32 @@ type Config struct { Overlay bool `default:"false" help:"toggle flag if overlay is enabled"` } +func newKeyValueStore(dbURLString string) (db storage.KeyValueStore, err error) { + dburl, err := utils.ParseURL(dbURLString) + if err != nil { + return nil, err + } + if dburl.Scheme == "bolt" { + db, err = boltdb.New(dburl.Path, BoltPointerBucket) + } else if dburl.Scheme == "postgresql" || dburl.Scheme == "postgres" { + db, err = postgreskv.New(dbURLString) + } else { + err = Error.New("unsupported db scheme: %s", dburl.Scheme) + } + return db, err +} + // Run implements the provider.Responsibility interface func (c Config) Run(ctx context.Context, server *provider.Provider) error { - dburl, err := utils.ParseURL(c.DatabaseURL) + db, err := newKeyValueStore(c.DatabaseURL) if err != nil { return err } - if dburl.Scheme != "bolt" { - return Error.New("unsupported db scheme: %s", dburl.Scheme) - } - - bdb, err := boltdb.New(dburl.Path, PointerBucket) - if err != nil { - return err - } - defer func() { _ = bdb.Close() }() + defer func() { _ = db.Close() }() cache := overlay.LoadFromContext(ctx) - bdblogged := storelogger.New(zap.L(), bdb) - pb.RegisterPointerDBServer(server.GRPC(), NewServer(bdblogged, cache, zap.L(), c, server.Identity())) + dblogged := storelogger.New(zap.L(), db) + pb.RegisterPointerDBServer(server.GRPC(), NewServer(dblogged, cache, zap.L(), c, server.Identity())) return server.Run(ctx) } diff --git a/storage/boltdb/client.go b/storage/boltdb/client.go index dc8cd5de6..bd08ef2dd 100644 --- a/storage/boltdb/client.go +++ b/storage/boltdb/client.go @@ -161,8 +161,8 @@ func (client *Client) Close() error { return nil } -// GetAll finds all values for the provided keys up to 100 keys -// if more keys are provided than the maximum an error will be returned. +// GetAll finds all values for the provided keys (up to storage.LookupLimit). +// If more keys are provided than the maximum, an error will be returned. func (client *Client) GetAll(keys storage.Keys) (storage.Values, error) { if len(keys) > storage.LookupLimit { return nil, storage.ErrLimitExceeded diff --git a/storage/boltdb/client_test.go b/storage/boltdb/client_test.go index e01d31d95..7a61b1001 100644 --- a/storage/boltdb/client_test.go +++ b/storage/boltdb/client_test.go @@ -4,11 +4,14 @@ package boltdb import ( + "fmt" "io/ioutil" "os" "path/filepath" "testing" + "storj.io/storj/pkg/utils" + "storj.io/storj/storage" "storj.io/storj/storage/testsuite" ) @@ -78,3 +81,61 @@ func TestSuiteShared(t *testing.T) { testsuite.RunTests(t, store) } } + +type boltLongBenchmarkStore struct { + *Client + dirPath string +} + +func (store *boltLongBenchmarkStore) BulkImport(iter storage.Iterator) (err error) { + // turn off syncing during import + oldval := store.db.NoSync + store.db.NoSync = true + defer func() { store.db.NoSync = oldval }() + + var item storage.ListItem + for iter.Next(&item) { + if err := store.Put(item.Key, item.Value); err != nil { + return fmt.Errorf("Failed to insert data (%q, %q): %v", item.Key, item.Value, err) + } + } + + return store.db.Sync() +} + +func (store *boltLongBenchmarkStore) BulkDelete() error { + // do nothing here; everything will be cleaned up later after the test completes. it's not + // worth it to wait for BoltDB to remove every key, one by one, and we can't just + // os.RemoveAll() the whole test directory at this point because those files are still open + // and unremoveable on Windows. + return nil +} + +func BenchmarkSuiteLong(b *testing.B) { + tempdir, err := ioutil.TempDir("", "storj-bolt") + if err != nil { + b.Fatal(err) + } + defer func() { + if err := os.RemoveAll(tempdir); err != nil { + b.Fatal(err) + } + }() + + dbname := filepath.Join(tempdir, "bolt.db") + store, err := New(dbname, "bucket") + if err != nil { + b.Fatalf("failed to create db: %v", err) + } + defer func() { + if err := utils.CombineErrors(store.Close(), os.RemoveAll(tempdir)); err != nil { + b.Fatalf("failed to close db: %v", err) + } + }() + + longStore := &boltLongBenchmarkStore{ + Client: store, + dirPath: tempdir, + } + testsuite.BenchmarkPathOperationsInLargeDb(b, longStore) +} diff --git a/storage/boltdb/common.go b/storage/boltdb/common.go index 71163fa13..f8a671672 100644 --- a/storage/boltdb/common.go +++ b/storage/boltdb/common.go @@ -9,6 +9,3 @@ import ( // Error is the default boltdb errs class var Error = errs.Class("boltdb error") - -// ErrKeyNotFound should occur when a key isn't found in a boltdb bucket (table) -var ErrKeyNotFound = errs.Class("key not found") diff --git a/storage/postgreskv/alternateclient.go b/storage/postgreskv/alternateclient.go new file mode 100644 index 000000000..010adc64b --- /dev/null +++ b/storage/postgreskv/alternateclient.go @@ -0,0 +1,165 @@ +// Copyright (C) 2018 Storj Labs, Inc. + +// See LICENSE for copying information. + +package postgreskv + +import ( + "database/sql" + + "storj.io/storj/pkg/utils" + "storj.io/storj/storage" +) + +const ( + alternateSQLSetup = ` +CREATE OR REPLACE FUNCTION local_path (fullpath BYTEA, prefix BYTEA, delimiter BYTEA) + RETURNS BYTEA AS $$ +DECLARE + relative BYTEA; + pos INTEGER; +BEGIN + relative := substring(fullpath FROM (octet_length(prefix)+1)); + pos := position(delimiter IN relative); + IF pos = 0 THEN + RETURN relative; + END IF; + RETURN substring(relative FOR pos); +END; +$$ LANGUAGE 'plpgsql' + IMMUTABLE STRICT; +` + + alternateSQLTeardown = ` +DROP FUNCTION local_path(BYTEA, BYTEA, BYTEA); +` + + alternateForwardQuery = ` +SELECT DISTINCT + $2::BYTEA || x.localpath AS p, + first_value(x.metadata) OVER (PARTITION BY x.localpath ORDER BY x.fullpath) AS m +FROM ( + SELECT + pd.fullpath, + local_path(pd.fullpath, $2::BYTEA, set_byte(' '::BYTEA, 0, b.delim)) AS localpath, + pd.metadata + FROM + pathdata pd, + buckets b + WHERE + b.bucketname = $1::BYTEA + AND pd.bucket = b.bucketname + AND pd.fullpath >= $2::BYTEA + AND ($2::BYTEA = ''::BYTEA OR pd.fullpath < bytea_increment($2::BYTEA)) + AND pd.fullpath >= $3::BYTEA +) x +ORDER BY p +LIMIT $4 +` + + alternateReverseQuery = ` +SELECT DISTINCT + $2::BYTEA || x.localpath AS p, + first_value(x.metadata) OVER (PARTITION BY x.localpath ORDER BY x.fullpath) AS m +FROM ( + SELECT + pd.fullpath, + local_path(pd.fullpath, $2::BYTEA, set_byte(' '::BYTEA, 0, b.delim)) AS localpath, + pd.metadata + FROM + pathdata pd, + buckets b + WHERE + b.bucketname = $1::BYTEA + AND pd.bucket = b.bucketname + AND pd.fullpath >= $2::BYTEA + AND ($2::BYTEA = ''::BYTEA OR pd.fullpath < bytea_increment($2::BYTEA)) + AND ($3::BYTEA = ''::BYTEA OR pd.fullpath <= $3::BYTEA) +) x +ORDER BY p DESC +LIMIT $4 +` +) + +// AlternateClient is the entrypoint into an alternate postgreskv data store +type AlternateClient struct { + *Client +} + +// AltNew instantiates a new postgreskv AlternateClient given db URL +func AltNew(dbURL string) (*AlternateClient, error) { + client, err := New(dbURL) + if err != nil { + return nil, err + } + _, err = client.pgConn.Exec(alternateSQLSetup) + if err != nil { + return nil, utils.CombineErrors(err, client.Close()) + } + return &AlternateClient{Client: client}, nil +} + +// Close closes an AlternateClient and frees its resources. +func (altClient *AlternateClient) Close() error { + _, err := altClient.pgConn.Exec(alternateSQLTeardown) + return utils.CombineErrors(err, altClient.Client.Close()) +} + +type alternateOrderedPostgresIterator struct { + *orderedPostgresIterator +} + +func (opi *alternateOrderedPostgresIterator) doNextQuery() (*sql.Rows, error) { + if opi.opts.Recurse { + return opi.orderedPostgresIterator.doNextQuery() + } + start := opi.lastKeySeen + if start == nil { + start = opi.opts.First + } + var query string + if opi.opts.Reverse { + query = alternateReverseQuery + } else { + query = alternateForwardQuery + } + return opi.client.pgConn.Query(query, []byte(opi.bucket), []byte(opi.opts.Prefix), []byte(start), opi.batchSize+1) +} + +func newAlternateOrderedPostgresIterator(altClient *AlternateClient, opts storage.IterateOptions, batchSize int) (*alternateOrderedPostgresIterator, error) { + if opts.Prefix == nil { + opts.Prefix = storage.Key("") + } + if opts.First == nil { + opts.First = storage.Key("") + } + opi1 := &orderedPostgresIterator{ + client: altClient.Client, + opts: &opts, + bucket: storage.Key(defaultBucket), + delimiter: byte('/'), + batchSize: batchSize, + curIndex: 0, + } + opi := &alternateOrderedPostgresIterator{orderedPostgresIterator: opi1} + opi.nextQuery = opi.doNextQuery + newRows, err := opi.nextQuery() + if err != nil { + return nil, err + } + opi.curRows = newRows + return opi, nil +} + +// Iterate iterates over items based on opts +func (altClient *AlternateClient) Iterate(opts storage.IterateOptions, fn func(storage.Iterator) error) (err error) { + opi, err := newAlternateOrderedPostgresIterator(altClient, opts, defaultBatchSize) + if err != nil { + return err + } + defer func() { + err = utils.CombineErrors(err, opi.Close()) + }() + + return fn(opi) +} diff --git a/storage/postgreskv/alternateclient_test.go b/storage/postgreskv/alternateclient_test.go new file mode 100644 index 000000000..720abdac0 --- /dev/null +++ b/storage/postgreskv/alternateclient_test.go @@ -0,0 +1,73 @@ +// Copyright (C) 2018 Storj Labs, Inc. +// See LICENSE for copying information. + +package postgreskv + +import ( + "flag" + "testing" + + "go.uber.org/zap/zaptest" + + "storj.io/storj/storage" + "storj.io/storj/storage/storelogger" + "storj.io/storj/storage/testsuite" +) + +var ( + doAltTests = flag.Bool("test-postgreskv-alt", false, "Run the KeyValueStore tests against the alternate PG implementation") +) + +func newTestAlternatePostgres(t testing.TB) (store *AlternateClient, cleanup func()) { + if !*doAltTests { + t.Skip("alternate-implementation PG tests not enabled.") + } + if *testPostgres == "" { + t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", defaultPostgresConn) + } + + pgdb, err := AltNew(*testPostgres) + if err != nil { + t.Fatalf("init: %v", err) + } + + return pgdb, func() { + if err := pgdb.Close(); err != nil { + t.Fatalf("failed to close db: %v", err) + } + } +} + +func TestSuiteAlt(t *testing.T) { + store, cleanup := newTestAlternatePostgres(t) + defer cleanup() + + zap := zaptest.NewLogger(t) + testsuite.RunTests(t, storelogger.New(zap, store)) +} + +func BenchmarkSuiteAlt(b *testing.B) { + store, cleanup := newTestAlternatePostgres(b) + defer cleanup() + + testsuite.RunBenchmarks(b, store) +} + +type pgAltLongBenchmarkStore struct { + *AlternateClient +} + +func (store *pgAltLongBenchmarkStore) BulkImport(iter storage.Iterator) error { + return bulkImport(store.pgConn, iter) +} + +func (store *pgAltLongBenchmarkStore) BulkDelete() error { + return bulkDelete(store.pgConn) +} + +func BenchmarkSuiteLongAlt(b *testing.B) { + store, cleanup := newTestAlternatePostgres(b) + defer cleanup() + + testsuite.BenchmarkPathOperationsInLargeDb(b, &pgAltLongBenchmarkStore{store}) +} diff --git a/storage/postgreskv/client.go b/storage/postgreskv/client.go new file mode 100644 index 000000000..20238ff81 --- /dev/null +++ b/storage/postgreskv/client.go @@ -0,0 +1,295 @@ +// Copyright (C) 2018 Storj Labs, Inc. +// See LICENSE for copying information. + +package postgreskv + +import ( + "database/sql" + "fmt" + + "github.com/lib/pq" + "github.com/zeebo/errs" + + "storj.io/storj/pkg/utils" + "storj.io/storj/storage" + "storj.io/storj/storage/postgreskv/schema" +) + +const ( + defaultBatchSize = 10000 + defaultBucket = "" +) + +// Client is the entrypoint into a postgreskv data store +type Client struct { + URL string + pgConn *sql.DB +} + +// New instantiates a new postgreskv client given db URL +func New(dbURL string) (*Client, error) { + pgConn, err := sql.Open("postgres", dbURL) + if err != nil { + return nil, err + } + err = schema.PrepareDB(pgConn) + if err != nil { + return nil, err + } + return &Client{ + URL: dbURL, + pgConn: pgConn, + }, nil +} + +// Put sets the value for the provided key. +func (client *Client) Put(key storage.Key, value storage.Value) error { + return client.PutPath(storage.Key(defaultBucket), key, value) +} + +// PutPath sets the value for the provided key (in the given bucket). +func (client *Client) PutPath(bucket, key storage.Key, value storage.Value) error { + if key.IsZero() { + return Error.New("invalid key") + } + q := ` + INSERT INTO pathdata (bucket, fullpath, metadata) + VALUES ($1::BYTEA, $2::BYTEA, $3::BYTEA) + ON CONFLICT (bucket, fullpath) DO UPDATE SET metadata = EXCLUDED.metadata + ` + _, err := client.pgConn.Exec(q, []byte(bucket), []byte(key), []byte(value)) + return err +} + +// Get looks up the provided key and returns its value (or an error). +func (client *Client) Get(key storage.Key) (storage.Value, error) { + return client.GetPath(storage.Key(defaultBucket), key) +} + +// GetPath looks up the provided key (in the given bucket) and returns its value (or an error). +func (client *Client) GetPath(bucket, key storage.Key) (storage.Value, error) { + q := "SELECT metadata FROM pathdata WHERE bucket = $1::BYTEA AND fullpath = $2::BYTEA" + row := client.pgConn.QueryRow(q, []byte(bucket), []byte(key)) + var val []byte + err := row.Scan(&val) + if err == sql.ErrNoRows { + return nil, storage.ErrKeyNotFound.New(key.String()) + } + if err != nil { + return nil, err + } + return val, nil +} + +// Delete deletes the given key and its associated value. +func (client *Client) Delete(key storage.Key) error { + return client.DeletePath(storage.Key(defaultBucket), key) +} + +// DeletePath deletes the given key (in the given bucket) and its associated value. +func (client *Client) DeletePath(bucket, key storage.Key) error { + q := "DELETE FROM pathdata WHERE bucket = $1::BYTEA AND fullpath = $2::BYTEA" + result, err := client.pgConn.Exec(q, []byte(bucket), []byte(key)) + if err != nil { + return err + } + numRows, err := result.RowsAffected() + if err != nil { + return err + } + if numRows == 0 { + return storage.ErrKeyNotFound.New(key.String()) + } + return nil +} + +// List returns either a list of known keys, in order, or an error. +func (client *Client) List(first storage.Key, limit int) (storage.Keys, error) { + return storage.ListKeys(client, first, limit) +} + +// ReverseList returns either a list of known keys, in reverse order, or an error. +// Starts from first and iterates backwards +func (client *Client) ReverseList(first storage.Key, limit int) (storage.Keys, error) { + return storage.ReverseListKeys(client, first, limit) +} + +// Close closes the client +func (client *Client) Close() error { + return client.pgConn.Close() +} + +// GetAll finds all values for the provided keys (up to storage.LookupLimit). +// If more keys are provided than the maximum, an error will be returned. +func (client *Client) GetAll(keys storage.Keys) (storage.Values, error) { + return client.GetAllPath(storage.Key(defaultBucket), keys) +} + +// GetAllPath finds all values for the provided keys (up to storage.LookupLimit) +// in the given bucket. if more keys are provided than the maximum, an error +// will be returned. +func (client *Client) GetAllPath(bucket storage.Key, keys storage.Keys) (storage.Values, error) { + if len(keys) > storage.LookupLimit { + return nil, storage.ErrLimitExceeded + } + + q := ` + SELECT metadata + FROM pathdata pd + RIGHT JOIN + unnest($2::BYTEA[]) WITH ORDINALITY pk(request, ord) + ON (pd.fullpath = pk.request AND pd.bucket = $1::BYTEA) + ORDER BY pk.ord + ` + rows, err := client.pgConn.Query(q, []byte(bucket), pq.ByteaArray(keys.ByteSlices())) + if err != nil { + return nil, errs.Wrap(err) + } + values := make([]storage.Value, 0, len(keys)) + for rows.Next() { + var value []byte + if err := rows.Scan(&value); err != nil { + return nil, errs.Wrap(utils.CombineErrors(err, rows.Close())) + } + values = append(values, storage.Value(value)) + } + return values, utils.CombineErrors(rows.Err(), rows.Close()) +} + +type orderedPostgresIterator struct { + client *Client + opts *storage.IterateOptions + bucket storage.Key + delimiter byte + batchSize int + curIndex int + curRows *sql.Rows + lastKeySeen storage.Key + errEncountered error + nextQuery func() (*sql.Rows, error) +} + +// Next fills in info for the next item in an ongoing listing. +func (opi *orderedPostgresIterator) Next(item *storage.ListItem) bool { + if !opi.curRows.Next() { + if err := opi.curRows.Close(); err != nil { + opi.errEncountered = errs.Wrap(err) + return false + } + if opi.curIndex < opi.batchSize { + return false + } + if err := opi.curRows.Err(); err != nil { + opi.errEncountered = errs.Wrap(err) + return false + } + newRows, err := opi.nextQuery() + if err != nil { + opi.errEncountered = errs.Wrap(err) + return false + } + opi.curRows = newRows + opi.curIndex = 0 + if !opi.curRows.Next() { + if err := opi.curRows.Close(); err != nil { + opi.errEncountered = errs.Wrap(err) + } + return false + } + } + var k, v []byte + err := opi.curRows.Scan(&k, &v) + if err != nil { + opi.errEncountered = utils.CombineErrors(errs.Wrap(err), errs.Wrap(opi.curRows.Close())) + return false + } + item.Key = storage.Key(k) + item.Value = storage.Value(v) + opi.curIndex++ + if opi.curIndex == 1 && opi.lastKeySeen.Equal(item.Key) { + return opi.Next(item) + } + if !opi.opts.Recurse && item.Key[len(item.Key)-1] == opi.delimiter && !item.Key.Equal(opi.opts.Prefix) { + item.IsPrefix = true + // i don't think this makes the most sense, but it's necessary to pass the storage testsuite + item.Value = nil + } else { + item.IsPrefix = false + } + opi.lastKeySeen = item.Key + return true +} + +func (opi *orderedPostgresIterator) doNextQuery() (*sql.Rows, error) { + start := opi.lastKeySeen + if start == nil { + start = opi.opts.First + } + var query string + if !opi.opts.Recurse { + if opi.opts.Reverse { + query = "SELECT p, m FROM list_directory_reverse($1::BYTEA, $2::BYTEA, $3::BYTEA, $4) ld(p, m)" + } else { + query = "SELECT p, m FROM list_directory($1::BYTEA, $2::BYTEA, $3::BYTEA, $4) ld(p, m)" + } + } else { + startCmp := ">=" + orderDir := "" + if opi.opts.Reverse { + startCmp = "<=" + orderDir = " DESC" + } + query = fmt.Sprintf(` + SELECT fullpath, metadata + FROM pathdata + WHERE bucket = $1::BYTEA + AND ($2::BYTEA = ''::BYTEA OR fullpath >= $2::BYTEA) + AND ($2::BYTEA = ''::BYTEA OR fullpath < bytea_increment($2::BYTEA)) + AND ($3::BYTEA = ''::BYTEA OR fullpath %s $3::BYTEA) + ORDER BY fullpath%s + LIMIT $4 + `, startCmp, orderDir) + } + return opi.client.pgConn.Query(query, []byte(opi.bucket), []byte(opi.opts.Prefix), []byte(start), opi.batchSize+1) +} + +func (opi *orderedPostgresIterator) Close() error { + return utils.CombineErrors(opi.errEncountered, opi.curRows.Close()) +} + +func newOrderedPostgresIterator(pgClient *Client, opts storage.IterateOptions, batchSize int) (*orderedPostgresIterator, error) { + if opts.Prefix == nil { + opts.Prefix = storage.Key("") + } + if opts.First == nil { + opts.First = storage.Key("") + } + opi := &orderedPostgresIterator{ + client: pgClient, + opts: &opts, + bucket: storage.Key(defaultBucket), + delimiter: byte('/'), + batchSize: batchSize, + curIndex: 0, + } + opi.nextQuery = opi.doNextQuery + newRows, err := opi.nextQuery() + if err != nil { + return nil, err + } + opi.curRows = newRows + return opi, nil +} + +// Iterate iterates over items based on opts +func (client *Client) Iterate(opts storage.IterateOptions, fn func(storage.Iterator) error) (err error) { + opi, err := newOrderedPostgresIterator(client, opts, defaultBatchSize) + if err != nil { + return err + } + defer func() { + err = utils.CombineErrors(err, opi.Close()) + }() + + return fn(opi) +} diff --git a/storage/postgreskv/client_test.go b/storage/postgreskv/client_test.go new file mode 100644 index 000000000..85ed3f845 --- /dev/null +++ b/storage/postgreskv/client_test.go @@ -0,0 +1,124 @@ +// Copyright (C) 2018 Storj Labs, Inc. +// See LICENSE for copying information. + +package postgreskv + +import ( + "database/sql" + "flag" + "os" + "testing" + + "github.com/lib/pq" + "github.com/zeebo/errs" + "go.uber.org/zap/zaptest" + + "storj.io/storj/pkg/utils" + "storj.io/storj/storage" + "storj.io/storj/storage/storelogger" + "storj.io/storj/storage/testsuite" +) + +const ( + // this connstring is expected to work under the storj-test docker-compose instance + defaultPostgresConn = "postgres://pointerdb:pg-secret-pass@test-postgres-pointerdb/pointerdb?sslmode=disable" +) + +var ( + testPostgres = flag.String("postgres-test-db", os.Getenv("STORJ_POSTGRESKV_TEST"), "PostgreSQL test database connection string") +) + +func newTestPostgres(t testing.TB) (store *Client, cleanup func()) { + if *testPostgres == "" { + t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", defaultPostgresConn) + } + + pgdb, err := New(*testPostgres) + if err != nil { + t.Fatalf("init: %v", err) + } + + return pgdb, func() { + if err := pgdb.Close(); err != nil { + t.Fatalf("failed to close db: %v", err) + } + } +} + +func TestSuite(t *testing.T) { + store, cleanup := newTestPostgres(t) + defer cleanup() + + zap := zaptest.NewLogger(t) + testsuite.RunTests(t, storelogger.New(zap, store)) +} + +func BenchmarkSuite(b *testing.B) { + store, cleanup := newTestPostgres(b) + defer cleanup() + + testsuite.RunBenchmarks(b, store) +} + +func bulkImport(db *sql.DB, iter storage.Iterator) (err error) { + txn, err2 := db.Begin() + if err2 != nil { + return errs.New("Failed to start transaction: %v", err2) + } + defer func() { + if err == nil { + err = utils.CombineErrors(err, txn.Commit()) + } else { + err = utils.CombineErrors(err, txn.Rollback()) + } + }() + + stmt, err2 := txn.Prepare(pq.CopyIn("pathdata", "bucket", "fullpath", "metadata")) + if err2 != nil { + return errs.New("Failed to initialize COPY FROM: %v", err) + } + defer func() { + err2 := stmt.Close() + if err2 != nil { + err = utils.CombineErrors(err, errs.New("Failed to close COPY FROM statement: %v", err2)) + } + }() + + var item storage.ListItem + for iter.Next(&item) { + if _, err := stmt.Exec([]byte(""), []byte(item.Key), []byte(item.Value)); err != nil { + return err + } + } + if _, err = stmt.Exec(); err != nil { + return errs.New("Failed to complete COPY FROM: %v", err) + } + return nil +} + +func bulkDelete(db *sql.DB) error { + _, err := db.Exec("TRUNCATE pathdata") + if err != nil { + return errs.New("Failed to TRUNCATE pathdata table: %v", err) + } + return nil +} + +type pgLongBenchmarkStore struct { + *Client +} + +func (store *pgLongBenchmarkStore) BulkImport(iter storage.Iterator) error { + return bulkImport(store.pgConn, iter) +} + +func (store *pgLongBenchmarkStore) BulkDelete() error { + return bulkDelete(store.pgConn) +} + +func BenchmarkSuiteLong(b *testing.B) { + store, cleanup := newTestPostgres(b) + defer cleanup() + + testsuite.BenchmarkPathOperationsInLargeDb(b, &pgLongBenchmarkStore{store}) +} diff --git a/storage/postgreskv/common.go b/storage/postgreskv/common.go new file mode 100644 index 000000000..648b36d27 --- /dev/null +++ b/storage/postgreskv/common.go @@ -0,0 +1,11 @@ +// Copyright (C) 2018 Storj Labs, Inc. +// See LICENSE for copying information. + +package postgreskv + +import ( + "github.com/zeebo/errs" +) + +// Error is the default postgreskv errs class +var Error = errs.Class("postgreskv error") diff --git a/storage/postgreskv/schema/2018092201_initial-tables.down.sql b/storage/postgreskv/schema/2018092201_initial-tables.down.sql new file mode 100644 index 000000000..c43ff05c9 --- /dev/null +++ b/storage/postgreskv/schema/2018092201_initial-tables.down.sql @@ -0,0 +1,9 @@ +DROP FUNCTION list_directory_reverse(BYTEA, BYTEA, BYTEA, INTEGER); +DROP FUNCTION list_directory(BYTEA, BYTEA, BYTEA, INTEGER); +DROP TYPE path_and_meta; +DROP FUNCTION component_increment(BYTEA, INTEGER); +DROP FUNCTION bytea_increment(BYTEA); +DROP FUNCTION truncate_after(BYTEA, INTEGER, INTEGER); +DROP VIEW pathdata_pretty; +DROP TABLE pathdata; +DROP TABLE buckets; diff --git a/storage/postgreskv/schema/2018092201_initial-tables.up.sql b/storage/postgreskv/schema/2018092201_initial-tables.up.sql new file mode 100644 index 000000000..c45ef7bf1 --- /dev/null +++ b/storage/postgreskv/schema/2018092201_initial-tables.up.sql @@ -0,0 +1,194 @@ +CREATE TABLE buckets ( + bucketname BYTEA + PRIMARY KEY, + delim INT + NOT NULL + CHECK (delim > 0 AND delim < 255) +); + +-- until the KeyValueStore interface supports passing the bucket separately, or +-- until storj actually supports changing the delimiter character per bucket, this +-- dummy row should suffice for everything. +INSERT INTO buckets (bucketname, delim) VALUES (''::BYTEA, ascii('/')); + + +CREATE TABLE pathdata ( + bucket BYTEA + NOT NULL + REFERENCES buckets (bucketname), + fullpath BYTEA + NOT NULL + CHECK (fullpath <> ''), + metadata BYTEA + NOT NULL, + + PRIMARY KEY (bucket, fullpath) +); + +CREATE VIEW pathdata_pretty AS + SELECT encode(bucket, 'escape') AS bucket, + encode(fullpath, 'escape') AS fullpath, + encode(metadata, 'escape') AS metadata + FROM pathdata; + + +-- given a path as might be found in the pathdata table, truncate it after the next delimiter to be +-- found at or after 'afterpos', if any. +-- +-- Examples: +-- +-- truncate_after(''::BYTEA, ascii('/'), 1) -> ''::BYTEA +-- truncate_after('foo'::BYTEA, ascii('/'), 1) -> 'foo'::BYTEA +-- truncate_after('foo/'::BYTEA, ascii('/'), 1) -> 'foo/'::BYTEA +-- truncate_after('foo/bar/baz'::BYTEA, ascii('/'), 4) -> 'foo/'::BYTEA +-- truncate_after('foo/bar/baz'::BYTEA, ascii('/'), 5) -> 'foo/bar/'::BYTEA +-- truncate_after('foo/bar/baz'::BYTEA, ascii('/'), 8) -> 'foo/bar/'::BYTEA +-- truncate_after('foo/bar/baz'::BYTEA, ascii('/'), 9) -> 'foo/bar/baz'::BYTEA +-- truncate_after('foo//bar/bz'::BYTEA, ascii('/'), 4) -> 'foo/'::BYTEA +-- truncate_after('foo//bar/bz'::BYTEA, ascii('/'), 5) -> 'foo//'::BYTEA +-- +CREATE FUNCTION truncate_after(bpath BYTEA, delim INTEGER, afterpos INTEGER) RETURNS BYTEA AS $$ +DECLARE + suff BYTEA; + delimpos INTEGER; +BEGIN + suff := substring(bpath FROM afterpos); + delimpos := position(set_byte(' '::BYTEA, 0, delim) IN suff); + IF delimpos > 0 THEN + RETURN substring(bpath FROM 1 FOR (afterpos + delimpos - 1)); + END IF; + RETURN bpath; +END; +$$ LANGUAGE 'plpgsql' IMMUTABLE STRICT; + + +CREATE FUNCTION bytea_increment(b BYTEA) RETURNS BYTEA AS $$ +BEGIN + WHILE b <> ''::BYTEA AND get_byte(b, octet_length(b) - 1) = 255 LOOP + b := substring(b FROM 1 FOR octet_length(b) - 1); + END LOOP; + IF b = ''::BYTEA THEN + RETURN NULL; + END IF; + RETURN set_byte(b, octet_length(b) - 1, get_byte(b, octet_length(b) - 1) + 1); +END; +$$ LANGUAGE 'plpgsql' IMMUTABLE STRICT; + + +-- Given a path as might be found in the pathdata table, with a delimeter appended if that path +-- has any sub-elements, return the next possible path that _could_ be in the table (skipping over +-- any potential sub-elements). +-- +-- Examples: +-- +-- component_increment('/'::BYTEA, ascii('/')) -> '0'::BYTEA +-- (nothing can be between '/' and '0' other than subpaths under '/') +-- +-- component_increment('/foo/bar/'::BYTEA, ascii('/')) -> '/foo/bar0'::BYTEA +-- +-- component_increment('/foo/barboom'::BYTEA, ascii('/')) -> ('/foo/barboom' || E'\\x00') +-- (nothing can be between '/foo/barboom' and '/foo/barboom\x00' in normal BYTEA ordering) +-- +-- component_increment(E'\\xFEFFFF'::BYTEA, 255) -> E'\\xFF'::BYTEA +-- +CREATE FUNCTION component_increment(bpath BYTEA, delim INTEGER) RETURNS BYTEA AS $$ + SELECT CASE WHEN get_byte(bpath, octet_length(bpath) - 1) = delim + THEN CASE WHEN delim = 255 + THEN bytea_increment(bpath) + ELSE set_byte(bpath, octet_length(bpath) - 1, delim + 1) + END + ELSE bpath || E'\\x00'::BYTEA + END; +$$ LANGUAGE 'sql' IMMUTABLE STRICT; + + +CREATE TYPE path_and_meta AS ( + fullpath BYTEA, + metadata BYTEA +); + +CREATE FUNCTION list_directory(bucket BYTEA, dirpath BYTEA, start_at BYTEA = ''::BYTEA, limit_to INTEGER = NULL) +RETURNS SETOF path_and_meta AS $$ + WITH RECURSIVE + inputs AS ( + SELECT CASE WHEN dirpath = ''::BYTEA THEN NULL ELSE dirpath END AS range_low, + CASE WHEN dirpath = ''::BYTEA THEN NULL ELSE bytea_increment(dirpath) END AS range_high, + octet_length(dirpath) + 1 AS component_start, + b.delim AS delim, + b.bucketname AS bucket + FROM buckets b + WHERE bucketname = bucket + ), + distinct_prefix (truncatedpath) AS ( + SELECT (SELECT truncate_after(pd.fullpath, i.delim, i.component_start) + FROM pathdata pd + WHERE (i.range_low IS NULL OR pd.fullpath > i.range_low) + AND (i.range_high IS NULL OR pd.fullpath < i.range_high) + AND (start_at = '' OR pd.fullpath >= start_at) + AND pd.bucket = i.bucket + ORDER BY pd.fullpath + LIMIT 1) + FROM inputs i + UNION ALL + SELECT (SELECT truncate_after(pd.fullpath, i.delim, i.component_start) + FROM pathdata pd + WHERE pd.fullpath >= component_increment(pfx.truncatedpath, i.delim) + AND (i.range_high IS NULL OR pd.fullpath < i.range_high) + AND pd.bucket = i.bucket + ORDER BY pd.fullpath + LIMIT 1) + FROM distinct_prefix pfx, inputs i + WHERE pfx.truncatedpath IS NOT NULL + ) + SELECT pfx.truncatedpath AS fullpath, + pd.metadata + FROM distinct_prefix pfx LEFT OUTER JOIN pathdata pd ON pfx.truncatedpath = pd.fullpath + WHERE pfx.truncatedpath IS NOT NULL + UNION ALL + -- this one, if it exists, can't be part of distinct_prefix (or it would cause us to skip over all + -- subcontents of the prefix we're looking for), so we tack it on here + SELECT pd.fullpath, pd.metadata FROM pathdata pd, inputs i WHERE pd.fullpath = i.range_low + ORDER BY fullpath + LIMIT limit_to; +$$ LANGUAGE 'sql' STABLE; + +CREATE FUNCTION list_directory_reverse(bucket BYTEA, dirpath BYTEA, start_at BYTEA = ''::BYTEA, limit_to INTEGER = NULL) +RETURNS SETOF path_and_meta AS $$ + WITH RECURSIVE + inputs AS ( + SELECT CASE WHEN dirpath = ''::BYTEA THEN NULL ELSE dirpath END AS range_low, + CASE WHEN dirpath = ''::BYTEA THEN NULL ELSE bytea_increment(dirpath) END AS range_high, + octet_length(dirpath) + 1 AS component_start, + b.delim AS delim, + b.bucketname AS bucket + FROM buckets b + WHERE bucketname = bucket + ), + distinct_prefix (truncatedpath) AS ( + SELECT (SELECT truncate_after(pd.fullpath, i.delim, i.component_start) + FROM pathdata pd + WHERE (i.range_low IS NULL OR pd.fullpath >= i.range_low) + AND (i.range_high IS NULL OR pd.fullpath < i.range_high) + AND (start_at = '' OR pd.fullpath <= start_at) + AND pd.bucket = i.bucket + ORDER BY pd.fullpath DESC + LIMIT 1) + FROM inputs i + UNION ALL + SELECT (SELECT truncate_after(pd.fullpath, i.delim, i.component_start) + FROM pathdata pd + WHERE (i.range_low IS NULL OR pd.fullpath >= i.range_low) + AND pd.fullpath < pfx.truncatedpath + AND pd.bucket = i.bucket + ORDER BY pd.fullpath DESC + LIMIT 1) + FROM distinct_prefix pfx, inputs i + WHERE pfx.truncatedpath IS NOT NULL + ) + SELECT pfx.truncatedpath AS fullpath, + pd.metadata + FROM distinct_prefix pfx LEFT OUTER JOIN pathdata pd ON pfx.truncatedpath = pd.fullpath + WHERE pfx.truncatedpath IS NOT NULL + ORDER BY fullpath DESC + LIMIT limit_to; +$$ LANGUAGE 'sql' STABLE; diff --git a/storage/postgreskv/schema/data.go b/storage/postgreskv/schema/data.go new file mode 100644 index 000000000..97e68973e --- /dev/null +++ b/storage/postgreskv/schema/data.go @@ -0,0 +1,256 @@ +// Code generated by go-bindata. DO NOT EDIT. +// sources: +// 2018092201_initial-tables.down.sql +// 2018092201_initial-tables.up.sql +package schema + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" +) + +func bindataRead(data []byte, name string) ([]byte, error) { + gz, err := gzip.NewReader(bytes.NewBuffer(data)) + if err != nil { + return nil, fmt.Errorf("Read %q: %v", name, err) + } + + var buf bytes.Buffer + _, err = io.Copy(&buf, gz) + clErr := gz.Close() + + if err != nil { + return nil, fmt.Errorf("Read %q: %v", name, err) + } + if clErr != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +type asset struct { + bytes []byte + info os.FileInfo +} + +type bindataFileInfo struct { + name string + size int64 + mode os.FileMode + modTime time.Time +} + +func (fi bindataFileInfo) Name() string { + return fi.name +} +func (fi bindataFileInfo) Size() int64 { + return fi.size +} +func (fi bindataFileInfo) Mode() os.FileMode { + return fi.mode +} +func (fi bindataFileInfo) ModTime() time.Time { + return fi.modTime +} +func (fi bindataFileInfo) IsDir() bool { + return false +} +func (fi bindataFileInfo) Sys() interface{} { + return nil +} + +var __2018092201_initialTablesDownSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\x8f\xc1\x0a\x82\x40\x10\x86\xef\x3e\xc5\x1e\x0d\x7a\x03\x4f\x5a\x5b\x08\xa1\x22\x5b\xe1\x69\x18\xd7\x89\xa4\x5c\x65\x1c\x03\xdf\x3e\x08\x0a\x54\xa8\x4e\x03\x33\xff\x7c\x7c\xff\x36\x4f\x33\xb5\x3b\x26\x1b\x13\xa7\x89\xba\xd7\xbd\x40\x55\x33\x59\x69\x79\x04\xa6\x07\x71\x4f\x7e\x54\x18\x1d\xae\xd5\x74\xc4\x89\xd1\x7b\x9d\xaf\x02\xef\x1b\xe3\xaf\x5f\x53\x64\x5a\x75\x28\x57\x40\x57\x41\x43\x82\x73\xa8\x6d\x9b\xae\x75\xe4\x04\x6a\x67\x99\x1a\x72\xe2\xff\xf0\x28\x47\x21\x9c\xc7\x17\x29\xe1\xc1\x59\x14\x02\xbc\x08\xf1\x8c\xb9\x80\x9f\x62\x7d\x7e\x89\x56\x28\x08\x1d\x93\xc8\xf8\xee\x10\x46\x07\xfd\xb9\x4d\x96\xe5\x60\x6f\x24\x7d\xe0\x3d\x03\x00\x00\xff\xff\x96\xd8\x40\x70\x6d\x01\x00\x00") + +func _2018092201_initialTablesDownSqlBytes() ([]byte, error) { + return bindataRead( + __2018092201_initialTablesDownSql, + "2018092201_initial-tables.down.sql", + ) +} + +func _2018092201_initialTablesDownSql() (*asset, error) { + bytes, err := _2018092201_initialTablesDownSqlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "2018092201_initial-tables.down.sql", size: 365, mode: os.FileMode(420), modTime: time.Unix(1539025200, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var __2018092201_initialTablesUpSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x58\x6d\x6f\xdb\xba\x15\xfe\xee\x5f\x71\x3e\x14\x90\x85\x2b\x27\xe9\xb0\x02\x5b\x52\x17\x70\x1d\x3a\xd1\xea\xc8\x85\xac\x34\x08\x70\x01\x83\x92\x68\x9b\xab\x4c\x6a\x22\xd5\xc4\xc3\xfd\xf1\x03\xa9\x37\x4a\x96\x9d\xdc\xad\x28\x0a\xac\xfa\x12\x47\x3a\x7c\xce\xe1\x73\x0e\xcf\x0b\xa7\x3e\x9a\x04\x08\x82\xc9\xc7\x39\x82\x30\x8f\xbe\x12\x29\x60\x38\x00\x80\xf2\x3f\x86\x77\x04\x3e\x3e\x06\x68\xa2\x5f\xaa\xe7\xb3\xef\xde\x4d\xfc\x47\xf8\x84\x1e\x1d\xfd\x32\x26\x09\xdd\x81\xeb\x05\xb5\x88\xb7\x08\xc0\xbb\x9f\xcf\xeb\x17\xd3\x5b\x34\xfd\x04\xc3\x42\xf2\x03\x5c\xc0\xc4\xbb\x2e\xd7\xbd\x87\xbf\xbc\x7b\x67\x0f\xec\xab\xc1\x60\x34\x82\x9c\x49\x9a\x80\xdc\x12\xf8\x44\xf6\x5f\x70\x92\x93\xa5\xe4\x19\x01\xca\x24\xc9\xd6\x38\x22\x20\xf2\x34\xe5\x99\x14\x90\x62\x21\x28\xdb\x68\xe1\xc2\x58\x10\x24\xc5\x19\x96\x24\xd9\x3b\xc0\xb3\x06\x4f\x48\x9e\xfd\x13\x70\x24\x73\x9c\x24\xfb\x06\x22\xda\x62\xb6\xa9\x30\xb4\x39\x54\x92\x4c\xbd\xce\x70\xa4\x7e\xa5\x24\x2b\xb1\x1d\x90\x5b\x2a\x14\x64\x9c\xef\x76\x7b\xc8\xf8\x13\x88\x2d\xcf\x93\x18\x44\xbe\x5e\xd3\x88\xc0\x9a\x67\x40\xbe\x91\x6c\x2f\xb7\x94\x6d\xce\x06\xae\xb7\x44\x7e\xa0\x88\x59\x34\xdc\x36\xb4\x3a\x85\x46\x1b\xbe\x4c\xe6\xf7\x68\x09\x43\xcb\xba\xbc\xd4\x54\x3b\x80\x45\x44\xe9\xd0\x3a\xb7\x6c\xc5\xcb\xa0\xe5\xa6\x14\xcb\x6d\x8c\x25\x6e\xf9\xa9\xe3\xa3\x03\x07\xf8\x68\x86\x7c\xe4\x4d\xd1\xb2\xcf\x14\xbb\x70\xe4\x3a\x4f\x12\x85\xfe\x12\x58\xe9\xcd\x5a\xfc\xfd\x07\xb0\xac\x12\x63\x47\x24\xd6\xd6\xf5\x63\x38\x83\x41\x27\x86\x2a\x3b\x9c\x5a\x7d\x11\x0c\xe5\x9e\xbf\xb8\xe8\xa1\xde\xf2\x2a\xcd\x88\x94\x7b\x98\x2c\x35\xca\x12\xcd\xd1\x34\x00\xc2\x22\x1e\x93\x1a\xc6\x22\x22\xc2\x29\xb1\x6c\x98\x54\x9b\x75\x6a\x3b\x00\x2a\xf1\x4a\x5b\x67\x41\xfd\xba\x67\x49\xb5\xb7\xce\x92\xea\x75\xb9\x62\xe6\x2f\xee\x6a\x8b\x95\xf7\x46\x23\xd8\xd0\x6f\x84\x01\xd6\xaf\x01\x0b\xd8\xd1\xcd\x56\x42\xa8\x62\x26\x67\x31\x50\xa6\x23\xb0\xf6\xac\xc4\x61\x42\x1c\x90\x59\xce\x22\x2c\x09\x50\x09\x78\xad\xe2\x51\x49\x31\xf2\x2c\x8d\x60\x95\x1c\x42\xa2\x74\x14\x50\x58\x02\xcf\x4a\x69\x4b\xff\x49\xb9\xb0\x1c\xa0\x6b\xc0\x6c\x7f\x36\x18\x8d\x94\x2c\x7a\xc6\xbb\x34\x21\xe2\xb2\xfc\x5f\x3d\x95\xb6\x95\x5e\x65\x46\xa3\xf1\x34\x81\xe9\xc0\x5b\x1b\x60\xf4\x01\xa0\x96\x3c\x86\xb4\xe6\xfc\x10\xac\x17\xc9\x90\x3c\x01\x76\x7e\x80\x76\x0c\xec\xfc\x35\x68\x21\xce\xce\x43\xfc\xef\xbe\xd3\xe7\xc0\x5f\xbf\x2b\xda\x3b\x13\x4d\x49\xfe\xcf\x88\x7f\xfb\xee\x88\x7f\xef\x22\x1a\x92\xa7\x40\x0b\xd9\xef\x45\xe3\x49\xb4\x16\x8d\x26\x5c\x95\x35\x66\xf7\xde\x34\x70\x17\x5e\x17\x3a\x6c\xd2\x9b\xd3\xd4\x2d\x74\x83\x7c\x07\xaa\xc3\x52\xbd\xb1\xc1\x47\xc1\xbd\xef\x2d\x0b\x79\x75\xd2\xdf\xbc\x19\x5c\xa3\xe9\x7c\xe2\x23\x7d\xd6\x55\xe2\x2f\x3e\x5e\x35\x85\xd0\x40\xb8\x1a\x7c\x44\x37\xae\xd7\xc8\x5e\x8e\x41\xe4\xa1\x90\x19\x65\x9b\xd2\x16\x9d\x2d\x2a\xd5\x76\x07\xe7\x72\x0c\x29\x17\x54\x52\xce\x86\x82\xc8\x55\xb8\x97\x64\x68\x41\x43\xca\x45\x5d\x44\x5c\x4f\xab\x28\x11\xdc\x59\x03\xa2\xca\x6d\x70\x8b\x3c\xa3\x16\xa8\x6d\xf5\x5b\xf2\x16\x66\x0b\x1f\x86\x35\x17\xbf\x35\x38\x23\x78\x6b\x97\xf0\xc8\xbb\x06\x77\x56\xfc\x2e\xd1\x34\xc6\xd5\x00\x79\xd7\x57\x83\x37\x6f\x60\x3e\xf1\x6e\xee\x27\x37\x08\xac\x34\x49\x37\xe2\x5f\x89\x05\xee\xdd\xdd\x7d\x51\xc1\x96\x81\xef\x4e\x03\xa3\xb0\xd5\xee\x52\x1b\xc4\x2b\xca\xa2\x8c\xec\x08\x93\xc3\xb0\xa0\xb7\xdf\x15\x0d\xb9\x0f\xb7\xae\xea\x5f\x8a\x42\x54\x92\xa3\x5b\x8c\x4d\x45\x5a\xe8\x00\x8f\x24\x91\xab\x84\xb0\x8d\xdc\x0e\x43\x5b\xef\x07\xc6\xaa\xfb\x80\xf9\x62\xf1\xb9\xa6\x27\xec\xb8\xc9\x24\xa6\x0f\xa3\xa1\x44\xc1\xd4\xfc\x87\x30\x36\x8c\xe9\x73\x80\xaa\x87\x47\xf9\x14\xa7\x2d\x77\x5e\xde\xda\x6f\xda\xb6\x3f\xe9\x90\xd1\x08\x6e\xfe\xab\x5a\xf5\x44\x95\x74\x11\x2d\x44\x15\x1f\x9c\xa6\x84\xc5\x24\x56\x95\x47\x6e\xb1\xd4\x2b\x14\xfe\x16\x0b\x55\x8a\x14\xc5\x23\x92\x68\x3f\x0b\x07\x32\x22\xf3\x8c\x35\x05\x2e\xe5\x42\xd0\x30\x29\x14\x15\x00\xab\x48\xb5\x5b\x2b\x65\x4a\x69\x84\xd6\x0d\x43\xf1\x95\xa6\xa9\x6a\xe3\xf8\x37\xa2\xbb\x3e\x05\x9f\x72\x49\x98\xa4\x38\x69\x29\xb2\x4f\x55\xc0\x88\xef\x52\xce\x08\x93\x46\x04\x5a\xe7\xbd\x5d\x59\x99\x7f\x2e\xba\xa9\x4c\x3d\x43\xc6\x75\x07\x08\x11\x66\xca\xd8\x90\xc8\x27\x42\x18\x58\xe7\x16\x60\x16\xab\x55\xc0\xe5\x56\x97\x73\xcc\x94\x79\x6a\x8f\x02\x72\x16\xab\xa2\x7d\x6e\xd9\x2f\xda\xd4\x4d\xf6\x7d\xb6\x55\x42\xa6\x8d\xaf\x42\x0d\x39\xdf\x9d\x00\xee\x08\xc2\x1f\x7f\x00\xb2\x7e\xff\xfd\xf9\xe2\x42\x1b\xfe\x0a\x1a\x5a\xcb\x35\x23\xe6\x2b\x8d\xa4\x3c\xcc\x78\xb6\xc3\x49\x79\xe4\x79\x16\x13\x75\x1e\x5f\xe2\x46\x9b\x32\x43\xb3\xd9\x6c\xd6\xec\x41\x4d\x18\x85\xf1\xc5\xe7\xd9\xa9\x92\xd1\x87\x7a\xbc\x6e\xf4\xa7\x26\xa3\x3b\x9d\x4e\x96\x08\x1e\x6e\x91\x67\x9c\xd8\xa2\xef\x6c\x9f\x5a\xdd\xf9\x56\x49\x49\xab\x30\x3b\x50\xfd\xa8\x24\x62\xe0\x15\x76\xe8\x0c\x76\x20\xda\x59\x74\x90\x57\x8b\x3e\xfb\xf8\x2a\x34\x5f\x22\x23\x07\x9d\x34\xb8\x62\x44\x25\x9c\x7e\x48\xe4\x5d\x1f\x7c\xd0\x1a\x0a\x5e\x8d\x08\xaa\xfc\xd2\x5e\xdc\xc9\x60\x2f\x95\x93\xe0\xf1\x73\x31\x26\xad\x30\x8b\x57\xaa\x3d\x57\x6e\x19\xf6\xcc\x38\xbd\x33\x8b\x31\x7d\xd4\x41\x91\x50\x21\x57\x31\xcd\x48\x24\x79\xb6\x1f\x9a\x43\x97\x03\x31\xcd\xcc\xf8\x10\x12\x67\x72\x85\xcb\xcf\x30\x2e\x06\x1f\xd0\x0d\xfb\x4a\xf2\x2a\x72\xca\x0f\xf6\xa0\x0a\xa0\x25\x0a\x16\xb3\x43\xbb\xcb\x70\x7a\x70\x83\x5b\xf0\xd1\xf4\xde\x5f\xba\x5f\x50\x4d\x10\x65\x69\x2e\x45\xb3\xbf\xea\x39\x88\xbe\xca\xc8\x6e\x55\xd2\x46\x14\xde\xa8\x44\x54\x39\x9a\x2c\x21\xc3\x6c\x43\x56\x09\x7f\x72\xfa\xbc\xfa\xa7\x90\xbb\xf1\x57\x2e\xb1\xdb\xaa\xb6\x74\xb3\xed\xd5\xd5\x0a\xbc\x7a\xed\x6f\xf0\x56\xad\x6d\xce\xab\x26\xbe\x17\x20\x3c\x2b\x42\x74\xb2\x2c\x62\xf5\x88\x90\x71\xe7\x51\xcf\x8e\x1d\x49\xdd\x0c\x54\x23\x74\xd8\xfe\xf8\x70\x8b\x7c\x64\x5e\x9c\x8c\xbb\x18\x76\xa3\x38\xa6\x42\x52\x16\x49\x35\xd3\xae\xe9\x33\x0c\xab\x56\x35\x2e\x76\x77\xcc\xa3\xc3\xf2\x6f\xa7\xb3\x4d\xe3\xb3\x66\xa2\xa5\xc5\x76\xd5\x8f\x0e\x3b\xc7\xce\x7c\x6b\x68\x85\x34\xee\x17\x2b\x36\x38\xa4\x67\x75\x68\x80\xbb\x2c\xdc\xbc\xf0\xc1\x30\x01\x3e\x80\x21\x74\x34\xd1\xa8\x16\xad\x46\x53\xde\x3f\x06\xf7\x1e\x4c\xa9\xd3\x78\xf5\xf1\x53\xf1\x78\x60\xd7\xb8\x3e\x9e\x27\x51\xd2\xb8\x0c\x06\x18\x03\x3d\xeb\x8d\x84\xf2\x59\xf8\xd7\xc8\x87\x8f\x8f\xa6\x9a\x7e\xc9\xb9\x7b\xe7\x06\x87\x39\x52\x53\x5f\x1e\x63\xda\xfa\x76\xef\xa9\xd4\x33\x31\x6e\x5e\xe0\x67\x8a\x83\x0e\xaf\x7d\x75\x33\x5d\x3f\x9f\xb5\xe2\xba\xb6\xe9\x87\xc6\xc4\x8f\xf6\x66\xf7\x6c\xa7\xeb\x67\xa7\xdf\xc5\x15\x95\x5d\x9e\xf4\xa6\xbb\xf7\x6e\xb6\xd9\x57\x1c\x2e\x39\x76\x7b\x95\xc6\x67\x7d\x57\x54\x3d\x46\xc2\x1c\xcd\x02\x58\xdc\x07\xc8\x87\x7f\x2c\x5c\xcf\x8c\x04\x58\x78\x3d\x3a\xc7\x87\x44\xbd\x76\x47\xed\xe8\x1e\x8d\xf4\x05\x2b\x70\x46\xf4\x65\x15\x95\x40\x9e\xa9\x50\xa3\x41\x84\x99\xa5\x07\x90\x14\x67\x12\xf8\xfa\x30\x75\xf2\x4c\xc9\x3f\xe9\xdb\xd8\x08\xe7\x82\x40\x2e\x40\x72\x50\x83\x81\x1e\x0a\x00\x27\x49\xa5\x46\xe4\x61\xc4\x99\x1a\x0e\x84\x02\xd3\xc3\x4c\x81\xf3\x44\xac\x8c\x40\xc2\xf9\x57\xd5\xb6\xae\x79\x66\x3b\x20\x38\x3c\xa9\x41\x23\xfa\xaa\x54\x70\x06\x5b\x92\x91\x96\x1f\xcc\x03\x67\x50\x7d\x70\x94\x9a\x08\xe8\x39\x3f\x63\x33\x5f\x0e\xda\xa1\xd8\xa6\xb7\x88\xbb\xaa\x99\xe8\xeb\x8c\x96\xba\x2d\x7a\xb1\x89\x59\x65\xe4\x1b\xc9\x04\xf9\xd5\xcc\xfc\x6a\x66\x7e\x35\x33\x27\x9b\x99\xf1\x4f\xda\xcd\xbc\xff\x21\xdd\x0c\x5c\xa3\xe5\xf4\xff\xa3\xa5\xf9\xce\xd1\xd0\x76\xfa\x41\x49\xfc\xe9\x5c\xf6\xab\x6f\x79\xcd\x8e\x0e\x0a\xb3\xc1\xf6\xeb\xab\xf3\x7f\x02\x00\x00\xff\xff\xa9\xf1\x2d\xb9\x7b\x1f\x00\x00") + +func _2018092201_initialTablesUpSqlBytes() ([]byte, error) { + return bindataRead( + __2018092201_initialTablesUpSql, + "2018092201_initial-tables.up.sql", + ) +} + +func _2018092201_initialTablesUpSql() (*asset, error) { + bytes, err := _2018092201_initialTablesUpSqlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "2018092201_initial-tables.up.sql", size: 8059, mode: os.FileMode(420), modTime: time.Unix(1539025200, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +// Asset loads and returns the asset for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func Asset(name string) ([]byte, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err) + } + return a.bytes, nil + } + return nil, fmt.Errorf("Asset %s not found", name) +} + +// MustAsset is like Asset but panics when Asset would return an error. +// It simplifies safe initialization of global variables. +func MustAsset(name string) []byte { + a, err := Asset(name) + if err != nil { + panic("asset: Asset(" + name + "): " + err.Error()) + } + + return a +} + +// AssetInfo loads and returns the asset info for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func AssetInfo(name string) (os.FileInfo, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err) + } + return a.info, nil + } + return nil, fmt.Errorf("AssetInfo %s not found", name) +} + +// AssetNames returns the names of the assets. +func AssetNames() []string { + names := make([]string, 0, len(_bindata)) + for name := range _bindata { + names = append(names, name) + } + return names +} + +// _bindata is a table, holding each asset generator, mapped to its name. +var _bindata = map[string]func() (*asset, error){ + "2018092201_initial-tables.down.sql": _2018092201_initialTablesDownSql, + "2018092201_initial-tables.up.sql": _2018092201_initialTablesUpSql, +} + +// AssetDir returns the file names below a certain +// directory embedded in the file by go-bindata. +// For example if you run go-bindata on data/... and data contains the +// following hierarchy: +// data/ +// foo.txt +// img/ +// a.png +// b.png +// then AssetDir("data") would return []string{"foo.txt", "img"} +// AssetDir("data/img") would return []string{"a.png", "b.png"} +// AssetDir("foo.txt") and AssetDir("notexist") would return an error +// AssetDir("") will return []string{"data"}. +func AssetDir(name string) ([]string, error) { + node := _bintree + if len(name) != 0 { + cannonicalName := strings.Replace(name, "\\", "/", -1) + pathList := strings.Split(cannonicalName, "/") + for _, p := range pathList { + node = node.Children[p] + if node == nil { + return nil, fmt.Errorf("Asset %s not found", name) + } + } + } + if node.Func != nil { + return nil, fmt.Errorf("Asset %s not found", name) + } + rv := make([]string, 0, len(node.Children)) + for childName := range node.Children { + rv = append(rv, childName) + } + return rv, nil +} + +type bintree struct { + Func func() (*asset, error) + Children map[string]*bintree +} +var _bintree = &bintree{nil, map[string]*bintree{ + "2018092201_initial-tables.down.sql": &bintree{_2018092201_initialTablesDownSql, map[string]*bintree{}}, + "2018092201_initial-tables.up.sql": &bintree{_2018092201_initialTablesUpSql, map[string]*bintree{}}, +}} + +// RestoreAsset restores an asset under the given directory +func RestoreAsset(dir, name string) error { + data, err := Asset(name) + if err != nil { + return err + } + info, err := AssetInfo(name) + if err != nil { + return err + } + err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755)) + if err != nil { + return err + } + err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) + if err != nil { + return err + } + err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) + if err != nil { + return err + } + return nil +} + +// RestoreAssets restores an asset under the given directory recursively +func RestoreAssets(dir, name string) error { + children, err := AssetDir(name) + // File + if err != nil { + return RestoreAsset(dir, name) + } + // Dir + for _, child := range children { + err = RestoreAssets(dir, filepath.Join(name, child)) + if err != nil { + return err + } + } + return nil +} + +func _filePath(dir, name string) string { + cannonicalName := strings.Replace(name, "\\", "/", -1) + return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) +} + diff --git a/storage/postgreskv/schema/migrate.go b/storage/postgreskv/schema/migrate.go new file mode 100644 index 000000000..6b1fddbf8 --- /dev/null +++ b/storage/postgreskv/schema/migrate.go @@ -0,0 +1,44 @@ +// Copyright (C) 2018 Storj Labs, Inc. +// See LICENSE for copying information. + +//go:generate go-bindata -o data.go -pkg schema -ignore ".*go" . + +package schema + +import ( + "database/sql" + + // this isn't strictly necessary to import, but we just need the go-bindata + // binary for the above generate, and go mod tidy wants to remove + // go-bindata because nothing else is importing it. better solutions? + _ "github.com/go-bindata/go-bindata" + + "github.com/golang-migrate/migrate/v3" + "github.com/golang-migrate/migrate/v3/database/postgres" + "github.com/golang-migrate/migrate/v3/source/go_bindata" +) + +// PrepareDB applies schema migrations as necessary to the given database to +// get it up to date. +func PrepareDB(db *sql.DB) error { + srcDriver, err := bindata.WithInstance(bindata.Resource(AssetNames(), + func(name string) ([]byte, error) { + return Asset(name) + })) + if err != nil { + return err + } + dbDriver, err := postgres.WithInstance(db, &postgres.Config{}) + if err != nil { + return err + } + m, err := migrate.NewWithInstance("go-bindata migrations", srcDriver, "postgreskv db", dbDriver) + if err != nil { + return err + } + err = m.Up() + if err == migrate.ErrNoChange { + err = nil + } + return err +} diff --git a/storage/redis/client.go b/storage/redis/client.go index 51c84a077..623536878 100644 --- a/storage/redis/client.go +++ b/storage/redis/client.go @@ -117,9 +117,9 @@ func (client *Client) Close() error { return client.db.Close() } -// GetAll is the bulk method for gets from the redis data store -// The maximum keys returned will be 100. If more than that is requested an -// error will be returned +// GetAll is the bulk method for gets from the redis data store. +// The maximum keys returned will be storage.LookupLimit. If more than that +// is requested, an error will be returned func (client *Client) GetAll(keys storage.Keys) (storage.Values, error) { if len(keys) > storage.LookupLimit { return nil, storage.ErrLimitExceeded diff --git a/storage/testsuite/long_bench.go b/storage/testsuite/long_bench.go new file mode 100644 index 000000000..5d4519723 --- /dev/null +++ b/storage/testsuite/long_bench.go @@ -0,0 +1,775 @@ +// Copyright (C) 2018 Storj Labs, Inc. +// See LICENSE for copying information. + +package testsuite + +import ( + "bufio" + "bytes" + "compress/gzip" + "flag" + "fmt" + "io" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/zeebo/errs" + + "storj.io/storj/pkg/utils" + "storj.io/storj/storage" +) + +const ( + maxProblems = 10 + + // the largest and deepest level-2 directory in the dataset + largestLevel2Directory = "Peronosporales/hateless/" + + // the directory in the dataset with the most immediate children + largestSingleDirectory = "Peronosporales/hateless/tod/unricht/sniveling/Puyallup/" +) + +var ( + // see https://github.com/storj/test-path-corpus + longBenchmarksData = flag.String("test-bench-long", "", "Run the long benchmark suite against eligible KeyValueStores using the given paths dataset") + + noInitDb = flag.Bool("test-bench-long-noinit", false, "Don't import the large dataset for the long benchmarks; assume it is already loaded") + noCleanDb = flag.Bool("test-bench-long-noclean", false, "Don't clean the long benchmarks KeyValueStore after running, for debug purposes") +) + +func interpolateInput(input []byte) ([]byte, error) { + output := make([]byte, 0, len(input)) + var bytesConsumed int + var next byte + + for pos := 0; pos < len(input); pos += bytesConsumed { + if input[pos] == '\\' { + bytesConsumed = 2 + if pos+1 >= len(input) { + return output, errs.New("encoding error in input: escape at end-of-string") + } + switch input[pos+1] { + case 'x': + if pos+3 >= len(input) { + return output, errs.New("encoding error in input: incomplete \\x escape") + } + nextVal, err := strconv.ParseUint(string(input[pos+2:pos+4]), 16, 8) + if err != nil { + return output, errs.New("encoding error in input: invalid \\x escape: %v", err) + } + next = byte(nextVal) + bytesConsumed = 4 + case 't': + next = '\t' + case 'n': + next = '\n' + case 'r': + next = '\r' + case '\\': + next = '\\' + default: + next = input[pos+1] + } + } else { + next = input[pos] + bytesConsumed = 1 + } + output = append(output, next) + } + return output, nil +} + +// KVInputIterator is passed to the BulkImport method on BulkImporter-satisfying objects. It will +// iterate over a fairly large list of paths that should be imported for testing purposes. +type KVInputIterator struct { + itemNo int + scanner *bufio.Scanner + fileName string + err error + reachedEnd bool + closeFunc func() error +} + +func newKVInputIterator(pathToFile string) (*KVInputIterator, error) { + kvi := &KVInputIterator{fileName: pathToFile} + pathData, err := os.Open(pathToFile) + if err != nil { + return nil, errs.New("Failed to open file with test data (expected at %q): %v", pathToFile, err) + } + var reader io.Reader = pathData + if strings.HasSuffix(pathToFile, ".gz") { + gzReader, err := gzip.NewReader(pathData) + if err != nil { + return nil, utils.CombineErrors( + errs.New("Failed to create gzip reader: %v", err), + pathData.Close()) + } + kvi.closeFunc = func() error { return utils.CombineErrors(gzReader.Close(), pathData.Close()) } + reader = gzReader + } else { + kvi.closeFunc = pathData.Close + } + kvi.scanner = bufio.NewScanner(reader) + return kvi, nil +} + +// Next should be called by BulkImporter instances in order to advance the iterator. It fills in +// a storage.ListItem instance, and returns a boolean indicating whether to continue. When false is +// returned, iteration should stop and nothing is expected to be changed in item. +func (kvi *KVInputIterator) Next(item *storage.ListItem) bool { + if !kvi.scanner.Scan() { + kvi.reachedEnd = true + kvi.err = kvi.scanner.Err() + return false + } + if kvi.err != nil { + return false + } + kvi.itemNo++ + parts := bytes.Split(kvi.scanner.Bytes(), []byte("\t")) + if len(parts) != 3 { + kvi.err = errs.New("Invalid data in %q on line %d: has %d fields", kvi.fileName, kvi.itemNo, len(parts)) + return false + } + k, err := interpolateInput(parts[1]) + if err != nil { + kvi.err = errs.New("Failed to read key data from %q on line %d: %v", kvi.fileName, kvi.itemNo, err) + return false + } + v, err := interpolateInput(parts[2]) + if err != nil { + kvi.err = errs.New("Failed to read value data from %q on line %d: %v", kvi.fileName, kvi.itemNo, err) + return false + } + item.Key = storage.Key(k) + item.Value = storage.Value(v) + item.IsPrefix = false + return true +} + +// Error() returns the last error encountered while iterating over the input file. This must be +// checked after iteration completes, at least. +func (kvi *KVInputIterator) Error() error { + return kvi.err +} + +func openTestData(tb testing.TB) *KVInputIterator { + tb.Helper() + inputIter, err := newKVInputIterator(*longBenchmarksData) + if err != nil { + tb.Fatal(err) + } + return inputIter +} + +// BulkImporter identifies KV storage facilities that can do bulk importing of items more +// efficiently than inserting one-by-one. +type BulkImporter interface { + BulkImport(storage.Iterator) error +} + +// BulkCleaner identifies KV storage facilities that can delete all items efficiently. +type BulkCleaner interface { + BulkDelete() error +} + +// BenchmarkPathOperationsInLargeDb runs the "long benchmarks" suite for KeyValueStore instances. +func BenchmarkPathOperationsInLargeDb(b *testing.B, store storage.KeyValueStore) { + if *longBenchmarksData == "" { + b.Skip("Long benchmarks not enabled.") + } + + initStore(b, store) + + doTest := func(name string, testFunc func(*testing.B, storage.KeyValueStore)) { + b.Run(name, func(bb *testing.B) { + for i := 0; i < bb.N; i++ { + testFunc(bb, store) + } + }) + } + + doTest("DeepRecursive", deepRecursive) + doTest("DeepRecursiveReverse", deepRecursiveReverse) + doTest("DeepNonRecursive", deepNonRecursive) + doTest("DeepNonRecursiveReverse", deepNonRecursiveReverse) + doTest("ShallowRecursive", shallowRecursive) + doTest("ShallowRecursiveReverse", shallowRecursiveReverse) + doTest("ShallowNonRecursive", shallowNonRecursive) + doTest("ShallowNonRecursiveReverse", shallowNonRecursiveReverse) + doTest("TopRecursiveLimit", topRecursiveLimit) + doTest("TopRecursiveLimitReverse", topRecursiveLimitReverse) + doTest("TopRecursiveStartAt", topRecursiveStartAt) + doTest("TopRecursiveStartAtReverse", topRecursiveStartAtReverse) + doTest("TopNonRecursive", topNonRecursive) + doTest("TopNonRecursiveReverse", topNonRecursiveReverse) + + cleanupStore(b, store) +} + +func importBigPathset(tb testing.TB, store storage.KeyValueStore) { + // make sure this is an empty db, or else refuse to run + if !isEmptyKVStore(tb, store) { + tb.Fatal("Provided KeyValueStore is not empty. The long benchmarks are destructive. Not running!") + } + + inputIter := openTestData(tb) + defer func() { + if err := inputIter.closeFunc(); err != nil { + tb.Logf("Failed to close test data stream: %v", err) + } + }() + + importer, ok := store.(BulkImporter) + if ok { + tb.Log("Performing bulk import...") + err := importer.BulkImport(inputIter) + + if err != nil { + errStr := "Provided KeyValueStore failed to import data" + if inputIter.reachedEnd { + errStr += " after iterating over all input data" + } else { + errStr += fmt.Sprintf(" after iterating over %d lines of input data", inputIter.itemNo) + } + tb.Fatalf("%s: %v", errStr, err) + } + } else { + tb.Log("Performing manual import...") + + var item storage.ListItem + for inputIter.Next(&item) { + if err := store.Put(item.Key, item.Value); err != nil { + tb.Fatalf("Provided KeyValueStore failed to insert data (%q, %q): %v", item.Key, item.Value, err) + } + } + } + if err := inputIter.Error(); err != nil { + tb.Fatalf("Failed to iterate over input data during import. Error was %v", err) + } + if !inputIter.reachedEnd { + tb.Fatal("Provided KeyValueStore failed to exhaust input iterator") + } +} + +func initStore(b *testing.B, store storage.KeyValueStore) { + b.Helper() + + if !*noInitDb { + // can't find a way to run the import and cleanup as sub-benchmarks, while still requiring + // that they be run once and only once, and aborting the whole benchmark if import fails. + // we don't want the time it takes to count against the first sub-benchmark only, so we + // stop the timer. however, we do care about the time that import and cleanup take, though, + // so we'll at least log it. + b.StopTimer() + tStart := time.Now() + importBigPathset(b, store) + b.Logf("importing took %s", time.Since(tStart).String()) + b.StartTimer() + } +} + +func cleanupStore(b *testing.B, store storage.KeyValueStore) { + b.Helper() + if !*noCleanDb { + tStart := time.Now() + cleanupBigPathset(b, store) + b.Logf("cleanup took %s", time.Since(tStart).String()) + } +} + +type verifyOpts struct { + iterateOpts storage.IterateOptions + doIterations int + batchSize int + expectCount int + expectLastKey storage.Key +} + +func benchAndVerifyIteration(b *testing.B, store storage.KeyValueStore, opts *verifyOpts) { + problems := 0 + iteration := 0 + + errMsg := func(tmpl string, args ...interface{}) string { + errMsg1 := fmt.Sprintf(tmpl, args...) + return fmt.Sprintf("[on iteration %d/%d, with opts %+v]: %s", iteration, opts.doIterations, opts.iterateOpts, errMsg1) + } + + errorf := func(tmpl string, args ...interface{}) { + b.Error(errMsg(tmpl, args...)) + problems++ + if problems > maxProblems { + b.Fatal("Too many problems") + } + } + + fatalf := func(tmpl string, args ...interface{}) { + b.Fatal(errMsg(tmpl, args...)) + } + + expectRemaining := opts.expectCount + totalFound := 0 + var lastKey storage.Key + var bytesTotal int64 + lookupSize := opts.batchSize + + for iteration = 1; iteration <= opts.doIterations; iteration++ { + results, err := iterateItems(store, opts.iterateOpts, lookupSize) + if err != nil { + fatalf("Failed to call iterateItems(): %v", err) + } + if len(results) == 0 { + // we can't continue to iterate + fatalf("iterateItems() got 0 items") + } + if len(results) > lookupSize { + fatalf("iterateItems() returned _more_ items than limit: %d>%d", len(results), lookupSize) + } + if iteration > 0 && results[0].Key.Equal(lastKey) { + // fine and normal + results = results[1:] + } + expectRemaining -= len(results) + if len(results) != opts.batchSize && expectRemaining != 0 { + errorf("iterateItems read %d items instead of %d", len(results), opts.batchSize) + } + for n, result := range results { + totalFound++ + bytesTotal += int64(len(result.Key)) + int64(len(result.Value)) + if result.Key.IsZero() { + errorf("got an empty key among the results at n=%d!", n) + continue + } + if result.Key.Equal(lastKey) { + errorf("got the same key (%q) twice in a row, not on a lookup boundary!", lastKey) + } + if opts.iterateOpts.Reverse && !lastKey.IsZero() && lastKey.Less(result.Key) { + errorf("KeyValueStore returned items out of order! %q > %q", result.Key, lastKey) + } + if !opts.iterateOpts.Reverse && result.Key.Less(lastKey) { + errorf("KeyValueStore returned items out of order! %q < %q", result.Key, lastKey) + } + if result.IsPrefix { + if !result.Value.IsZero() { + errorf("Expected no metadata for IsPrefix item %q, but got %q", result.Key, result.Value) + } + if result.Key[len(result.Key)-1] != byte('/') { + errorf("Expected key for IsPrefix item %q to end in /, but it does not", result.Key) + } + } else { + valAsNum, err := strconv.ParseUint(string(result.Value), 10, 32) + if err != nil { + errorf("Expected metadata for key %q to hold a decimal integer, but it has %q", result.Key, result.Value) + } else if int(valAsNum) != len(result.Key) { + errorf("Expected metadata for key %q to be %d, but it has %q", result.Key, len(result.Key), result.Value) + } + } + lastKey = result.Key + } + if len(results) > 0 { + opts.iterateOpts.First = results[len(results)-1].Key + } + lookupSize = opts.batchSize + 1 // subsequent queries will start with the last element previously returned + } + b.SetBytes(bytesTotal) + + if totalFound != opts.expectCount { + b.Fatalf("Expected to read %d items in total, but got %d", opts.expectCount, totalFound) + } + if !opts.expectLastKey.IsZero() { + if diff := cmp.Diff(opts.expectLastKey.String(), lastKey.String()); diff != "" { + b.Fatalf("KeyValueStore got wrong last item: (-want +got)\n%s", diff) + } + } +} + +func deepRecursive(b *testing.B, store storage.KeyValueStore) { + opts := &verifyOpts{ + iterateOpts: storage.IterateOptions{ + Prefix: storage.Key(largestLevel2Directory), + Recurse: true, + }, + } + + // these are not expected to exhaust all available items + opts.doIterations = 500 + opts.batchSize = storage.LookupLimit + opts.expectCount = opts.doIterations * opts.batchSize + + // verify with: + // select encode(fullpath, 'escape') from ( + // select rank() over (order by fullpath), fullpath from pathdata where fullpath > $1::bytea + // ) x where rank = ($2 * $3); + // where $1 = largestLevel2Directory, $2 = doIterations, and $3 = batchSize + opts.expectLastKey = storage.Key("Peronosporales/hateless/tod/extrastate/firewood/renomination/cletch/herotheism/aluminiferous/nub") + + benchAndVerifyIteration(b, store, opts) +} + +func deepRecursiveReverse(b *testing.B, store storage.KeyValueStore) { + opts := &verifyOpts{ + iterateOpts: storage.IterateOptions{ + Prefix: storage.Key(largestLevel2Directory), + Recurse: true, + Reverse: true, + }, + } + + // these are not expected to exhaust all available items + opts.doIterations = 500 + opts.batchSize = storage.LookupLimit + opts.expectCount = opts.doIterations * opts.batchSize + + // verify with: + // select encode(fullpath, 'escape') from ( + // select rank() over (order by fullpath desc), fullpath from pathdata + // where fullpath < bytea_increment($1::bytea) + // ) x where rank = ($2 * $3); + // where $1 = largestLevel2Directory, $2 = doIterations, and $3 = batchSize + opts.expectLastKey = storage.Key("Peronosporales/hateless/apetaly/poikilocythemia/capped/abrash/dugout/notodontid/jasponyx/cassican/brunelliaceous") + + benchAndVerifyIteration(b, store, opts) +} + +func deepNonRecursive(b *testing.B, store storage.KeyValueStore) { + opts := &verifyOpts{ + iterateOpts: storage.IterateOptions{ + Prefix: storage.Key(largestLevel2Directory), + Recurse: false, + }, + doIterations: 1, + batchSize: 10000, + } + + // verify with: + // select count(*) from list_directory(''::bytea, $1::bytea) ld(fp, md); + // where $1 is largestLevel2Directory + opts.expectCount = 119 + + // verify with: + // select encode(fp, 'escape') from ( + // select * from list_directory(''::bytea, $1::bytea) ld(fp, md) + // ) x order by fp desc limit 1; + // where $1 is largestLevel2Directory + opts.expectLastKey = storage.Key("Peronosporales/hateless/xerophily/") + + benchAndVerifyIteration(b, store, opts) +} + +func deepNonRecursiveReverse(b *testing.B, store storage.KeyValueStore) { + opts := &verifyOpts{ + iterateOpts: storage.IterateOptions{ + Prefix: storage.Key(largestLevel2Directory), + Recurse: false, + Reverse: true, + }, + doIterations: 1, + batchSize: 10000, + } + + // verify with: + // select count(*) from list_directory(''::bytea, $1::bytea) ld(fp, md); + // where $1 is largestLevel2Directory + opts.expectCount = 119 + + // verify with: + // select encode(fp, 'escape') from ( + // select * from list_directory(''::bytea, $1::bytea) ld(fp, md) + // ) x order by fp limit 1; + // where $1 is largestLevel2Directory + opts.expectLastKey = storage.Key("Peronosporales/hateless/Absyrtus") + + benchAndVerifyIteration(b, store, opts) +} + +func shallowRecursive(b *testing.B, store storage.KeyValueStore) { + opts := &verifyOpts{ + iterateOpts: storage.IterateOptions{ + Prefix: storage.Key(largestSingleDirectory), + Recurse: true, + }, + } + + // verify with: + // select count(*) from pathdata + // where fullpath > $1::bytea and fullpath < bytea_increment($1::bytea); + // where $1 = largestSingleDirectory + opts.expectCount = 18574 + + // verify with: + // select convert_from(fullpath, 'UTF8') from pathdata + // where fullpath > $1::bytea and fullpath < bytea_increment($1::bytea) + // order by fullpath desc limit 1; + // where $1 = largestSingleDirectory + opts.expectLastKey = storage.Key("Peronosporales/hateless/tod/unricht/sniveling/Puyallup/élite") + + // i didn't plan it this way, but expectedCount happens to have some nicely-sized factors for + // our purposes with no messy remainder. 74 * 251 = 18574 + opts.doIterations = 74 + opts.batchSize = 251 + + benchAndVerifyIteration(b, store, opts) +} + +func shallowRecursiveReverse(b *testing.B, store storage.KeyValueStore) { + opts := &verifyOpts{ + iterateOpts: storage.IterateOptions{ + Prefix: storage.Key(largestSingleDirectory), + Recurse: true, + Reverse: true, + }, + } + + // verify with: + // select count(*) from pathdata + // where fullpath > $1::bytea and fullpath < bytea_increment($1::bytea); + // where $1 = largestSingleDirectory + opts.expectCount = 18574 + + // verify with: + // select convert_from(fullpath, 'UTF8') from pathdata + // where fullpath > $1::bytea and fullpath < bytea_increment($1::bytea) + // order by fullpath limit 1; + // where $1 = largestSingleDirectory + opts.expectLastKey = storage.Key("Peronosporales/hateless/tod/unricht/sniveling/Puyallup/Aaronite") + + // i didn't plan it this way, but expectedCount happens to have some nicely-sized factors for + // our purposes with no messy remainder. 74 * 251 = 18574 + opts.doIterations = 74 + opts.batchSize = 251 + + benchAndVerifyIteration(b, store, opts) +} + +func shallowNonRecursive(b *testing.B, store storage.KeyValueStore) { + opts := &verifyOpts{ + iterateOpts: storage.IterateOptions{ + Prefix: storage.Key(largestSingleDirectory), + Recurse: false, + }, + doIterations: 2, + batchSize: 10000, + } + + // verify with: + // select count(*) from list_directory(''::bytea, $1::bytea) ld(fp, md); + // where $1 is largestSingleDirectory + opts.expectCount = 18574 + + // verify with: + // select encode(fp, 'escape') from ( + // select * from list_directory(''::bytea, $1::bytea) ld(fp, md) + // ) x order by fp desc limit 1; + // where $1 = largestSingleDirectory + opts.expectLastKey = storage.Key("Peronosporales/hateless/tod/unricht/sniveling/Puyallup/élite") + + benchAndVerifyIteration(b, store, opts) +} + +func shallowNonRecursiveReverse(b *testing.B, store storage.KeyValueStore) { + opts := &verifyOpts{ + iterateOpts: storage.IterateOptions{ + Prefix: storage.Key(largestSingleDirectory), + Recurse: false, + Reverse: true, + }, + doIterations: 2, + batchSize: 10000, + } + + // verify with: + // select count(*) from list_directory(''::bytea, $1::bytea) ld(fp, md); + // where $1 is largestSingleDirectory + opts.expectCount = 18574 + + // verify with: + // select encode(fp, 'escape') from ( + // select * from list_directory(''::bytea, $1::bytea) ld(fp, md) + // ) x order by fp limit 1; + // where $1 = largestSingleDirectory + opts.expectLastKey = storage.Key("Peronosporales/hateless/tod/unricht/sniveling/Puyallup/Aaronite") + + benchAndVerifyIteration(b, store, opts) +} + +func topRecursiveLimit(b *testing.B, store storage.KeyValueStore) { + opts := &verifyOpts{ + iterateOpts: storage.IterateOptions{ + Recurse: true, + }, + doIterations: 100, + batchSize: 10000, + } + + // not expected to exhaust items + opts.expectCount = opts.doIterations * opts.batchSize + + // verify with: + // select encode(fullpath, 'escape') from ( + // select rank() over (order by fullpath), fullpath from pathdata + // ) x where rank = $1; + // where $1 = expectCount + opts.expectLastKey = storage.Key("nonresuscitation/synchronically/bechern/hemangiomatosis") + + benchAndVerifyIteration(b, store, opts) +} + +func topRecursiveLimitReverse(b *testing.B, store storage.KeyValueStore) { + opts := &verifyOpts{ + iterateOpts: storage.IterateOptions{ + Recurse: true, + Reverse: true, + }, + doIterations: 100, + batchSize: 10000, + } + + // not expected to exhaust items + opts.expectCount = opts.doIterations * opts.batchSize + + // verify with: + // select encode(fullpath, 'escape') from ( + // select rank() over (order by fullpath desc), fullpath from pathdata + // ) x where rank = $1; + // where $1 = expectCount + opts.expectLastKey = storage.Key("nonresuscitation/synchronically/cabook/homeozoic/inclinatorium/iguanodont/thiophenol/congeliturbation/Alaric") + + benchAndVerifyIteration(b, store, opts) +} + +func topRecursiveStartAt(b *testing.B, store storage.KeyValueStore) { + opts := &verifyOpts{ + iterateOpts: storage.IterateOptions{ + Recurse: true, + }, + doIterations: 100, + batchSize: 10000, + } + + // this is pretty arbitrary. just the key 100 positions before the end of the Peronosporales/hateless/ dir. + opts.iterateOpts.First = storage.Key("Peronosporales/hateless/warrener/anthropomancy/geisotherm/wickerwork") + + // not expected to exhaust items + opts.expectCount = opts.doIterations * opts.batchSize + + // verify with: + // select encode(fullpath, 'escape') from ( + // select fullpath from pathdata where fullpath >= $1::bytea order by fullpath limit $2 + // ) x order by fullpath desc limit 1; + // where $1 = iterateOpts.First and $2 = expectCount + opts.expectLastKey = storage.Key("raptured/heathbird/histrionism/vermifugous/barefaced/beechdrops/lamber/phlegmatic/blended/Gershon/scallop/burglarproof/incompensated/allanite/alehouse/embroilment/lienotoxin/monotonically/cumbersomeness") + + benchAndVerifyIteration(b, store, opts) +} + +func topRecursiveStartAtReverse(b *testing.B, store storage.KeyValueStore) { + opts := &verifyOpts{ + iterateOpts: storage.IterateOptions{ + Recurse: true, + Reverse: true, + }, + doIterations: 61, + batchSize: 10000, + } + + // this is pretty arbitrary. just the key 100 positions before the end of the Peronosporales/hateless/ dir. + opts.iterateOpts.First = storage.Key("Peronosporales/hateless/warrener/anthropomancy/geisotherm/wickerwork") + + // we *do* expect to exhaust the available items this time. + // verify with: + // select count(*) from ( + // select fullpath from pathdata where fullpath <= $1::bytea order by fullpath desc limit $2 + // ) x; + // where $1 = iterateOpts.First and $2 = (doIterations * batchSize) + opts.expectCount = 608405 + + // since expectCount < (doIterations * batchSize), and we're going in reverse, the last key read + // should be the first one lexicographically. + // verify with: + // select encode(fullpath, 'escape') from pathdata order by fullpath limit 1; + opts.expectLastKey = storage.Key("Lissamphibia") + + benchAndVerifyIteration(b, store, opts) +} + +func topNonRecursive(b *testing.B, store storage.KeyValueStore) { + opts := &verifyOpts{ + iterateOpts: storage.IterateOptions{ + Recurse: false, + }, + doIterations: 1, + batchSize: 10000, + } + + // verify with: + // select count(*) from list_directory(''::bytea, ''::bytea); + opts.expectCount = 21 + + // verify with: + // select encode(fp, 'escape') from ( + // select * from list_directory(''::bytea, ''::bytea) ld(fp, md) + // ) x order by fp desc limit 1; + opts.expectLastKey = storage.Key("vejoces") + + benchAndVerifyIteration(b, store, opts) +} + +func topNonRecursiveReverse(b *testing.B, store storage.KeyValueStore) { + opts := &verifyOpts{ + iterateOpts: storage.IterateOptions{ + Recurse: false, + Reverse: true, + }, + doIterations: 1, + batchSize: 10000, + } + + // verify with: + // select count(*) from list_directory(''::bytea, ''::bytea); + opts.expectCount = 21 + + // verify with: + // select encode(fullpath, 'escape') from pathdata order by fullpath limit 1; + opts.expectLastKey = storage.Key("Lissamphibia") + + benchAndVerifyIteration(b, store, opts) +} + +func cleanupBigPathset(tb testing.TB, store storage.KeyValueStore) { + if *noCleanDb { + tb.Skip("Instructed not to clean up this KeyValueStore after long benchmarks are complete.") + } + + cleaner, ok := store.(BulkCleaner) + if ok { + tb.Log("Performing bulk cleanup...") + err := cleaner.BulkDelete() + + if err != nil { + tb.Fatalf("Provided KeyValueStore failed to perform bulk delete: %v", err) + } + } else { + inputIter := openTestData(tb) + defer func() { + if err := inputIter.closeFunc(); err != nil { + tb.Logf("Failed to close input data stream: %v", err) + } + }() + + tb.Log("Performing manual cleanup...") + + var item storage.ListItem + for inputIter.Next(&item) { + if err := store.Delete(item.Key); err != nil { + tb.Fatalf("Provided KeyValueStore failed to delete item %q during cleanup: %v", item.Key, err) + } + } + if err := inputIter.Error(); err != nil { + tb.Fatalf("Failed to iterate over input data: %v", err) + } + } +} diff --git a/storage/testsuite/utils.go b/storage/testsuite/utils.go index 82fa7785a..aa419bdf0 100644 --- a/storage/testsuite/utils.go +++ b/storage/testsuite/utils.go @@ -35,26 +35,44 @@ type iterationTest struct { func testIterations(t *testing.T, store storage.KeyValueStore, tests []iterationTest) { t.Helper() for _, test := range tests { - collect := &collector{} - err := store.Iterate(test.Options, collect.include) + items, err := iterateItems(store, test.Options, -1) if err != nil { t.Errorf("%s: %v", test.Name, err) continue } - if diff := cmp.Diff(test.Expected, collect.Items, cmpopts.EquateEmpty()); diff != "" { + if diff := cmp.Diff(test.Expected, items, cmpopts.EquateEmpty()); diff != "" { t.Errorf("%s: (-want +got)\n%s", test.Name, diff) } } } +func isEmptyKVStore(tb testing.TB, store storage.KeyValueStore) bool { + tb.Helper() + keys, err := store.List(storage.Key(""), 1) + if err != nil { + tb.Fatalf("Failed to check if KeyValueStore is empty: %v", err) + } + return len(keys) == 0 +} + type collector struct { Items storage.Items + Limit int } func (collect *collector) include(it storage.Iterator) error { var item storage.ListItem - for it.Next(&item) { + for (collect.Limit < 0 || len(collect.Items) < collect.Limit) && it.Next(&item) { collect.Items = append(collect.Items, storage.CloneItem(item)) } return nil } + +func iterateItems(store storage.KeyValueStore, opts storage.IterateOptions, limit int) (storage.Items, error) { + collect := &collector{Limit: limit} + err := store.Iterate(opts, collect.include) + if err != nil { + return nil, err + } + return collect.Items, nil +}