offer PostgreSQL storage for pointerdb (#440)

..although it ought to work for other storage.KeyValueStore needs as
well. it's just optimized to work pretty well for a largish hierarchy of
paths.

This includes the addition of "long benchmarks" for KeyValueStore
testing. These will only be run when -test-bench-long is added to the
test flags. In these benchmarks, a large corpus of paths matching a
natural ("real-life") hierarchy is read from paths.data.gz (which you
can get from https://github.com/storj/path-test-corpus) and imported
into a particular KeyValueStore. Recursive and non-recursive queries are
run on it to detect performance problems that arise only at scale.

This also includes alternate implementation of the postgreskv client,
which works in a less-bizarre way for non-recursive queries, but suffers
from poor performance in tests such as the long benchmarks. Once this
alternate impl is committed to the tree, we can remove it again; I just
want it to be available for future reference.

.travis.yml

@@ -12,6 +12,7 @@ cache:
 
 services:
   - redis
+  - postgresql
 
 before_script:
   # Add an IPv6 config - see the corresponding Travis issue
@@ -19,7 +20,8 @@ before_script:
   - if [ "${TRAVIS_OS_NAME}" == "linux" ]; then 
       sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6';
       sudo sh -c 'echo "\n::1 localhost\n" >> /etc/hosts';
-    fi
+      psql -c 'create database pointerdb' -U postgres;
+    fi;
 
 before_install:
   - source scripts/setup-gopath.sh
@@ -28,7 +30,9 @@ matrix:
     - os: windows # allow failures on windows because it's slow
   include:
     ### tests ###
-    - env: MODE=tests
+    - env:
+        - MODE=tests
+        - STORJ_POSTGRESKV_TEST=postgres://postgres@localhost/pointerdb?sslmode=disable
       install:
         - pushd ~
         - GOBIN=${GOPATH}/bin GOPATH=~/gotools go get github.com/mattn/goveralls
@@ -70,3 +74,6 @@ matrix:
         - goveralls -coverprofile=.coverprofile -service=travis-ci
 
   fast_finish: true
+
+addons:
+  postgresql: "9.5"

docker-compose.yaml

@@ -5,8 +5,15 @@ services:
       context: .
       dockerfile: test/Dockerfile
     network_mode: service:test-redis
+    depends_on:
+    - test-postgres-pointerdb
   test-redis:
     image: redis
+  test-postgres-pointerdb:
+    image: postgres
+    environment:
+    - POSTGRES_USER=pointerdb
+    - POSTGRES_PASSWORD=pg-secret-pass
 
   storagenode:
     image: storjlabs/storagenode:${VERSION}

go.mod

@@ -7,7 +7,6 @@ require (
 	github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 // indirect
 	github.com/alicebob/miniredis v0.0.0-20180911162847-3657542c8629
 	github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect
-	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
 	github.com/boltdb/bolt v1.3.1
 	github.com/cheggaaa/pb v1.0.5-0.20160713104425-73ae1d68fe0b
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
@@ -20,19 +19,15 @@ require (
 	github.com/elazarl/go-bindata-assetfs v1.0.0 // indirect
 	github.com/fatih/color v1.7.0 // indirect
 	github.com/fatih/structs v1.0.0 // indirect
-	github.com/go-ini/ini v1.38.2 // indirect
+	github.com/go-bindata/go-bindata v1.0.0
 	github.com/go-redis/redis v6.14.1+incompatible
-	github.com/go-sql-driver/mysql v1.4.0 // indirect
 	github.com/gogo/protobuf v1.1.1
+	github.com/golang-migrate/migrate/v3 v3.5.2
 	github.com/golang/mock v1.1.1
 	github.com/golang/protobuf v1.2.0
-	github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect
 	github.com/gomodule/redigo v2.0.0+incompatible // indirect
 	github.com/google/go-cmp v0.2.0
-	github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c // indirect
-	github.com/gorilla/context v1.1.1 // indirect
 	github.com/gorilla/handlers v1.4.0 // indirect
-	github.com/gorilla/mux v1.6.2 // indirect
 	github.com/gorilla/rpc v1.1.0 // indirect
 	github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69
 	github.com/hashicorp/go-immutable-radix v1.0.0 // indirect
@@ -41,19 +36,16 @@ require (
 	github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c // indirect
 	github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect
 	github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6
-	github.com/jtolds/gls v4.2.1+incompatible // indirect
 	github.com/jtolds/monkit-hw v0.0.0-20180827162413-5a254051f35d
 	github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e // indirect
 	github.com/klauspost/reedsolomon v0.0.0-20180704173009-925cb01d6510 // indirect
-	github.com/kr/pretty v0.1.0 // indirect
+	github.com/lib/pq v1.0.0
-	github.com/lib/pq v0.0.0-20180523175426-90697d60dd84 // indirect
 	github.com/loov/hrtime v0.0.0-20180911122900-a9e82bc6c180
 	github.com/loov/plot v0.0.0-20180510142208-e59891ae1271
 	github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 // indirect
 	github.com/mattn/go-isatty v0.0.4 // indirect
 	github.com/mattn/go-runewidth v0.0.3 // indirect
 	github.com/mattn/go-sqlite3 v1.9.0
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/minio/cli v1.3.0
 	github.com/minio/dsync v0.0.0-20180124070302-439a0961af70 // indirect
 	github.com/minio/highwayhash v0.0.0-20180501080913-85fc8a2dacad // indirect
@@ -70,7 +62,6 @@ require (
 	github.com/nats-io/nats v1.6.0 // indirect
 	github.com/nats-io/nats-streaming-server v0.11.0 // indirect
 	github.com/nats-io/nuid v1.0.0 // indirect
-	github.com/onsi/gomega v1.4.2 // indirect
 	github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c // indirect
 	github.com/pierrec/lz4 v2.0.5+incompatible // indirect
 	github.com/pkg/profile v1.2.1 // indirect
@@ -79,8 +70,6 @@ require (
 	github.com/rs/cors v1.5.0 // indirect
 	github.com/shirou/gopsutil v2.17.12+incompatible
 	github.com/skyrings/skyring-common v0.0.0-20160929130248-d1c0bb1cbd5e // indirect
-	github.com/smartystreets/assertions v0.0.0-20180820201707-7c9eb446e3cf // indirect
-	github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a // indirect
 	github.com/spacemonkeygo/errors v0.0.0-20171212215202-9064522e9fd1 // indirect
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/pflag v1.0.3
@@ -104,9 +93,7 @@ require (
 	golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect
 	google.golang.org/grpc v1.15.0
 	gopkg.in/Shopify/sarama.v1 v1.18.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/cheggaaa/pb.v1 v1.0.25 // indirect
-	gopkg.in/ini.v1 v1.38.2 // indirect
 	gopkg.in/olivere/elastic.v5 v5.0.76 // indirect
 	gopkg.in/spacemonkeygo/monkit.v2 v2.0.0-20180827161543-6ebf5a752f9b
 	gopkg.in/vmihailenco/msgpack.v2 v2.9.1 // indirect
@@ -122,12 +109,8 @@ require (
 
 	github.com/minio/minio v0.0.0-20180508161510-54cd29b51c38
 	github.com/mitchellh/mapstructure v1.1.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
 
 	github.com/prometheus/client_golang v0.9.0-pre1.0.20180416233856-82f5ff156b29 // indirect
-	github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5 // indirect
-	github.com/prometheus/common v0.0.0-20180326160409-38c53a9f4bfc // indirect
-	github.com/prometheus/procfs v0.0.0-20180408092902-8b1c2da0d56d // indirect
 	github.com/segmentio/go-prompt v1.2.1-0.20161017233205-f0d19b6901ad // indirect
 )
 

go.sum

@@ -1,9 +1,16 @@
 cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.27.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+contrib.go.opencensus.io/exporter/stackdriver v0.6.0/go.mod h1:QeFzMJDAw8TXt5+aRaSuE8l5BwaMIOIlaVkBOPRuMuw=
+git.apache.org/thrift.git v0.0.0-20180807212849-6e67faa92827/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q=
+github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Shopify/toxiproxy v2.1.3+incompatible h1:awiJqUYH4q4OmoBiRccJykjd7B+w0loJi2keSna4X/M=
 github.com/Shopify/toxiproxy v2.1.3+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/Sirupsen/logrus v1.0.6 h1:HCAGQRk48dRVPA5Y+Yh0qdCSTzPOyU1tBJ7Q9YzotII=
+github.com/Sirupsen/logrus v1.0.6/go.mod h1:rmk17hk6i8ZSAJkSDa7nOxamrG+SP4P0mm+DAvExv4U=
 github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f h1:5ZfJxyXo8KyX8DgGXC5B7ILL8y51fci/qYz2B4j8iLY=
 github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 h1:45bxf7AZMwWcqkLzDAQugVEwedisr5nRJ1r+7LYnv0U=
@@ -12,8 +19,11 @@ github.com/alicebob/miniredis v0.0.0-20180911162847-3657542c8629 h1:gLoh8jzwIxdi
 github.com/alicebob/miniredis v0.0.0-20180911162847-3657542c8629/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/aws/aws-sdk-go v1.15.34/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/cheggaaa/pb v1.0.5-0.20160713104425-73ae1d68fe0b h1:CMRCnhHx4xVxJy+wPsS67xmi9RHGNctLMoVn9Q1Kit8=
@@ -21,12 +31,33 @@ github.com/cheggaaa/pb v1.0.5-0.20160713104425-73ae1d68fe0b/go.mod h1:pQciLPpbU0
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudfoundry/gosigar v1.1.0 h1:V/dVCzhKOdIU3WRB5inQU20s4yIgL9Dxx/Mhi0SF8eM=
 github.com/cloudfoundry/gosigar v1.1.0/go.mod h1:3qLfc2GlfmwOx2+ZDaRGH3Y9fwQ0sQeaAleo2GV5pH0=
+github.com/cockroachdb/cockroach-go v0.0.0-20180212155653-59c0560478b7/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk=
+github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8=
+github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f/go.mod h1:8S58EK26zhXSxzv7NQFpnliaOQsmDUxvoQO3rt154Vg=
+github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc=
+github.com/cznic/internal v0.0.0-20180608152220-f44710a21d00/go.mod h1:olo7eAdKwJdXxb55TKGLiJ6xt1H0/tiiRCWKVLmtjY4=
+github.com/cznic/lldb v1.1.0/go.mod h1:FIZVUmYUVhPwRiPzL8nD/mpFcJ/G7SSXjjXYG4uRI3A=
+github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
+github.com/cznic/ql v1.2.0/go.mod h1:FbpzhyZrqr0PVlK6ury+PoW3T0ODUV22OeWIxcaOrSE=
+github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ=
+github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc=
+github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc/go.mod h1:Y1SNZ4dRUOKXshKUbwUapqNncRrho4mkjQebgEHZLj8=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg=
 github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8=
+github.com/docker/distribution v0.0.0-20180720172123-0dae0957e5fe h1:ZRQNMB7Sw5jf9g/0imDbI+vTFNk4J7qBdtFI5/zf1kg=
+github.com/docker/distribution v0.0.0-20180720172123-0dae0957e5fe/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v0.0.0-20170502054910-90d35abf7b35 h1:zQuy/ry/KtSvCczhEPo+ud47S8alruA/Z9NDlz7EzVo=
+github.com/docker/docker v0.0.0-20170502054910-90d35abf7b35/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
+github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4=
+github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
 github.com/dustin/go-humanize v0.0.0-20180713052910-9f541cc9db5d h1:lDrio3iIdNb0Gw9CgH7cQF+iuB5mOOjdJ9ERNJCBgb4=
 github.com/dustin/go-humanize v0.0.0-20180713052910-9f541cc9db5d/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU=
@@ -37,6 +68,7 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/eclipse/paho.mqtt.golang v1.1.1 h1:iPJYXJLaViCshRTW/PSqImSS6HJ2Rf671WR0bXZ2GIU=
 github.com/eclipse/paho.mqtt.golang v1.1.1/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts=
+github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/elazarl/go-bindata-assetfs v1.0.0 h1:G/bYguwHIzWq9ZoyUQqrjTmJbbYn3j3CKKpKinvZLFk=
 github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
@@ -47,8 +79,12 @@ github.com/fortytw2/leaktest v1.2.0 h1:cj6GCiwJDH7l3tMHLjZDo0QqPtrXJiWSI9JgpeQKw
 github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsouza/fake-gcs-server v1.2.0/go.mod h1:rM69NBSmfAkTlKDhzXC41OeX9lQxQXnkimkiGWDU1Ek=
 github.com/garyburd/redigo v1.0.1-0.20170216214944-0d253a66e6e1 h1:YmyuMm99D7kezPc0ZVWYnaUIWfMKR81lVVXttKTnDbw=
 github.com/garyburd/redigo v1.0.1-0.20170216214944-0d253a66e6e1/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
+github.com/go-bindata/go-bindata v1.0.0 h1:upAQEGKG3hFv00/4cU/VhzF+NpCHz+Jk3rO946wWr+A=
+github.com/go-bindata/go-bindata v1.0.0/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
+github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-ini/ini v1.38.2 h1:6Hl/z3p3iFkA0dlDfzYxuFuUGD+kaweypF6btsR2/Q4=
 github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E=
@@ -57,8 +93,11 @@ github.com/go-redis/redis v6.14.1+incompatible h1:kSJohAREGMr344uMa8PzuIg5OU6ylC
 github.com/go-redis/redis v6.14.1+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/gocql/gocql v0.0.0-20180913072538-864d5908455a/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0=
 github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang-migrate/migrate/v3 v3.5.2 h1:SUWSv6PD8Lr2TGx1lmVW7W2lRoQiVny3stM4He6jczQ=
+github.com/golang-migrate/migrate/v3 v3.5.2/go.mod h1:QDa9JH6Bdwbi+1jIEpOIWnTNoYRbtaVgByErXU0844E=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
@@ -66,12 +105,17 @@ github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
 github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
+github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/martian v2.0.0-beta.2+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
 github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c h1:16eHWuMGvCjSfgRJKqIzapE78onvvTbdi1rMkU00lZw=
 github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
@@ -84,6 +128,7 @@ github.com/gorilla/rpc v1.1.0 h1:marKfvVP0Gpd/jHlVBKCQ8RAoUPdX7K1Nuh6l1BNh7A=
 github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
 github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 h1:7xsUJsB2NrdcttQPa7JLEaGzvdbk7KvfrjgHZXOQRo0=
 github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo=
+github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
 github.com/hanwen/go-fuse v0.0.0-20181011180456-b760b55765be h1:RXF6Da5rbJlRUosxKxuy/3OrLLG77aXRrZYcjDs6aB4=
 github.com/hanwen/go-fuse v0.0.0-20181011180456-b760b55765be/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok=
 github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
@@ -108,6 +153,8 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6 h1:4zOlv2my+vf98jT1nQt4bT/yKWUImevYPJ2H344CloE=
 github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6/go.mod h1:r/8JmuR0qjuCiEhAolkfvdZgmPiHTnJaG0UXCSeR1Zo=
+github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/jtolds/monkit-hw v0.0.0-20180827162413-5a254051f35d h1:+TWXq24sHXNI6DVobckzgJCbCvb4bYjDZuKRR+d/Nuw=
@@ -123,8 +170,9 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/lib/pq v0.0.0-20180523175426-90697d60dd84 h1:it29sI2IM490luSc3RAhp5WuCYnc6RtbfLVAB7nmC5M=
+github.com/kshvakov/clickhouse v1.3.4/go.mod h1:DMzX7FxRymoNkVgizH0DWAL8Cur7wHLgx3MUnGwJqpE=
-github.com/lib/pq v0.0.0-20180523175426-90697d60dd84/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/loov/hrtime v0.0.0-20180911122900-a9e82bc6c180 h1:kLwg5eA/kaWQ/RwANTH7Gg+VdxmdjbcSWyaS/1VQGkA=
 github.com/loov/hrtime v0.0.0-20180911122900-a9e82bc6c180/go.mod h1:2871C3urfEJnq/bpTYjFdMOdgxVd8otLLEL6vMNy/Iw=
 github.com/loov/plot v0.0.0-20180510142208-e59891ae1271 h1:51ToN6N0TDtCruf681gufYuEhO9qFHQzM3RFTS/n6XE=
@@ -187,6 +235,9 @@ github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
+github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
@@ -199,14 +250,15 @@ github.com/pkg/profile v1.2.1 h1:F++O52m40owAmADcojzM+9gyjmMOY/T4oYJkgFDH8RE=
 github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.0-pre1.0.20180416233856-82f5ff156b29 h1:NWk5kfSrcwyaUbo3Ee6EECK/fXsY1sJt0U6kYnFNYmc=
 github.com/prometheus/client_golang v0.9.0-pre1.0.20180416233856-82f5ff156b29/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5 h1:cLL6NowurKLMfCeQy4tIeph12XNQWgANCNvdyrOYKV4=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
-github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/common v0.0.0-20180326160409-38c53a9f4bfc h1:tyg3EcZAmwCUe90Jzl4Qw6Af+ajuW8S9b1VFitMNOQs=
+github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e h1:n/3MEhJQjQxrOUCzh1Y3Re6aJUUWRp2M9+Oc3eVn/54=
-github.com/prometheus/common v0.0.0-20180326160409-38c53a9f4bfc/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/procfs v0.0.0-20180408092902-8b1c2da0d56d h1:RCcsxyRr6+/pLg6wr0cUjPovhEhSNOtPh0SOz6u3hGU=
+github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 h1:agujYaXJSxSo18YNX3jzl+4G6Bstwt+kqv47GS12uL0=
-github.com/prometheus/procfs v0.0.0-20180408092902-8b1c2da0d56d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/rcrowley/go-metrics v0.0.0-20180503174638-e2704e165165 h1:nkcn14uNmFEuGCb2mBZbBb24RdNRL08b/wb+xBOYpuk=
 github.com/rcrowley/go-metrics v0.0.0-20180503174638-e2704e165165/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rs/cors v1.5.0 h1:dgSHE6+ia18arGOTIYQKKGWLvEbGvmbNE6NfxhoNHUY=
@@ -215,6 +267,8 @@ github.com/segmentio/go-prompt v1.2.1-0.20161017233205-f0d19b6901ad h1:EqOdoSJGI
 github.com/segmentio/go-prompt v1.2.1-0.20161017233205-f0d19b6901ad/go.mod h1:B3ehdD1xPoWDKgrQgUaGk+m8H1xb1J5TyYDfKpKNeEE=
 github.com/shirou/gopsutil v2.17.12+incompatible h1:FNbznluSK3DQggqiVw3wK/tFKJrKlLPBuQ+V8XkkCOc=
 github.com/shirou/gopsutil v2.17.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/sirupsen/logrus v1.0.6 h1:hcP1GmhGigz/O7h1WVUM5KklBp1JoNS9FggWKdj/j3s=
+github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/skyrings/skyring-common v0.0.0-20160929130248-d1c0bb1cbd5e h1:jrZSSgPUDtBeJbGXqgGUeupQH8I+ZvGXfhpIahye2Bc=
 github.com/skyrings/skyring-common v0.0.0-20160929130248-d1c0bb1cbd5e/go.mod h1:d8hQseuYt4rJoOo21lFzYJdhMjmDqLY++ayArbgYjWI=
 github.com/smartystreets/assertions v0.0.0-20180820201707-7c9eb446e3cf h1:6V1qxN6Usn4jy8unvggSJz/NC790tefw8Zdy6OZS5co=
@@ -262,33 +316,34 @@ github.com/zeebo/float16 v0.1.0 h1:kRqxv5og6z1emEyz5FpW0/BVHe5VfxEAw6b1ljCZlUc=
 github.com/zeebo/float16 v0.1.0/go.mod h1:fssGvvXu+XS8MH57cKmyrLB/cqioYeYX/2mXCN3a5wo=
 github.com/zeebo/incenc v0.0.0-20180505221441-0d92902eec54 h1:+cwNE5KJ3pika4HuzmDHkDlK5myo0G9Sv+eO7WWxnUQ=
 github.com/zeebo/incenc v0.0.0-20180505221441-0d92902eec54/go.mod h1:EI8LcOBDlSL3POyqwC1eJhOYlMBMidES+613EtmmT5w=
+go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=
+go.opencensus.io v0.16.0/go.mod h1:0TeCCqcQSLNZtiq/62+vUzqwnjqF5el6hjmuZaFtyNk=
 go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o=
 go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4 h1:Vk3wNqEZwyGyei9yq5ekj7frek2u7HUfffJ1/opblzc=
+golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941 h1:qBTHLajHecfu+xzRI9PqVDcqx7SdHj9d4B+EzSn3tAc=
 golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/net v0.0.0-20180821023952-922f4815f713/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181003013248-f5e5bdd77824 h1:MkjFNbaZJyH98M67Q3umtwZ+EdVdrNJLqSwZp5vcv60=
 golang.org/x/net v0.0.0-20181003013248-f5e5bdd77824/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181005035420-146acd28ed58 h1:otZG8yDCO4LVps5+9bxOeNiCvgmOyt96J3roHTYs7oE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180821140842-3b58ed4ad339/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180824143301-4910a1d54f87 h1:GqwDwfvIpC33dK9bA1fD+JiDUNsuAiQiEkpHqUKze4o=
 golang.org/x/sys v0.0.0-20180824143301-4910a1d54f87/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181004145325-8469e314837c h1:SJ7JoQNVl3mC7EWkkONgBWgCno8LcABIJwFMkWBC+EY=
-golang.org/x/sys v0.0.0-20181004145325-8469e314837c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181005133103-4497e2df6f9e h1:EfdBzeKbFSvOjoIqSZcfS8wp0FBLokGBEs9lz1OtSg0=
 golang.org/x/sys v0.0.0-20181005133103-4497e2df6f9e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
@@ -297,14 +352,23 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2I
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52 h1:JG/0uqcGdTNgq7FdU+61l5Pdmb8putNZlXb65bJBROs=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180911133044-677d2ff680c1/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+google.golang.org/api v0.0.0-20180818000503-e21acd801f91/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/api v0.0.0-20180826000528-7954115fcf34/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20180912233945-5a2fd4cab2d6 h1:YN6g8chEdBTmaERxJzbJ6WKLrW3+Bf6rznOBkWNSQP0=
+google.golang.org/genproto v0.0.0-20180912233945-5a2fd4cab2d6/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.15.0 h1:Az/KuahOM4NAidTEuJCv/RonAA7rYsTPkqXVjr+8OOw=
 google.golang.org/grpc v1.15.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 gopkg.in/Shopify/sarama.v1 v1.18.0 h1:f9aTXuIEFEjVvLG9p+kMSk01dMfFumHsySRk1okTdqU=
 gopkg.in/Shopify/sarama.v1 v1.18.0/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc=
+gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo=
+gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -312,6 +376,9 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25 h1:Ev7yu1/f6+d+b3pi5vPdRPc6nNtP1umSfcWiEfRqv6I=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0=
+gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.38.2 h1:dGcbywv4RufeGeiMycPT/plKB5FtmLKLnWKwBiLhUA4=
 gopkg.in/ini.v1 v1.38.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/olivere/elastic.v5 v5.0.76 h1:A6W7X4yLPQDINHiYAqIwqev+rD5hIQ4G0e1d5H//VXk=

pkg/pointerdb/config.go

@@ -12,13 +12,15 @@ import (
 	"storj.io/storj/pkg/pb"
 	"storj.io/storj/pkg/provider"
 	"storj.io/storj/pkg/utils"
+	"storj.io/storj/storage"
 	"storj.io/storj/storage/boltdb"
+	"storj.io/storj/storage/postgreskv"
 	"storj.io/storj/storage/storelogger"
 )
 
 const (
-	// PointerBucket is the string representing the bucket used for `PointerEntries`
+	// BoltPointerBucket is the string representing the bucket used for `PointerEntries` in BoltDB
-	PointerBucket = "pointers"
+	BoltPointerBucket = "pointers"
 )
 
 // Config is a configuration struct that is everything you need to start a
@@ -30,25 +32,32 @@ type Config struct {
 	Overlay              bool   `default:"false" help:"toggle flag if overlay is enabled"`
 }
 
+func newKeyValueStore(dbURLString string) (db storage.KeyValueStore, err error) {
+	dburl, err := utils.ParseURL(dbURLString)
+	if err != nil {
+		return nil, err
+	}
+	if dburl.Scheme == "bolt" {
+		db, err = boltdb.New(dburl.Path, BoltPointerBucket)
+	} else if dburl.Scheme == "postgresql" || dburl.Scheme == "postgres" {
+		db, err = postgreskv.New(dbURLString)
+	} else {
+		err = Error.New("unsupported db scheme: %s", dburl.Scheme)
+	}
+	return db, err
+}
+
 // Run implements the provider.Responsibility interface
 func (c Config) Run(ctx context.Context, server *provider.Provider) error {
-	dburl, err := utils.ParseURL(c.DatabaseURL)
+	db, err := newKeyValueStore(c.DatabaseURL)
 	if err != nil {
 		return err
 	}
-	if dburl.Scheme != "bolt" {
+	defer func() { _ = db.Close() }()
-		return Error.New("unsupported db scheme: %s", dburl.Scheme)
-	}
-
-	bdb, err := boltdb.New(dburl.Path, PointerBucket)
-	if err != nil {
-		return err
-	}
-	defer func() { _ = bdb.Close() }()
 
 	cache := overlay.LoadFromContext(ctx)
-	bdblogged := storelogger.New(zap.L(), bdb)
+	dblogged := storelogger.New(zap.L(), db)
-	pb.RegisterPointerDBServer(server.GRPC(), NewServer(bdblogged, cache, zap.L(), c, server.Identity()))
+	pb.RegisterPointerDBServer(server.GRPC(), NewServer(dblogged, cache, zap.L(), c, server.Identity()))
 
 	return server.Run(ctx)
 }

storage/boltdb/client.go

@@ -161,8 +161,8 @@ func (client *Client) Close() error {
 	return nil
 }
 
-// GetAll finds all values for the provided keys up to 100 keys
+// GetAll finds all values for the provided keys (up to storage.LookupLimit).
-// if more keys are provided than the maximum an error will be returned.
+// If more keys are provided than the maximum, an error will be returned.
 func (client *Client) GetAll(keys storage.Keys) (storage.Values, error) {
 	if len(keys) > storage.LookupLimit {
 		return nil, storage.ErrLimitExceeded

storage/boltdb/client_test.go

@@ -4,11 +4,14 @@
 package boltdb
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
 
+	"storj.io/storj/pkg/utils"
+	"storj.io/storj/storage"
 	"storj.io/storj/storage/testsuite"
 )
 
@@ -78,3 +81,61 @@ func TestSuiteShared(t *testing.T) {
 		testsuite.RunTests(t, store)
 	}
 }
+
+type boltLongBenchmarkStore struct {
+	*Client
+	dirPath string
+}
+
+func (store *boltLongBenchmarkStore) BulkImport(iter storage.Iterator) (err error) {
+	// turn off syncing during import
+	oldval := store.db.NoSync
+	store.db.NoSync = true
+	defer func() { store.db.NoSync = oldval }()
+
+	var item storage.ListItem
+	for iter.Next(&item) {
+		if err := store.Put(item.Key, item.Value); err != nil {
+			return fmt.Errorf("Failed to insert data (%q, %q): %v", item.Key, item.Value, err)
+		}
+	}
+
+	return store.db.Sync()
+}
+
+func (store *boltLongBenchmarkStore) BulkDelete() error {
+	// do nothing here; everything will be cleaned up later after the test completes. it's not
+	// worth it to wait for BoltDB to remove every key, one by one, and we can't just
+	// os.RemoveAll() the whole test directory at this point because those files are still open
+	// and unremoveable on Windows.
+	return nil
+}
+
+func BenchmarkSuiteLong(b *testing.B) {
+	tempdir, err := ioutil.TempDir("", "storj-bolt")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer func() {
+		if err := os.RemoveAll(tempdir); err != nil {
+			b.Fatal(err)
+		}
+	}()
+
+	dbname := filepath.Join(tempdir, "bolt.db")
+	store, err := New(dbname, "bucket")
+	if err != nil {
+		b.Fatalf("failed to create db: %v", err)
+	}
+	defer func() {
+		if err := utils.CombineErrors(store.Close(), os.RemoveAll(tempdir)); err != nil {
+			b.Fatalf("failed to close db: %v", err)
+		}
+	}()
+
+	longStore := &boltLongBenchmarkStore{
+		Client:  store,
+		dirPath: tempdir,
+	}
+	testsuite.BenchmarkPathOperationsInLargeDb(b, longStore)
+}

storage/boltdb/common.go

@@ -9,6 +9,3 @@ import (
 
 // Error is the default boltdb errs class
 var Error = errs.Class("boltdb error")
-
-// ErrKeyNotFound should occur when a key isn't found in a boltdb bucket (table)
-var ErrKeyNotFound = errs.Class("key not found")

storage/postgreskv/alternateclient.go

@@ -0,0 +1,165 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+
+// See LICENSE for copying information.
+
+package postgreskv
+
+import (
+	"database/sql"
+
+	"storj.io/storj/pkg/utils"
+	"storj.io/storj/storage"
+)
+
+const (
+	alternateSQLSetup = `
+CREATE OR REPLACE FUNCTION local_path (fullpath BYTEA, prefix BYTEA, delimiter BYTEA)
+	RETURNS BYTEA AS $$
+DECLARE
+	relative BYTEA;
+	pos INTEGER;
+BEGIN
+	relative := substring(fullpath FROM (octet_length(prefix)+1));
+	pos := position(delimiter IN relative);
+	IF pos = 0 THEN
+		RETURN relative;
+	END IF;
+	RETURN substring(relative FOR pos);
+END;
+$$ LANGUAGE 'plpgsql'
+	IMMUTABLE STRICT;
+`
+
+	alternateSQLTeardown = `
+DROP FUNCTION local_path(BYTEA, BYTEA, BYTEA);
+`
+
+	alternateForwardQuery = `
+SELECT DISTINCT
+	$2::BYTEA || x.localpath AS p,
+	first_value(x.metadata) OVER (PARTITION BY x.localpath ORDER BY x.fullpath) AS m
+FROM (
+	SELECT
+		pd.fullpath,
+		local_path(pd.fullpath, $2::BYTEA, set_byte(' '::BYTEA, 0, b.delim)) AS localpath,
+		pd.metadata
+	FROM
+		pathdata pd,
+		buckets b
+	WHERE
+		b.bucketname = $1::BYTEA
+		AND pd.bucket = b.bucketname
+		AND pd.fullpath >= $2::BYTEA
+		AND ($2::BYTEA = ''::BYTEA OR pd.fullpath < bytea_increment($2::BYTEA))
+		AND pd.fullpath >= $3::BYTEA
+) x
+ORDER BY p
+LIMIT $4
+`
+
+	alternateReverseQuery = `
+SELECT DISTINCT
+	$2::BYTEA || x.localpath AS p,
+	first_value(x.metadata) OVER (PARTITION BY x.localpath ORDER BY x.fullpath) AS m
+FROM (
+	SELECT
+		pd.fullpath,
+		local_path(pd.fullpath, $2::BYTEA, set_byte(' '::BYTEA, 0, b.delim)) AS localpath,
+		pd.metadata
+	FROM
+		pathdata pd,
+		buckets b
+	WHERE
+		b.bucketname = $1::BYTEA
+		AND pd.bucket = b.bucketname
+		AND pd.fullpath >= $2::BYTEA
+		AND ($2::BYTEA = ''::BYTEA OR pd.fullpath < bytea_increment($2::BYTEA))
+		AND ($3::BYTEA = ''::BYTEA OR pd.fullpath <= $3::BYTEA)
+) x
+ORDER BY p DESC
+LIMIT $4
+`
+)
+
+// AlternateClient is the entrypoint into an alternate postgreskv data store
+type AlternateClient struct {
+	*Client
+}
+
+// AltNew instantiates a new postgreskv AlternateClient given db URL
+func AltNew(dbURL string) (*AlternateClient, error) {
+	client, err := New(dbURL)
+	if err != nil {
+		return nil, err
+	}
+	_, err = client.pgConn.Exec(alternateSQLSetup)
+	if err != nil {
+		return nil, utils.CombineErrors(err, client.Close())
+	}
+	return &AlternateClient{Client: client}, nil
+}
+
+// Close closes an AlternateClient and frees its resources.
+func (altClient *AlternateClient) Close() error {
+	_, err := altClient.pgConn.Exec(alternateSQLTeardown)
+	return utils.CombineErrors(err, altClient.Client.Close())
+}
+
+type alternateOrderedPostgresIterator struct {
+	*orderedPostgresIterator
+}
+
+func (opi *alternateOrderedPostgresIterator) doNextQuery() (*sql.Rows, error) {
+	if opi.opts.Recurse {
+		return opi.orderedPostgresIterator.doNextQuery()
+	}
+	start := opi.lastKeySeen
+	if start == nil {
+		start = opi.opts.First
+	}
+	var query string
+	if opi.opts.Reverse {
+		query = alternateReverseQuery
+	} else {
+		query = alternateForwardQuery
+	}
+	return opi.client.pgConn.Query(query, []byte(opi.bucket), []byte(opi.opts.Prefix), []byte(start), opi.batchSize+1)
+}
+
+func newAlternateOrderedPostgresIterator(altClient *AlternateClient, opts storage.IterateOptions, batchSize int) (*alternateOrderedPostgresIterator, error) {
+	if opts.Prefix == nil {
+		opts.Prefix = storage.Key("")
+	}
+	if opts.First == nil {
+		opts.First = storage.Key("")
+	}
+	opi1 := &orderedPostgresIterator{
+		client:    altClient.Client,
+		opts:      &opts,
+		bucket:    storage.Key(defaultBucket),
+		delimiter: byte('/'),
+		batchSize: batchSize,
+		curIndex:  0,
+	}
+	opi := &alternateOrderedPostgresIterator{orderedPostgresIterator: opi1}
+	opi.nextQuery = opi.doNextQuery
+	newRows, err := opi.nextQuery()
+	if err != nil {
+		return nil, err
+	}
+	opi.curRows = newRows
+	return opi, nil
+}
+
+// Iterate iterates over items based on opts
+func (altClient *AlternateClient) Iterate(opts storage.IterateOptions, fn func(storage.Iterator) error) (err error) {
+	opi, err := newAlternateOrderedPostgresIterator(altClient, opts, defaultBatchSize)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err = utils.CombineErrors(err, opi.Close())
+	}()
+
+	return fn(opi)
+}

storage/postgreskv/alternateclient_test.go

@@ -0,0 +1,73 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package postgreskv
+
+import (
+	"flag"
+	"testing"
+
+	"go.uber.org/zap/zaptest"
+
+	"storj.io/storj/storage"
+	"storj.io/storj/storage/storelogger"
+	"storj.io/storj/storage/testsuite"
+)
+
+var (
+	doAltTests = flag.Bool("test-postgreskv-alt", false, "Run the KeyValueStore tests against the alternate PG implementation")
+)
+
+func newTestAlternatePostgres(t testing.TB) (store *AlternateClient, cleanup func()) {
+	if !*doAltTests {
+		t.Skip("alternate-implementation PG tests not enabled.")
+	}
+	if *testPostgres == "" {
+		t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", defaultPostgresConn)
+	}
+
+	pgdb, err := AltNew(*testPostgres)
+	if err != nil {
+		t.Fatalf("init: %v", err)
+	}
+
+	return pgdb, func() {
+		if err := pgdb.Close(); err != nil {
+			t.Fatalf("failed to close db: %v", err)
+		}
+	}
+}
+
+func TestSuiteAlt(t *testing.T) {
+	store, cleanup := newTestAlternatePostgres(t)
+	defer cleanup()
+
+	zap := zaptest.NewLogger(t)
+	testsuite.RunTests(t, storelogger.New(zap, store))
+}
+
+func BenchmarkSuiteAlt(b *testing.B) {
+	store, cleanup := newTestAlternatePostgres(b)
+	defer cleanup()
+
+	testsuite.RunBenchmarks(b, store)
+}
+
+type pgAltLongBenchmarkStore struct {
+	*AlternateClient
+}
+
+func (store *pgAltLongBenchmarkStore) BulkImport(iter storage.Iterator) error {
+	return bulkImport(store.pgConn, iter)
+}
+
+func (store *pgAltLongBenchmarkStore) BulkDelete() error {
+	return bulkDelete(store.pgConn)
+}
+
+func BenchmarkSuiteLongAlt(b *testing.B) {
+	store, cleanup := newTestAlternatePostgres(b)
+	defer cleanup()
+
+	testsuite.BenchmarkPathOperationsInLargeDb(b, &pgAltLongBenchmarkStore{store})
+}

storage/postgreskv/client.go

@@ -0,0 +1,295 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package postgreskv
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/lib/pq"
+	"github.com/zeebo/errs"
+
+	"storj.io/storj/pkg/utils"
+	"storj.io/storj/storage"
+	"storj.io/storj/storage/postgreskv/schema"
+)
+
+const (
+	defaultBatchSize = 10000
+	defaultBucket    = ""
+)
+
+// Client is the entrypoint into a postgreskv data store
+type Client struct {
+	URL    string
+	pgConn *sql.DB
+}
+
+// New instantiates a new postgreskv client given db URL
+func New(dbURL string) (*Client, error) {
+	pgConn, err := sql.Open("postgres", dbURL)
+	if err != nil {
+		return nil, err
+	}
+	err = schema.PrepareDB(pgConn)
+	if err != nil {
+		return nil, err
+	}
+	return &Client{
+		URL:    dbURL,
+		pgConn: pgConn,
+	}, nil
+}
+
+// Put sets the value for the provided key.
+func (client *Client) Put(key storage.Key, value storage.Value) error {
+	return client.PutPath(storage.Key(defaultBucket), key, value)
+}
+
+// PutPath sets the value for the provided key (in the given bucket).
+func (client *Client) PutPath(bucket, key storage.Key, value storage.Value) error {
+	if key.IsZero() {
+		return Error.New("invalid key")
+	}
+	q := `
+		INSERT INTO pathdata (bucket, fullpath, metadata)
+			VALUES ($1::BYTEA, $2::BYTEA, $3::BYTEA)
+			ON CONFLICT (bucket, fullpath) DO UPDATE SET metadata = EXCLUDED.metadata
+	`
+	_, err := client.pgConn.Exec(q, []byte(bucket), []byte(key), []byte(value))
+	return err
+}
+
+// Get looks up the provided key and returns its value (or an error).
+func (client *Client) Get(key storage.Key) (storage.Value, error) {
+	return client.GetPath(storage.Key(defaultBucket), key)
+}
+
+// GetPath looks up the provided key (in the given bucket) and returns its value (or an error).
+func (client *Client) GetPath(bucket, key storage.Key) (storage.Value, error) {
+	q := "SELECT metadata FROM pathdata WHERE bucket = $1::BYTEA AND fullpath = $2::BYTEA"
+	row := client.pgConn.QueryRow(q, []byte(bucket), []byte(key))
+	var val []byte
+	err := row.Scan(&val)
+	if err == sql.ErrNoRows {
+		return nil, storage.ErrKeyNotFound.New(key.String())
+	}
+	if err != nil {
+		return nil, err
+	}
+	return val, nil
+}
+
+// Delete deletes the given key and its associated value.
+func (client *Client) Delete(key storage.Key) error {
+	return client.DeletePath(storage.Key(defaultBucket), key)
+}
+
+// DeletePath deletes the given key (in the given bucket) and its associated value.
+func (client *Client) DeletePath(bucket, key storage.Key) error {
+	q := "DELETE FROM pathdata WHERE bucket = $1::BYTEA AND fullpath = $2::BYTEA"
+	result, err := client.pgConn.Exec(q, []byte(bucket), []byte(key))
+	if err != nil {
+		return err
+	}
+	numRows, err := result.RowsAffected()
+	if err != nil {
+		return err
+	}
+	if numRows == 0 {
+		return storage.ErrKeyNotFound.New(key.String())
+	}
+	return nil
+}
+
+// List returns either a list of known keys, in order, or an error.
+func (client *Client) List(first storage.Key, limit int) (storage.Keys, error) {
+	return storage.ListKeys(client, first, limit)
+}
+
+// ReverseList returns either a list of known keys, in reverse order, or an error.
+// Starts from first and iterates backwards
+func (client *Client) ReverseList(first storage.Key, limit int) (storage.Keys, error) {
+	return storage.ReverseListKeys(client, first, limit)
+}
+
+// Close closes the client
+func (client *Client) Close() error {
+	return client.pgConn.Close()
+}
+
+// GetAll finds all values for the provided keys (up to storage.LookupLimit).
+// If more keys are provided than the maximum, an error will be returned.
+func (client *Client) GetAll(keys storage.Keys) (storage.Values, error) {
+	return client.GetAllPath(storage.Key(defaultBucket), keys)
+}
+
+// GetAllPath finds all values for the provided keys (up to storage.LookupLimit)
+// in the given bucket. if more keys are provided than the maximum, an error
+// will be returned.
+func (client *Client) GetAllPath(bucket storage.Key, keys storage.Keys) (storage.Values, error) {
+	if len(keys) > storage.LookupLimit {
+		return nil, storage.ErrLimitExceeded
+	}
+
+	q := `
+		SELECT metadata
+		FROM pathdata pd
+			RIGHT JOIN
+				unnest($2::BYTEA[]) WITH ORDINALITY pk(request, ord)
+			ON (pd.fullpath = pk.request AND pd.bucket = $1::BYTEA)
+		ORDER BY pk.ord
+	`
+	rows, err := client.pgConn.Query(q, []byte(bucket), pq.ByteaArray(keys.ByteSlices()))
+	if err != nil {
+		return nil, errs.Wrap(err)
+	}
+	values := make([]storage.Value, 0, len(keys))
+	for rows.Next() {
+		var value []byte
+		if err := rows.Scan(&value); err != nil {
+			return nil, errs.Wrap(utils.CombineErrors(err, rows.Close()))
+		}
+		values = append(values, storage.Value(value))
+	}
+	return values, utils.CombineErrors(rows.Err(), rows.Close())
+}
+
+type orderedPostgresIterator struct {
+	client         *Client
+	opts           *storage.IterateOptions
+	bucket         storage.Key
+	delimiter      byte
+	batchSize      int
+	curIndex       int
+	curRows        *sql.Rows
+	lastKeySeen    storage.Key
+	errEncountered error
+	nextQuery      func() (*sql.Rows, error)
+}
+
+// Next fills in info for the next item in an ongoing listing.
+func (opi *orderedPostgresIterator) Next(item *storage.ListItem) bool {
+	if !opi.curRows.Next() {
+		if err := opi.curRows.Close(); err != nil {
+			opi.errEncountered = errs.Wrap(err)
+			return false
+		}
+		if opi.curIndex < opi.batchSize {
+			return false
+		}
+		if err := opi.curRows.Err(); err != nil {
+			opi.errEncountered = errs.Wrap(err)
+			return false
+		}
+		newRows, err := opi.nextQuery()
+		if err != nil {
+			opi.errEncountered = errs.Wrap(err)
+			return false
+		}
+		opi.curRows = newRows
+		opi.curIndex = 0
+		if !opi.curRows.Next() {
+			if err := opi.curRows.Close(); err != nil {
+				opi.errEncountered = errs.Wrap(err)
+			}
+			return false
+		}
+	}
+	var k, v []byte
+	err := opi.curRows.Scan(&k, &v)
+	if err != nil {
+		opi.errEncountered = utils.CombineErrors(errs.Wrap(err), errs.Wrap(opi.curRows.Close()))
+		return false
+	}
+	item.Key = storage.Key(k)
+	item.Value = storage.Value(v)
+	opi.curIndex++
+	if opi.curIndex == 1 && opi.lastKeySeen.Equal(item.Key) {
+		return opi.Next(item)
+	}
+	if !opi.opts.Recurse && item.Key[len(item.Key)-1] == opi.delimiter && !item.Key.Equal(opi.opts.Prefix) {
+		item.IsPrefix = true
+		// i don't think this makes the most sense, but it's necessary to pass the storage testsuite
+		item.Value = nil
+	} else {
+		item.IsPrefix = false
+	}
+	opi.lastKeySeen = item.Key
+	return true
+}
+
+func (opi *orderedPostgresIterator) doNextQuery() (*sql.Rows, error) {
+	start := opi.lastKeySeen
+	if start == nil {
+		start = opi.opts.First
+	}
+	var query string
+	if !opi.opts.Recurse {
+		if opi.opts.Reverse {
+			query = "SELECT p, m FROM list_directory_reverse($1::BYTEA, $2::BYTEA, $3::BYTEA, $4) ld(p, m)"
+		} else {
+			query = "SELECT p, m FROM list_directory($1::BYTEA, $2::BYTEA, $3::BYTEA, $4) ld(p, m)"
+		}
+	} else {
+		startCmp := ">="
+		orderDir := ""
+		if opi.opts.Reverse {
+			startCmp = "<="
+			orderDir = " DESC"
+		}
+		query = fmt.Sprintf(`
+			SELECT fullpath, metadata
+			  FROM pathdata
+			 WHERE bucket = $1::BYTEA
+			   AND ($2::BYTEA = ''::BYTEA OR fullpath >= $2::BYTEA)
+			   AND ($2::BYTEA = ''::BYTEA OR fullpath < bytea_increment($2::BYTEA))
+			   AND ($3::BYTEA = ''::BYTEA OR fullpath %s $3::BYTEA)
+			 ORDER BY fullpath%s
+			 LIMIT $4
+		`, startCmp, orderDir)
+	}
+	return opi.client.pgConn.Query(query, []byte(opi.bucket), []byte(opi.opts.Prefix), []byte(start), opi.batchSize+1)
+}
+
+func (opi *orderedPostgresIterator) Close() error {
+	return utils.CombineErrors(opi.errEncountered, opi.curRows.Close())
+}
+
+func newOrderedPostgresIterator(pgClient *Client, opts storage.IterateOptions, batchSize int) (*orderedPostgresIterator, error) {
+	if opts.Prefix == nil {
+		opts.Prefix = storage.Key("")
+	}
+	if opts.First == nil {
+		opts.First = storage.Key("")
+	}
+	opi := &orderedPostgresIterator{
+		client:    pgClient,
+		opts:      &opts,
+		bucket:    storage.Key(defaultBucket),
+		delimiter: byte('/'),
+		batchSize: batchSize,
+		curIndex:  0,
+	}
+	opi.nextQuery = opi.doNextQuery
+	newRows, err := opi.nextQuery()
+	if err != nil {
+		return nil, err
+	}
+	opi.curRows = newRows
+	return opi, nil
+}
+
+// Iterate iterates over items based on opts
+func (client *Client) Iterate(opts storage.IterateOptions, fn func(storage.Iterator) error) (err error) {
+	opi, err := newOrderedPostgresIterator(client, opts, defaultBatchSize)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err = utils.CombineErrors(err, opi.Close())
+	}()
+
+	return fn(opi)
+}

storage/postgreskv/client_test.go

@@ -0,0 +1,124 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package postgreskv
+
+import (
+	"database/sql"
+	"flag"
+	"os"
+	"testing"
+
+	"github.com/lib/pq"
+	"github.com/zeebo/errs"
+	"go.uber.org/zap/zaptest"
+
+	"storj.io/storj/pkg/utils"
+	"storj.io/storj/storage"
+	"storj.io/storj/storage/storelogger"
+	"storj.io/storj/storage/testsuite"
+)
+
+const (
+	// this connstring is expected to work under the storj-test docker-compose instance
+	defaultPostgresConn = "postgres://pointerdb:pg-secret-pass@test-postgres-pointerdb/pointerdb?sslmode=disable"
+)
+
+var (
+	testPostgres = flag.String("postgres-test-db", os.Getenv("STORJ_POSTGRESKV_TEST"), "PostgreSQL test database connection string")
+)
+
+func newTestPostgres(t testing.TB) (store *Client, cleanup func()) {
+	if *testPostgres == "" {
+		t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", defaultPostgresConn)
+	}
+
+	pgdb, err := New(*testPostgres)
+	if err != nil {
+		t.Fatalf("init: %v", err)
+	}
+
+	return pgdb, func() {
+		if err := pgdb.Close(); err != nil {
+			t.Fatalf("failed to close db: %v", err)
+		}
+	}
+}
+
+func TestSuite(t *testing.T) {
+	store, cleanup := newTestPostgres(t)
+	defer cleanup()
+
+	zap := zaptest.NewLogger(t)
+	testsuite.RunTests(t, storelogger.New(zap, store))
+}
+
+func BenchmarkSuite(b *testing.B) {
+	store, cleanup := newTestPostgres(b)
+	defer cleanup()
+
+	testsuite.RunBenchmarks(b, store)
+}
+
+func bulkImport(db *sql.DB, iter storage.Iterator) (err error) {
+	txn, err2 := db.Begin()
+	if err2 != nil {
+		return errs.New("Failed to start transaction: %v", err2)
+	}
+	defer func() {
+		if err == nil {
+			err = utils.CombineErrors(err, txn.Commit())
+		} else {
+			err = utils.CombineErrors(err, txn.Rollback())
+		}
+	}()
+
+	stmt, err2 := txn.Prepare(pq.CopyIn("pathdata", "bucket", "fullpath", "metadata"))
+	if err2 != nil {
+		return errs.New("Failed to initialize COPY FROM: %v", err)
+	}
+	defer func() {
+		err2 := stmt.Close()
+		if err2 != nil {
+			err = utils.CombineErrors(err, errs.New("Failed to close COPY FROM statement: %v", err2))
+		}
+	}()
+
+	var item storage.ListItem
+	for iter.Next(&item) {
+		if _, err := stmt.Exec([]byte(""), []byte(item.Key), []byte(item.Value)); err != nil {
+			return err
+		}
+	}
+	if _, err = stmt.Exec(); err != nil {
+		return errs.New("Failed to complete COPY FROM: %v", err)
+	}
+	return nil
+}
+
+func bulkDelete(db *sql.DB) error {
+	_, err := db.Exec("TRUNCATE pathdata")
+	if err != nil {
+		return errs.New("Failed to TRUNCATE pathdata table: %v", err)
+	}
+	return nil
+}
+
+type pgLongBenchmarkStore struct {
+	*Client
+}
+
+func (store *pgLongBenchmarkStore) BulkImport(iter storage.Iterator) error {
+	return bulkImport(store.pgConn, iter)
+}
+
+func (store *pgLongBenchmarkStore) BulkDelete() error {
+	return bulkDelete(store.pgConn)
+}
+
+func BenchmarkSuiteLong(b *testing.B) {
+	store, cleanup := newTestPostgres(b)
+	defer cleanup()
+
+	testsuite.BenchmarkPathOperationsInLargeDb(b, &pgLongBenchmarkStore{store})
+}

storage/postgreskv/common.go

@@ -0,0 +1,11 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package postgreskv
+
+import (
+	"github.com/zeebo/errs"
+)
+
+// Error is the default postgreskv errs class
+var Error = errs.Class("postgreskv error")

storage/postgreskv/schema/2018092201_initial-tables.down.sql

@@ -0,0 +1,9 @@
+DROP FUNCTION list_directory_reverse(BYTEA, BYTEA, BYTEA, INTEGER);
+DROP FUNCTION list_directory(BYTEA, BYTEA, BYTEA, INTEGER);
+DROP TYPE path_and_meta;
+DROP FUNCTION component_increment(BYTEA, INTEGER);
+DROP FUNCTION bytea_increment(BYTEA);
+DROP FUNCTION truncate_after(BYTEA, INTEGER, INTEGER);
+DROP VIEW pathdata_pretty;
+DROP TABLE pathdata;
+DROP TABLE buckets;

storage/postgreskv/schema/2018092201_initial-tables.up.sql

@@ -0,0 +1,194 @@
+CREATE TABLE buckets (
+    bucketname BYTEA
+        PRIMARY KEY,
+    delim INT
+        NOT NULL
+        CHECK (delim > 0 AND delim < 255)
+);
+
+-- until the KeyValueStore interface supports passing the bucket separately, or
+-- until storj actually supports changing the delimiter character per bucket, this
+-- dummy row should suffice for everything.
+INSERT INTO buckets (bucketname, delim) VALUES (''::BYTEA, ascii('/'));
+
+
+CREATE TABLE pathdata (
+    bucket BYTEA
+        NOT NULL
+        REFERENCES buckets (bucketname),
+    fullpath BYTEA
+        NOT NULL
+        CHECK (fullpath <> ''),
+    metadata BYTEA
+        NOT NULL,
+
+    PRIMARY KEY (bucket, fullpath)
+);
+
+CREATE VIEW pathdata_pretty AS
+    SELECT encode(bucket, 'escape') AS bucket,
+           encode(fullpath, 'escape') AS fullpath,
+           encode(metadata, 'escape') AS metadata
+      FROM pathdata;
+
+
+-- given a path as might be found in the pathdata table, truncate it after the next delimiter to be
+-- found at or after 'afterpos', if any.
+--
+-- Examples:
+--
+--     truncate_after(''::BYTEA,            ascii('/'), 1)  ->  ''::BYTEA
+--     truncate_after('foo'::BYTEA,         ascii('/'), 1)  ->  'foo'::BYTEA
+--     truncate_after('foo/'::BYTEA,        ascii('/'), 1)  ->  'foo/'::BYTEA
+--     truncate_after('foo/bar/baz'::BYTEA, ascii('/'), 4)  ->  'foo/'::BYTEA
+--     truncate_after('foo/bar/baz'::BYTEA, ascii('/'), 5)  ->  'foo/bar/'::BYTEA
+--     truncate_after('foo/bar/baz'::BYTEA, ascii('/'), 8)  ->  'foo/bar/'::BYTEA
+--     truncate_after('foo/bar/baz'::BYTEA, ascii('/'), 9)  ->  'foo/bar/baz'::BYTEA
+--     truncate_after('foo//bar/bz'::BYTEA, ascii('/'), 4)  ->  'foo/'::BYTEA
+--     truncate_after('foo//bar/bz'::BYTEA, ascii('/'), 5)  ->  'foo//'::BYTEA
+--
+CREATE FUNCTION truncate_after(bpath BYTEA, delim INTEGER, afterpos INTEGER) RETURNS BYTEA AS $$
+DECLARE
+    suff BYTEA;
+    delimpos INTEGER;
+BEGIN
+    suff := substring(bpath FROM afterpos);
+    delimpos := position(set_byte(' '::BYTEA, 0, delim) IN suff);
+    IF delimpos > 0 THEN
+        RETURN substring(bpath FROM 1 FOR (afterpos + delimpos - 1));
+    END IF;
+    RETURN bpath;
+END;
+$$ LANGUAGE 'plpgsql' IMMUTABLE STRICT;
+
+
+CREATE FUNCTION bytea_increment(b BYTEA) RETURNS BYTEA AS $$
+BEGIN
+    WHILE b <> ''::BYTEA AND get_byte(b, octet_length(b) - 1) = 255 LOOP
+        b := substring(b FROM 1 FOR octet_length(b) - 1);
+    END LOOP;
+    IF b = ''::BYTEA THEN
+        RETURN NULL;
+    END IF;
+    RETURN set_byte(b, octet_length(b) - 1, get_byte(b, octet_length(b) - 1) + 1);
+END;
+$$ LANGUAGE 'plpgsql' IMMUTABLE STRICT;
+
+
+-- Given a path as might be found in the pathdata table, with a delimeter appended if that path
+-- has any sub-elements, return the next possible path that _could_ be in the table (skipping over
+-- any potential sub-elements).
+--
+-- Examples:
+--
+--     component_increment('/'::BYTEA, ascii('/'))  ->  '0'::BYTEA
+--         (nothing can be between '/' and '0' other than subpaths under '/')
+--
+--     component_increment('/foo/bar/'::BYTEA, ascii('/'))  ->  '/foo/bar0'::BYTEA
+--
+--     component_increment('/foo/barboom'::BYTEA, ascii('/'))  ->  ('/foo/barboom' || E'\\x00')
+--         (nothing can be between '/foo/barboom' and '/foo/barboom\x00' in normal BYTEA ordering)
+--
+--     component_increment(E'\\xFEFFFF'::BYTEA, 255)  ->  E'\\xFF'::BYTEA
+--
+CREATE FUNCTION component_increment(bpath BYTEA, delim INTEGER) RETURNS BYTEA AS $$
+    SELECT CASE WHEN get_byte(bpath, octet_length(bpath) - 1) = delim
+                THEN CASE WHEN delim = 255
+                          THEN bytea_increment(bpath)
+                          ELSE set_byte(bpath, octet_length(bpath) - 1, delim + 1)
+                     END
+                ELSE bpath || E'\\x00'::BYTEA
+           END;
+$$ LANGUAGE 'sql' IMMUTABLE STRICT;
+
+
+CREATE TYPE path_and_meta AS (
+    fullpath BYTEA,
+    metadata BYTEA
+);
+
+CREATE FUNCTION list_directory(bucket BYTEA, dirpath BYTEA, start_at BYTEA = ''::BYTEA, limit_to INTEGER = NULL)
+RETURNS SETOF path_and_meta AS $$
+    WITH RECURSIVE
+        inputs AS (
+            SELECT CASE WHEN dirpath = ''::BYTEA THEN NULL ELSE dirpath END AS range_low,
+                   CASE WHEN dirpath = ''::BYTEA THEN NULL ELSE bytea_increment(dirpath) END AS range_high,
+                   octet_length(dirpath) + 1 AS component_start,
+                   b.delim AS delim,
+                   b.bucketname AS bucket
+              FROM buckets b
+             WHERE bucketname = bucket
+        ),
+        distinct_prefix (truncatedpath) AS (
+            SELECT (SELECT truncate_after(pd.fullpath, i.delim, i.component_start)
+                      FROM pathdata pd
+                     WHERE (i.range_low IS NULL OR pd.fullpath > i.range_low)
+                       AND (i.range_high IS NULL OR pd.fullpath < i.range_high)
+                       AND (start_at = '' OR pd.fullpath >= start_at)
+                       AND pd.bucket = i.bucket
+                     ORDER BY pd.fullpath
+                     LIMIT 1)
+              FROM inputs i
+            UNION ALL
+            SELECT (SELECT truncate_after(pd.fullpath, i.delim, i.component_start)
+                      FROM pathdata pd
+                     WHERE pd.fullpath >= component_increment(pfx.truncatedpath, i.delim)
+                       AND (i.range_high IS NULL OR pd.fullpath < i.range_high)
+                       AND pd.bucket = i.bucket
+                     ORDER BY pd.fullpath
+                     LIMIT 1)
+              FROM distinct_prefix pfx, inputs i
+             WHERE pfx.truncatedpath IS NOT NULL
+        )
+    SELECT pfx.truncatedpath AS fullpath,
+           pd.metadata
+      FROM distinct_prefix pfx LEFT OUTER JOIN pathdata pd ON pfx.truncatedpath = pd.fullpath
+     WHERE pfx.truncatedpath IS NOT NULL
+    UNION ALL
+    -- this one, if it exists, can't be part of distinct_prefix (or it would cause us to skip over all
+    -- subcontents of the prefix we're looking for), so we tack it on here
+    SELECT pd.fullpath, pd.metadata FROM pathdata pd, inputs i WHERE pd.fullpath = i.range_low
+     ORDER BY fullpath
+     LIMIT limit_to;
+$$ LANGUAGE 'sql' STABLE;
+
+CREATE FUNCTION list_directory_reverse(bucket BYTEA, dirpath BYTEA, start_at BYTEA = ''::BYTEA, limit_to INTEGER = NULL)
+RETURNS SETOF path_and_meta AS $$
+    WITH RECURSIVE
+        inputs AS (
+            SELECT CASE WHEN dirpath = ''::BYTEA THEN NULL ELSE dirpath END AS range_low,
+                   CASE WHEN dirpath = ''::BYTEA THEN NULL ELSE bytea_increment(dirpath) END AS range_high,
+                   octet_length(dirpath) + 1 AS component_start,
+                   b.delim AS delim,
+                   b.bucketname AS bucket
+              FROM buckets b
+             WHERE bucketname = bucket
+        ),
+        distinct_prefix (truncatedpath) AS (
+            SELECT (SELECT truncate_after(pd.fullpath, i.delim, i.component_start)
+                      FROM pathdata pd
+                     WHERE (i.range_low IS NULL OR pd.fullpath >= i.range_low)
+                       AND (i.range_high IS NULL OR pd.fullpath < i.range_high)
+                       AND (start_at = '' OR pd.fullpath <= start_at)
+                       AND pd.bucket = i.bucket
+                     ORDER BY pd.fullpath DESC
+                     LIMIT 1)
+              FROM inputs i
+            UNION ALL
+            SELECT (SELECT truncate_after(pd.fullpath, i.delim, i.component_start)
+                      FROM pathdata pd
+                     WHERE (i.range_low IS NULL OR pd.fullpath >= i.range_low)
+                       AND pd.fullpath < pfx.truncatedpath
+                       AND pd.bucket = i.bucket
+                     ORDER BY pd.fullpath DESC
+                     LIMIT 1)
+              FROM distinct_prefix pfx, inputs i
+             WHERE pfx.truncatedpath IS NOT NULL
+        )
+    SELECT pfx.truncatedpath AS fullpath,
+           pd.metadata
+      FROM distinct_prefix pfx LEFT OUTER JOIN pathdata pd ON pfx.truncatedpath = pd.fullpath
+     WHERE pfx.truncatedpath IS NOT NULL
+     ORDER BY fullpath DESC
+     LIMIT limit_to;
+$$ LANGUAGE 'sql' STABLE;

storage/postgreskv/schema/migrate.go

@@ -0,0 +1,44 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+//go:generate go-bindata -o data.go -pkg schema -ignore ".*go" .
+
+package schema
+
+import (
+	"database/sql"
+
+	// this isn't strictly necessary to import, but we just need the go-bindata
+	// binary for the above generate, and go mod tidy wants to remove
+	// go-bindata because nothing else is importing it. better solutions?
+	_ "github.com/go-bindata/go-bindata"
+
+	"github.com/golang-migrate/migrate/v3"
+	"github.com/golang-migrate/migrate/v3/database/postgres"
+	"github.com/golang-migrate/migrate/v3/source/go_bindata"
+)
+
+// PrepareDB applies schema migrations as necessary to the given database to
+// get it up to date.
+func PrepareDB(db *sql.DB) error {
+	srcDriver, err := bindata.WithInstance(bindata.Resource(AssetNames(),
+		func(name string) ([]byte, error) {
+			return Asset(name)
+		}))
+	if err != nil {
+		return err
+	}
+	dbDriver, err := postgres.WithInstance(db, &postgres.Config{})
+	if err != nil {
+		return err
+	}
+	m, err := migrate.NewWithInstance("go-bindata migrations", srcDriver, "postgreskv db", dbDriver)
+	if err != nil {
+		return err
+	}
+	err = m.Up()
+	if err == migrate.ErrNoChange {
+		err = nil
+	}
+	return err
+}

storage/redis/client.go

@@ -117,9 +117,9 @@ func (client *Client) Close() error {
 	return client.db.Close()
 }
 
-// GetAll is the bulk method for gets from the redis data store
+// GetAll is the bulk method for gets from the redis data store.
-// The maximum keys returned will be 100. If more than that is requested an
+// The maximum keys returned will be storage.LookupLimit. If more than that
-// error will be returned
+// is requested, an error will be returned
 func (client *Client) GetAll(keys storage.Keys) (storage.Values, error) {
 	if len(keys) > storage.LookupLimit {
 		return nil, storage.ErrLimitExceeded

storage/testsuite/long_bench.go

@@ -0,0 +1,775 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package testsuite
+
+import (
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/zeebo/errs"
+
+	"storj.io/storj/pkg/utils"
+	"storj.io/storj/storage"
+)
+
+const (
+	maxProblems = 10
+
+	// the largest and deepest level-2 directory in the dataset
+	largestLevel2Directory = "Peronosporales/hateless/"
+
+	// the directory in the dataset with the most immediate children
+	largestSingleDirectory = "Peronosporales/hateless/tod/unricht/sniveling/Puyallup/"
+)
+
+var (
+	// see https://github.com/storj/test-path-corpus
+	longBenchmarksData = flag.String("test-bench-long", "", "Run the long benchmark suite against eligible KeyValueStores using the given paths dataset")
+
+	noInitDb  = flag.Bool("test-bench-long-noinit", false, "Don't import the large dataset for the long benchmarks; assume it is already loaded")
+	noCleanDb = flag.Bool("test-bench-long-noclean", false, "Don't clean the long benchmarks KeyValueStore after running, for debug purposes")
+)
+
+func interpolateInput(input []byte) ([]byte, error) {
+	output := make([]byte, 0, len(input))
+	var bytesConsumed int
+	var next byte
+
+	for pos := 0; pos < len(input); pos += bytesConsumed {
+		if input[pos] == '\\' {
+			bytesConsumed = 2
+			if pos+1 >= len(input) {
+				return output, errs.New("encoding error in input: escape at end-of-string")
+			}
+			switch input[pos+1] {
+			case 'x':
+				if pos+3 >= len(input) {
+					return output, errs.New("encoding error in input: incomplete \\x escape")
+				}
+				nextVal, err := strconv.ParseUint(string(input[pos+2:pos+4]), 16, 8)
+				if err != nil {
+					return output, errs.New("encoding error in input: invalid \\x escape: %v", err)
+				}
+				next = byte(nextVal)
+				bytesConsumed = 4
+			case 't':
+				next = '\t'
+			case 'n':
+				next = '\n'
+			case 'r':
+				next = '\r'
+			case '\\':
+				next = '\\'
+			default:
+				next = input[pos+1]
+			}
+		} else {
+			next = input[pos]
+			bytesConsumed = 1
+		}
+		output = append(output, next)
+	}
+	return output, nil
+}
+
+// KVInputIterator is passed to the BulkImport method on BulkImporter-satisfying objects. It will
+// iterate over a fairly large list of paths that should be imported for testing purposes.
+type KVInputIterator struct {
+	itemNo     int
+	scanner    *bufio.Scanner
+	fileName   string
+	err        error
+	reachedEnd bool
+	closeFunc  func() error
+}
+
+func newKVInputIterator(pathToFile string) (*KVInputIterator, error) {
+	kvi := &KVInputIterator{fileName: pathToFile}
+	pathData, err := os.Open(pathToFile)
+	if err != nil {
+		return nil, errs.New("Failed to open file with test data (expected at %q): %v", pathToFile, err)
+	}
+	var reader io.Reader = pathData
+	if strings.HasSuffix(pathToFile, ".gz") {
+		gzReader, err := gzip.NewReader(pathData)
+		if err != nil {
+			return nil, utils.CombineErrors(
+				errs.New("Failed to create gzip reader: %v", err),
+				pathData.Close())
+		}
+		kvi.closeFunc = func() error { return utils.CombineErrors(gzReader.Close(), pathData.Close()) }
+		reader = gzReader
+	} else {
+		kvi.closeFunc = pathData.Close
+	}
+	kvi.scanner = bufio.NewScanner(reader)
+	return kvi, nil
+}
+
+// Next should be called by BulkImporter instances in order to advance the iterator. It fills in
+// a storage.ListItem instance, and returns a boolean indicating whether to continue. When false is
+// returned, iteration should stop and nothing is expected to be changed in item.
+func (kvi *KVInputIterator) Next(item *storage.ListItem) bool {
+	if !kvi.scanner.Scan() {
+		kvi.reachedEnd = true
+		kvi.err = kvi.scanner.Err()
+		return false
+	}
+	if kvi.err != nil {
+		return false
+	}
+	kvi.itemNo++
+	parts := bytes.Split(kvi.scanner.Bytes(), []byte("\t"))
+	if len(parts) != 3 {
+		kvi.err = errs.New("Invalid data in %q on line %d: has %d fields", kvi.fileName, kvi.itemNo, len(parts))
+		return false
+	}
+	k, err := interpolateInput(parts[1])
+	if err != nil {
+		kvi.err = errs.New("Failed to read key data from %q on line %d: %v", kvi.fileName, kvi.itemNo, err)
+		return false
+	}
+	v, err := interpolateInput(parts[2])
+	if err != nil {
+		kvi.err = errs.New("Failed to read value data from %q on line %d: %v", kvi.fileName, kvi.itemNo, err)
+		return false
+	}
+	item.Key = storage.Key(k)
+	item.Value = storage.Value(v)
+	item.IsPrefix = false
+	return true
+}
+
+// Error() returns the last error encountered while iterating over the input file. This must be
+// checked after iteration completes, at least.
+func (kvi *KVInputIterator) Error() error {
+	return kvi.err
+}
+
+func openTestData(tb testing.TB) *KVInputIterator {
+	tb.Helper()
+	inputIter, err := newKVInputIterator(*longBenchmarksData)
+	if err != nil {
+		tb.Fatal(err)
+	}
+	return inputIter
+}
+
+// BulkImporter identifies KV storage facilities that can do bulk importing of items more
+// efficiently than inserting one-by-one.
+type BulkImporter interface {
+	BulkImport(storage.Iterator) error
+}
+
+// BulkCleaner identifies KV storage facilities that can delete all items efficiently.
+type BulkCleaner interface {
+	BulkDelete() error
+}
+
+// BenchmarkPathOperationsInLargeDb runs the "long benchmarks" suite for KeyValueStore instances.
+func BenchmarkPathOperationsInLargeDb(b *testing.B, store storage.KeyValueStore) {
+	if *longBenchmarksData == "" {
+		b.Skip("Long benchmarks not enabled.")
+	}
+
+	initStore(b, store)
+
+	doTest := func(name string, testFunc func(*testing.B, storage.KeyValueStore)) {
+		b.Run(name, func(bb *testing.B) {
+			for i := 0; i < bb.N; i++ {
+				testFunc(bb, store)
+			}
+		})
+	}
+
+	doTest("DeepRecursive", deepRecursive)
+	doTest("DeepRecursiveReverse", deepRecursiveReverse)
+	doTest("DeepNonRecursive", deepNonRecursive)
+	doTest("DeepNonRecursiveReverse", deepNonRecursiveReverse)
+	doTest("ShallowRecursive", shallowRecursive)
+	doTest("ShallowRecursiveReverse", shallowRecursiveReverse)
+	doTest("ShallowNonRecursive", shallowNonRecursive)
+	doTest("ShallowNonRecursiveReverse", shallowNonRecursiveReverse)
+	doTest("TopRecursiveLimit", topRecursiveLimit)
+	doTest("TopRecursiveLimitReverse", topRecursiveLimitReverse)
+	doTest("TopRecursiveStartAt", topRecursiveStartAt)
+	doTest("TopRecursiveStartAtReverse", topRecursiveStartAtReverse)
+	doTest("TopNonRecursive", topNonRecursive)
+	doTest("TopNonRecursiveReverse", topNonRecursiveReverse)
+
+	cleanupStore(b, store)
+}
+
+func importBigPathset(tb testing.TB, store storage.KeyValueStore) {
+	// make sure this is an empty db, or else refuse to run
+	if !isEmptyKVStore(tb, store) {
+		tb.Fatal("Provided KeyValueStore is not empty. The long benchmarks are destructive. Not running!")
+	}
+
+	inputIter := openTestData(tb)
+	defer func() {
+		if err := inputIter.closeFunc(); err != nil {
+			tb.Logf("Failed to close test data stream: %v", err)
+		}
+	}()
+
+	importer, ok := store.(BulkImporter)
+	if ok {
+		tb.Log("Performing bulk import...")
+		err := importer.BulkImport(inputIter)
+
+		if err != nil {
+			errStr := "Provided KeyValueStore failed to import data"
+			if inputIter.reachedEnd {
+				errStr += " after iterating over all input data"
+			} else {
+				errStr += fmt.Sprintf(" after iterating over %d lines of input data", inputIter.itemNo)
+			}
+			tb.Fatalf("%s: %v", errStr, err)
+		}
+	} else {
+		tb.Log("Performing manual import...")
+
+		var item storage.ListItem
+		for inputIter.Next(&item) {
+			if err := store.Put(item.Key, item.Value); err != nil {
+				tb.Fatalf("Provided KeyValueStore failed to insert data (%q, %q): %v", item.Key, item.Value, err)
+			}
+		}
+	}
+	if err := inputIter.Error(); err != nil {
+		tb.Fatalf("Failed to iterate over input data during import. Error was %v", err)
+	}
+	if !inputIter.reachedEnd {
+		tb.Fatal("Provided KeyValueStore failed to exhaust input iterator")
+	}
+}
+
+func initStore(b *testing.B, store storage.KeyValueStore) {
+	b.Helper()
+
+	if !*noInitDb {
+		// can't find a way to run the import and cleanup as sub-benchmarks, while still requiring
+		// that they be run once and only once, and aborting the whole benchmark if import fails.
+		// we don't want the time it takes to count against the first sub-benchmark only, so we
+		// stop the timer. however, we do care about the time that import and cleanup take, though,
+		// so we'll at least log it.
+		b.StopTimer()
+		tStart := time.Now()
+		importBigPathset(b, store)
+		b.Logf("importing took %s", time.Since(tStart).String())
+		b.StartTimer()
+	}
+}
+
+func cleanupStore(b *testing.B, store storage.KeyValueStore) {
+	b.Helper()
+	if !*noCleanDb {
+		tStart := time.Now()
+		cleanupBigPathset(b, store)
+		b.Logf("cleanup took %s", time.Since(tStart).String())
+	}
+}
+
+type verifyOpts struct {
+	iterateOpts   storage.IterateOptions
+	doIterations  int
+	batchSize     int
+	expectCount   int
+	expectLastKey storage.Key
+}
+
+func benchAndVerifyIteration(b *testing.B, store storage.KeyValueStore, opts *verifyOpts) {
+	problems := 0
+	iteration := 0
+
+	errMsg := func(tmpl string, args ...interface{}) string {
+		errMsg1 := fmt.Sprintf(tmpl, args...)
+		return fmt.Sprintf("[on iteration %d/%d, with opts %+v]: %s", iteration, opts.doIterations, opts.iterateOpts, errMsg1)
+	}
+
+	errorf := func(tmpl string, args ...interface{}) {
+		b.Error(errMsg(tmpl, args...))
+		problems++
+		if problems > maxProblems {
+			b.Fatal("Too many problems")
+		}
+	}
+
+	fatalf := func(tmpl string, args ...interface{}) {
+		b.Fatal(errMsg(tmpl, args...))
+	}
+
+	expectRemaining := opts.expectCount
+	totalFound := 0
+	var lastKey storage.Key
+	var bytesTotal int64
+	lookupSize := opts.batchSize
+
+	for iteration = 1; iteration <= opts.doIterations; iteration++ {
+		results, err := iterateItems(store, opts.iterateOpts, lookupSize)
+		if err != nil {
+			fatalf("Failed to call iterateItems(): %v", err)
+		}
+		if len(results) == 0 {
+			// we can't continue to iterate
+			fatalf("iterateItems() got 0 items")
+		}
+		if len(results) > lookupSize {
+			fatalf("iterateItems() returned _more_ items than limit: %d>%d", len(results), lookupSize)
+		}
+		if iteration > 0 && results[0].Key.Equal(lastKey) {
+			// fine and normal
+			results = results[1:]
+		}
+		expectRemaining -= len(results)
+		if len(results) != opts.batchSize && expectRemaining != 0 {
+			errorf("iterateItems read %d items instead of %d", len(results), opts.batchSize)
+		}
+		for n, result := range results {
+			totalFound++
+			bytesTotal += int64(len(result.Key)) + int64(len(result.Value))
+			if result.Key.IsZero() {
+				errorf("got an empty key among the results at n=%d!", n)
+				continue
+			}
+			if result.Key.Equal(lastKey) {
+				errorf("got the same key (%q) twice in a row, not on a lookup boundary!", lastKey)
+			}
+			if opts.iterateOpts.Reverse && !lastKey.IsZero() && lastKey.Less(result.Key) {
+				errorf("KeyValueStore returned items out of order! %q > %q", result.Key, lastKey)
+			}
+			if !opts.iterateOpts.Reverse && result.Key.Less(lastKey) {
+				errorf("KeyValueStore returned items out of order! %q < %q", result.Key, lastKey)
+			}
+			if result.IsPrefix {
+				if !result.Value.IsZero() {
+					errorf("Expected no metadata for IsPrefix item %q, but got %q", result.Key, result.Value)
+				}
+				if result.Key[len(result.Key)-1] != byte('/') {
+					errorf("Expected key for IsPrefix item %q to end in /, but it does not", result.Key)
+				}
+			} else {
+				valAsNum, err := strconv.ParseUint(string(result.Value), 10, 32)
+				if err != nil {
+					errorf("Expected metadata for key %q to hold a decimal integer, but it has %q", result.Key, result.Value)
+				} else if int(valAsNum) != len(result.Key) {
+					errorf("Expected metadata for key %q to be %d, but it has %q", result.Key, len(result.Key), result.Value)
+				}
+			}
+			lastKey = result.Key
+		}
+		if len(results) > 0 {
+			opts.iterateOpts.First = results[len(results)-1].Key
+		}
+		lookupSize = opts.batchSize + 1 // subsequent queries will start with the last element previously returned
+	}
+	b.SetBytes(bytesTotal)
+
+	if totalFound != opts.expectCount {
+		b.Fatalf("Expected to read %d items in total, but got %d", opts.expectCount, totalFound)
+	}
+	if !opts.expectLastKey.IsZero() {
+		if diff := cmp.Diff(opts.expectLastKey.String(), lastKey.String()); diff != "" {
+			b.Fatalf("KeyValueStore got wrong last item: (-want +got)\n%s", diff)
+		}
+	}
+}
+
+func deepRecursive(b *testing.B, store storage.KeyValueStore) {
+	opts := &verifyOpts{
+		iterateOpts: storage.IterateOptions{
+			Prefix:  storage.Key(largestLevel2Directory),
+			Recurse: true,
+		},
+	}
+
+	// these are not expected to exhaust all available items
+	opts.doIterations = 500
+	opts.batchSize = storage.LookupLimit
+	opts.expectCount = opts.doIterations * opts.batchSize
+
+	// verify with:
+	//     select encode(fullpath, 'escape') from (
+	//         select rank() over (order by fullpath), fullpath from pathdata where fullpath > $1::bytea
+	//     ) x where rank = ($2 * $3);
+	// where $1 = largestLevel2Directory, $2 = doIterations, and $3 = batchSize
+	opts.expectLastKey = storage.Key("Peronosporales/hateless/tod/extrastate/firewood/renomination/cletch/herotheism/aluminiferous/nub")
+
+	benchAndVerifyIteration(b, store, opts)
+}
+
+func deepRecursiveReverse(b *testing.B, store storage.KeyValueStore) {
+	opts := &verifyOpts{
+		iterateOpts: storage.IterateOptions{
+			Prefix:  storage.Key(largestLevel2Directory),
+			Recurse: true,
+			Reverse: true,
+		},
+	}
+
+	// these are not expected to exhaust all available items
+	opts.doIterations = 500
+	opts.batchSize = storage.LookupLimit
+	opts.expectCount = opts.doIterations * opts.batchSize
+
+	// verify with:
+	//     select encode(fullpath, 'escape') from (
+	//         select rank() over (order by fullpath desc), fullpath from pathdata
+	//             where fullpath < bytea_increment($1::bytea)
+	//     ) x where rank = ($2 * $3);
+	// where $1 = largestLevel2Directory, $2 = doIterations, and $3 = batchSize
+	opts.expectLastKey = storage.Key("Peronosporales/hateless/apetaly/poikilocythemia/capped/abrash/dugout/notodontid/jasponyx/cassican/brunelliaceous")
+
+	benchAndVerifyIteration(b, store, opts)
+}
+
+func deepNonRecursive(b *testing.B, store storage.KeyValueStore) {
+	opts := &verifyOpts{
+		iterateOpts: storage.IterateOptions{
+			Prefix:  storage.Key(largestLevel2Directory),
+			Recurse: false,
+		},
+		doIterations: 1,
+		batchSize:    10000,
+	}
+
+	// verify with:
+	//     select count(*) from list_directory(''::bytea, $1::bytea) ld(fp, md);
+	// where $1 is largestLevel2Directory
+	opts.expectCount = 119
+
+	// verify with:
+	//     select encode(fp, 'escape') from (
+	//         select * from list_directory(''::bytea, $1::bytea) ld(fp, md)
+	//     ) x order by fp desc limit 1;
+	// where $1 is largestLevel2Directory
+	opts.expectLastKey = storage.Key("Peronosporales/hateless/xerophily/")
+
+	benchAndVerifyIteration(b, store, opts)
+}
+
+func deepNonRecursiveReverse(b *testing.B, store storage.KeyValueStore) {
+	opts := &verifyOpts{
+		iterateOpts: storage.IterateOptions{
+			Prefix:  storage.Key(largestLevel2Directory),
+			Recurse: false,
+			Reverse: true,
+		},
+		doIterations: 1,
+		batchSize:    10000,
+	}
+
+	// verify with:
+	//     select count(*) from list_directory(''::bytea, $1::bytea) ld(fp, md);
+	// where $1 is largestLevel2Directory
+	opts.expectCount = 119
+
+	// verify with:
+	//     select encode(fp, 'escape') from (
+	//         select * from list_directory(''::bytea, $1::bytea) ld(fp, md)
+	//     ) x order by fp limit 1;
+	// where $1 is largestLevel2Directory
+	opts.expectLastKey = storage.Key("Peronosporales/hateless/Absyrtus")
+
+	benchAndVerifyIteration(b, store, opts)
+}
+
+func shallowRecursive(b *testing.B, store storage.KeyValueStore) {
+	opts := &verifyOpts{
+		iterateOpts: storage.IterateOptions{
+			Prefix:  storage.Key(largestSingleDirectory),
+			Recurse: true,
+		},
+	}
+
+	// verify with:
+	//     select count(*) from pathdata
+	//         where fullpath > $1::bytea and fullpath < bytea_increment($1::bytea);
+	// where $1 = largestSingleDirectory
+	opts.expectCount = 18574
+
+	// verify with:
+	//     select convert_from(fullpath, 'UTF8') from pathdata
+	//         where fullpath > $1::bytea and fullpath < bytea_increment($1::bytea)
+	//         order by fullpath desc limit 1;
+	// where $1 = largestSingleDirectory
+	opts.expectLastKey = storage.Key("Peronosporales/hateless/tod/unricht/sniveling/Puyallup/élite")
+
+	// i didn't plan it this way, but expectedCount happens to have some nicely-sized factors for
+	// our purposes with no messy remainder. 74 * 251 = 18574
+	opts.doIterations = 74
+	opts.batchSize = 251
+
+	benchAndVerifyIteration(b, store, opts)
+}
+
+func shallowRecursiveReverse(b *testing.B, store storage.KeyValueStore) {
+	opts := &verifyOpts{
+		iterateOpts: storage.IterateOptions{
+			Prefix:  storage.Key(largestSingleDirectory),
+			Recurse: true,
+			Reverse: true,
+		},
+	}
+
+	// verify with:
+	//     select count(*) from pathdata
+	//         where fullpath > $1::bytea and fullpath < bytea_increment($1::bytea);
+	// where $1 = largestSingleDirectory
+	opts.expectCount = 18574
+
+	// verify with:
+	//     select convert_from(fullpath, 'UTF8') from pathdata
+	//         where fullpath > $1::bytea and fullpath < bytea_increment($1::bytea)
+	//         order by fullpath limit 1;
+	// where $1 = largestSingleDirectory
+	opts.expectLastKey = storage.Key("Peronosporales/hateless/tod/unricht/sniveling/Puyallup/Aaronite")
+
+	// i didn't plan it this way, but expectedCount happens to have some nicely-sized factors for
+	// our purposes with no messy remainder. 74 * 251 = 18574
+	opts.doIterations = 74
+	opts.batchSize = 251
+
+	benchAndVerifyIteration(b, store, opts)
+}
+
+func shallowNonRecursive(b *testing.B, store storage.KeyValueStore) {
+	opts := &verifyOpts{
+		iterateOpts: storage.IterateOptions{
+			Prefix:  storage.Key(largestSingleDirectory),
+			Recurse: false,
+		},
+		doIterations: 2,
+		batchSize:    10000,
+	}
+
+	// verify with:
+	//     select count(*) from list_directory(''::bytea, $1::bytea) ld(fp, md);
+	// where $1 is largestSingleDirectory
+	opts.expectCount = 18574
+
+	// verify with:
+	//     select encode(fp, 'escape') from (
+	//         select * from list_directory(''::bytea, $1::bytea) ld(fp, md)
+	//     ) x order by fp desc limit 1;
+	// where $1 = largestSingleDirectory
+	opts.expectLastKey = storage.Key("Peronosporales/hateless/tod/unricht/sniveling/Puyallup/élite")
+
+	benchAndVerifyIteration(b, store, opts)
+}
+
+func shallowNonRecursiveReverse(b *testing.B, store storage.KeyValueStore) {
+	opts := &verifyOpts{
+		iterateOpts: storage.IterateOptions{
+			Prefix:  storage.Key(largestSingleDirectory),
+			Recurse: false,
+			Reverse: true,
+		},
+		doIterations: 2,
+		batchSize:    10000,
+	}
+
+	// verify with:
+	//     select count(*) from list_directory(''::bytea, $1::bytea) ld(fp, md);
+	// where $1 is largestSingleDirectory
+	opts.expectCount = 18574
+
+	// verify with:
+	//     select encode(fp, 'escape') from (
+	//         select * from list_directory(''::bytea, $1::bytea) ld(fp, md)
+	//     ) x order by fp limit 1;
+	// where $1 = largestSingleDirectory
+	opts.expectLastKey = storage.Key("Peronosporales/hateless/tod/unricht/sniveling/Puyallup/Aaronite")
+
+	benchAndVerifyIteration(b, store, opts)
+}
+
+func topRecursiveLimit(b *testing.B, store storage.KeyValueStore) {
+	opts := &verifyOpts{
+		iterateOpts: storage.IterateOptions{
+			Recurse: true,
+		},
+		doIterations: 100,
+		batchSize:    10000,
+	}
+
+	// not expected to exhaust items
+	opts.expectCount = opts.doIterations * opts.batchSize
+
+	// verify with:
+	//     select encode(fullpath, 'escape') from (
+	//         select rank() over (order by fullpath), fullpath from pathdata
+	//     ) x where rank = $1;
+	// where $1 = expectCount
+	opts.expectLastKey = storage.Key("nonresuscitation/synchronically/bechern/hemangiomatosis")
+
+	benchAndVerifyIteration(b, store, opts)
+}
+
+func topRecursiveLimitReverse(b *testing.B, store storage.KeyValueStore) {
+	opts := &verifyOpts{
+		iterateOpts: storage.IterateOptions{
+			Recurse: true,
+			Reverse: true,
+		},
+		doIterations: 100,
+		batchSize:    10000,
+	}
+
+	// not expected to exhaust items
+	opts.expectCount = opts.doIterations * opts.batchSize
+
+	// verify with:
+	//     select encode(fullpath, 'escape') from (
+	//         select rank() over (order by fullpath desc), fullpath from pathdata
+	//     ) x where rank = $1;
+	// where $1 = expectCount
+	opts.expectLastKey = storage.Key("nonresuscitation/synchronically/cabook/homeozoic/inclinatorium/iguanodont/thiophenol/congeliturbation/Alaric")
+
+	benchAndVerifyIteration(b, store, opts)
+}
+
+func topRecursiveStartAt(b *testing.B, store storage.KeyValueStore) {
+	opts := &verifyOpts{
+		iterateOpts: storage.IterateOptions{
+			Recurse: true,
+		},
+		doIterations: 100,
+		batchSize:    10000,
+	}
+
+	// this is pretty arbitrary. just the key 100 positions before the end of the Peronosporales/hateless/ dir.
+	opts.iterateOpts.First = storage.Key("Peronosporales/hateless/warrener/anthropomancy/geisotherm/wickerwork")
+
+	// not expected to exhaust items
+	opts.expectCount = opts.doIterations * opts.batchSize
+
+	// verify with:
+	//     select encode(fullpath, 'escape') from (
+	//         select fullpath from pathdata where fullpath >= $1::bytea order by fullpath limit $2
+	//     ) x order by fullpath desc limit 1;
+	// where $1 = iterateOpts.First and $2 = expectCount
+	opts.expectLastKey = storage.Key("raptured/heathbird/histrionism/vermifugous/barefaced/beechdrops/lamber/phlegmatic/blended/Gershon/scallop/burglarproof/incompensated/allanite/alehouse/embroilment/lienotoxin/monotonically/cumbersomeness")
+
+	benchAndVerifyIteration(b, store, opts)
+}
+
+func topRecursiveStartAtReverse(b *testing.B, store storage.KeyValueStore) {
+	opts := &verifyOpts{
+		iterateOpts: storage.IterateOptions{
+			Recurse: true,
+			Reverse: true,
+		},
+		doIterations: 61,
+		batchSize:    10000,
+	}
+
+	// this is pretty arbitrary. just the key 100 positions before the end of the Peronosporales/hateless/ dir.
+	opts.iterateOpts.First = storage.Key("Peronosporales/hateless/warrener/anthropomancy/geisotherm/wickerwork")
+
+	// we *do* expect to exhaust the available items this time.
+	// verify with:
+	//     select count(*) from (
+	//         select fullpath from pathdata where fullpath <= $1::bytea order by fullpath desc limit $2
+	//     ) x;
+	// where $1 = iterateOpts.First and $2 = (doIterations * batchSize)
+	opts.expectCount = 608405
+
+	// since expectCount < (doIterations * batchSize), and we're going in reverse, the last key read
+	// should be the first one lexicographically.
+	// verify with:
+	//     select encode(fullpath, 'escape') from pathdata order by fullpath limit 1;
+	opts.expectLastKey = storage.Key("Lissamphibia")
+
+	benchAndVerifyIteration(b, store, opts)
+}
+
+func topNonRecursive(b *testing.B, store storage.KeyValueStore) {
+	opts := &verifyOpts{
+		iterateOpts: storage.IterateOptions{
+			Recurse: false,
+		},
+		doIterations: 1,
+		batchSize:    10000,
+	}
+
+	// verify with:
+	//     select count(*) from list_directory(''::bytea, ''::bytea);
+	opts.expectCount = 21
+
+	// verify with:
+	//     select encode(fp, 'escape') from (
+	//         select * from list_directory(''::bytea, ''::bytea) ld(fp, md)
+	//     ) x order by fp desc limit 1;
+	opts.expectLastKey = storage.Key("vejoces")
+
+	benchAndVerifyIteration(b, store, opts)
+}
+
+func topNonRecursiveReverse(b *testing.B, store storage.KeyValueStore) {
+	opts := &verifyOpts{
+		iterateOpts: storage.IterateOptions{
+			Recurse: false,
+			Reverse: true,
+		},
+		doIterations: 1,
+		batchSize:    10000,
+	}
+
+	// verify with:
+	//     select count(*) from list_directory(''::bytea, ''::bytea);
+	opts.expectCount = 21
+
+	// verify with:
+	//     select encode(fullpath, 'escape') from pathdata order by fullpath limit 1;
+	opts.expectLastKey = storage.Key("Lissamphibia")
+
+	benchAndVerifyIteration(b, store, opts)
+}
+
+func cleanupBigPathset(tb testing.TB, store storage.KeyValueStore) {
+	if *noCleanDb {
+		tb.Skip("Instructed not to clean up this KeyValueStore after long benchmarks are complete.")
+	}
+
+	cleaner, ok := store.(BulkCleaner)
+	if ok {
+		tb.Log("Performing bulk cleanup...")
+		err := cleaner.BulkDelete()
+
+		if err != nil {
+			tb.Fatalf("Provided KeyValueStore failed to perform bulk delete: %v", err)
+		}
+	} else {
+		inputIter := openTestData(tb)
+		defer func() {
+			if err := inputIter.closeFunc(); err != nil {
+				tb.Logf("Failed to close input data stream: %v", err)
+			}
+		}()
+
+		tb.Log("Performing manual cleanup...")
+
+		var item storage.ListItem
+		for inputIter.Next(&item) {
+			if err := store.Delete(item.Key); err != nil {
+				tb.Fatalf("Provided KeyValueStore failed to delete item %q during cleanup: %v", item.Key, err)
+			}
+		}
+		if err := inputIter.Error(); err != nil {
+			tb.Fatalf("Failed to iterate over input data: %v", err)
+		}
+	}
+}

storage/testsuite/utils.go

@@ -35,26 +35,44 @@ type iterationTest struct {
 func testIterations(t *testing.T, store storage.KeyValueStore, tests []iterationTest) {
 	t.Helper()
 	for _, test := range tests {
-		collect := &collector{}
+		items, err := iterateItems(store, test.Options, -1)
-		err := store.Iterate(test.Options, collect.include)
 		if err != nil {
 			t.Errorf("%s: %v", test.Name, err)
 			continue
 		}
-		if diff := cmp.Diff(test.Expected, collect.Items, cmpopts.EquateEmpty()); diff != "" {
+		if diff := cmp.Diff(test.Expected, items, cmpopts.EquateEmpty()); diff != "" {
 			t.Errorf("%s: (-want +got)\n%s", test.Name, diff)
 		}
 	}
 }
 
+func isEmptyKVStore(tb testing.TB, store storage.KeyValueStore) bool {
+	tb.Helper()
+	keys, err := store.List(storage.Key(""), 1)
+	if err != nil {
+		tb.Fatalf("Failed to check if KeyValueStore is empty: %v", err)
+	}
+	return len(keys) == 0
+}
+
 type collector struct {
 	Items storage.Items
+	Limit int
 }
 
 func (collect *collector) include(it storage.Iterator) error {
 	var item storage.ListItem
-	for it.Next(&item) {
+	for (collect.Limit < 0 || len(collect.Items) < collect.Limit) && it.Next(&item) {
 		collect.Items = append(collect.Items, storage.CloneItem(item))
 	}
 	return nil
 }
+
+func iterateItems(store storage.KeyValueStore, opts storage.IterateOptions, limit int) (storage.Items, error) {
+	collect := &collector{Limit: limit}
+	err := store.Iterate(opts, collect.include)
+	if err != nil {
+		return nil, err
+	}
+	return collect.Items, nil
+}