satellite/metainfo: added rate limiting support to the metainfo loop.

As per discussed we decided to rate limit how fast we iterate through
the metainfo database in the metainfo loop. This puts in place a
mechanism for rate limiting and burst limiting if need be in the future.

The default for this rate limiting is still no limits so it stays the
same as our previous functionality.

Change-Id: I950f7192962b0e49f082d2c4284e2d52b0a925c7

cmd/segment-reaper/observer.go

@@ -22,6 +22,7 @@ import (
 const (
 	maxNumOfSegments = 64
 	lastSegment      = int(-1)
+	rateLimit        = 0
 )
 
 // object represents object with segments.
@@ -188,7 +189,7 @@ func (obsvr *observer) processSegment(ctx context.Context, path metainfo.ScopedP
 }
 
 func (obsvr *observer) detectZombieSegments(ctx context.Context) error {
-	err := metainfo.IterateDatabase(ctx, obsvr.db, obsvr)
+	err := metainfo.IterateDatabase(ctx, rateLimit, obsvr.db, obsvr)
 	if err != nil {
 		return err
 	}

go.mod

@@ -103,6 +103,7 @@ require (
 	golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
 	golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3
+	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
 	google.golang.org/appengine v1.6.0 // indirect
 	google.golang.org/grpc v1.23.1
 	gopkg.in/Shopify/sarama.v1 v1.18.0 // indirect

satellite/metainfo/loop.go

@@ -11,6 +11,8 @@ import (
 	"github.com/skyrings/skyring-common/tools/uuid"
 	"github.com/zeebo/errs"
 
+	"golang.org/x/time/rate"
+
 	"storj.io/common/pb"
 	"storj.io/common/storj"
 	"storj.io/storj/storage"
@@ -72,6 +74,7 @@ func (observer *observerContext) Wait() error {
 // LoopConfig contains configurable values for the metainfo loop.
 type LoopConfig struct {
 	CoalesceDuration time.Duration `help:"how long to wait for new observers before starting iteration" releaseDefault:"5s" devDefault:"5s"`
+	RateLimit        float64       `help:"metainfo loop rate limit (default is 0 which is unlimited segments per second)" default:"0"`
 }
 
 // Loop is a metainfo loop service.
@@ -165,12 +168,11 @@ waitformore:
 			return ctx.Err()
 		}
 	}
-
+	return iterateDatabase(ctx, loop.db, observers, rate.NewLimiter(rate.Limit(loop.config.RateLimit), 1))
-	return iterateDatabase(ctx, loop.db, observers)
 }
 
 // IterateDatabase iterates over PointerDB and notifies specified observers about results.
-func IterateDatabase(ctx context.Context, db PointerDB, observers ...Observer) error {
+func IterateDatabase(ctx context.Context, rateLimit float64, db PointerDB, observers ...Observer) error {
 	obsContexts := make([]*observerContext, len(observers))
 	for i, observer := range observers {
 		obsContexts[i] = &observerContext{
@@ -179,7 +181,7 @@ func IterateDatabase(ctx context.Context, db PointerDB, observers ...Observer) e
 			done:     make(chan error),
 		}
 	}
-	return iterateDatabase(ctx, db, obsContexts)
+	return iterateDatabase(ctx, db, obsContexts, rate.NewLimiter(rate.Limit(rateLimit), 1))
 }
 
 // handlePointer deals with a pointer for a single observer
@@ -219,7 +221,7 @@ func (loop *Loop) Wait() {
 	<-loop.done
 }
 
-func iterateDatabase(ctx context.Context, db PointerDB, observers []*observerContext) (err error) {
+func iterateDatabase(ctx context.Context, db PointerDB, observers []*observerContext, rateLimiter *rate.Limiter) (err error) {
 	defer func() {
 		if err != nil {
 			for _, observer := range observers {
@@ -237,6 +239,12 @@ func iterateDatabase(ctx context.Context, db PointerDB, observers []*observerCon
 			// iterate over every segment in metainfo
 		nextSegment:
 			for it.Next(ctx, &item) {
+				if err := rateLimiter.Wait(ctx); err != nil {
+					// We don't really execute concurrent batches so we should never
+					// exceed the burst size of 1 and this should never happen.
+					return LoopError.New("unexpected error rate limiting metainfo loop %s", err)
+				}
+
 				rawPath := item.Key.String()
 				pointer := &pb.Pointer{}
 

scripts/testdata/satellite-config.yaml.lock

@@ -217,6 +217,9 @@ identity.key-path: /root/.local/share/storj/identity/satellite/identity.key
 # how long to wait for new observers before starting iteration
 # metainfo.loop.coalesce-duration: 5s
 
+# metainfo loop rate limit (default is 0 which is unlimited segments per second)
+# metainfo.loop.rate-limit: 0
+
 # maximum time allowed to pass between creating and committing a segment
 # metainfo.max-commit-interval: 48h0m0s