satellite/metainfo/piecedeletion: try to make batches larger

Currently it was possible that PopAll returns 1010 items, then
makes one RPC call with 1000 items, then RPC call 10 items. Meanwhile,
there have been added 500 new items added to the queue.

This change ensures that we pull items from the queue early and
try to make rpc batches as large as possible.

Change-Id: I1a30dde9164c2ff7b90c906a9544593c4f1cf0e9

satellite/metainfo/piecedeletion/combiner.go

@@ -25,10 +25,15 @@ type NewQueue func() Queue
 type Queue interface {
 	// TryPush tries to push a new job to the queue.
 	TryPush(job Job) bool
+
 	// PopAll fetches all jobs in the queue.
 	//
 	// When there are no more jobs, the queue must stop accepting new jobs.
 	PopAll() ([]Job, bool)
+
+	// PopAllWithoutClose fetches all jobs in the queue,
+	// but without closing the queue for new requests.
+	PopAllWithoutClose() []Job
 }
 
 // Job is a single of deletion.

satellite/metainfo/piecedeletion/dialer.go

@@ -99,6 +99,8 @@ func (dialer *Dialer) Handle(ctx context.Context, node *pb.Node, queue Queue) {
 				}
 				break
 			}
+
+			jobs = append(jobs, queue.PopAllWithoutClose()...)
 		}
 
 		// if we failed early, remaining jobs should be marked as failures

satellite/metainfo/piecedeletion/queue.go

@@ -30,13 +30,6 @@ func NewLimitedJobs(maxPiecesPerBatch int) *LimitedJobs {
 //
 // maxPiecesPerBatch < 0, means no limit
 func (jobs *LimitedJobs) TryPush(job Job) bool {
-	return jobs.tryPush(job, jobs.maxPiecesPerBatch)
-}
-
-// tryPush tries to add a job to the queue.
-//
-// maxPiecesPerBatch < 0, means no limit
-func (jobs *LimitedJobs) tryPush(job Job, maxPiecesPerBatch int) bool {
 	jobs.mu.Lock()
 	defer jobs.mu.Unlock()
 
@@ -50,7 +43,7 @@ func (jobs *LimitedJobs) tryPush(job Job, maxPiecesPerBatch int) bool {
 	jobs.count += len(job.Pieces)
 
 	// check whether the queue is at capacity
-	if maxPiecesPerBatch >= 0 && jobs.count >= maxPiecesPerBatch {
+	if jobs.maxPiecesPerBatch >= 0 && jobs.count >= jobs.maxPiecesPerBatch {
 		jobs.done = true
 	}
 
@@ -73,3 +66,13 @@ func (jobs *LimitedJobs) PopAll() (_ []Job, ok bool) {
 	jobs.list = nil
 	return list, true
 }
+
+// PopAllWithoutClose returns all the jobs in this list without closing the queue.
+func (jobs *LimitedJobs) PopAllWithoutClose() []Job {
+	jobs.mu.Lock()
+	defer jobs.mu.Unlock()
+
+	list := jobs.list
+	jobs.list = nil
+	return list
+}

satellite/metainfo/piecedeletion/queue_test.go

@@ -38,7 +38,7 @@ func TestLimitedJobs(t *testing.T) {
 
 		list, ok := q.PopAll()
 		require.True(t, ok)
-		require.Equal(t, list, []piecedeletion.Job{job1, job2})
+		require.Equal(t, []piecedeletion.Job{job1, job2}, list)
 
 		// should be empty
 		list, ok = q.PopAll()
@@ -71,6 +71,24 @@ func TestLimitedJobs_Limiting(t *testing.T) {
 	}
 }
 
+func TestLimitedJobs_NoClose(t *testing.T) {
+	{
+		q := piecedeletion.NewLimitedJobs(2)
+		job1, job2 := randomJob(1), randomJob(1)
+
+		require.True(t, q.TryPush(job1))
+		list := q.PopAllWithoutClose()
+		require.Equal(t, []piecedeletion.Job{job1}, list)
+
+		list = q.PopAllWithoutClose()
+		require.Empty(t, list)
+
+		require.True(t, q.TryPush(job2))
+		list = q.PopAllWithoutClose()
+		require.Equal(t, []piecedeletion.Job{job2}, list)
+	}
+}
+
 func TestLimitedJobs_Concurrent(t *testing.T) {
 	ctx := testcontext.New(t)
 	defer ctx.Cleanup()
@@ -145,6 +163,10 @@ func TestLimitedJobs_NoRace(t *testing.T) {
 		_, _ = q.PopAll()
 		return nil
 	})
+	ctx.Go(func() error {
+		_ = q.PopAllWithoutClose()
+		return nil
+	})
 }
 
 func randomJob(n int) piecedeletion.Job {