storagenode/{monitor,piecestore}: if free disk < expected available space, return free disk

We only sync free disk and available space, if necessary, on startup. If the
SNs disk fills up with non-storj data, we will not know about it when reporting
available space to the satellite.

Solution: whenever we check the node's capacity, double check free disk.
If free disk < than expected available space, return free disk.

Change-Id: I66265c16e03be45b6e1f5817c70df7eac0a76455

storagenode/monitor/monitor.go

@@ -68,6 +68,7 @@ func (service *Service) Run(ctx context.Context) (err error) {
 	defer mon.Task()(&ctx)(&err)
 
 	// get the disk space details
+
 	// The returned path ends in a slash only if it represents a root directory, such as "/" on Unix or `C:\` on Windows.
 	storageStatus, err := service.store.StorageStatus(ctx)
 	if err != nil {
@@ -148,13 +149,12 @@ func (service *Service) Close() (err error) {
 func (service *Service) updateNodeInformation(ctx context.Context) (err error) {
 	defer mon.Task()(&ctx)(&err)
 
-	usedSpace, err := service.usedSpace(ctx)
+	freeSpace, err := service.AvailableSpace(ctx)
 	if err != nil {
-		return Error.Wrap(err)
+		return err
 	}
-
 	service.contact.UpdateSelf(&pb.NodeCapacity{
-		FreeDisk: service.allocatedDiskSpace - usedSpace,
+		FreeDisk: freeSpace,
 	})
 
 	return nil
@@ -176,11 +176,21 @@ func (service *Service) AvailableSpace(ctx context.Context) (_ int64, err error)
 	if err != nil {
 		return 0, Error.Wrap(err)
 	}
-	allocatedSpace := service.allocatedDiskSpace
 
-	mon.IntVal("allocated_space").Observe(allocatedSpace)
+	freeSpaceForStorj := service.allocatedDiskSpace - usedSpace
+
+	diskStatus, err := service.store.StorageStatus(ctx)
+	if err != nil {
+		return 0, Error.Wrap(err)
+	}
+
+	if diskStatus.DiskFree < freeSpaceForStorj {
+		freeSpaceForStorj = diskStatus.DiskFree
+	}
+
+	mon.IntVal("allocated_space").Observe(service.allocatedDiskSpace)
 	mon.IntVal("used_space").Observe(usedSpace)
-	mon.IntVal("available_space").Observe(allocatedSpace - usedSpace)
+	mon.IntVal("available_space").Observe(freeSpaceForStorj)
 
-	return allocatedSpace - usedSpace, nil
+	return freeSpaceForStorj, nil
 }

storagenode/piecestore/endpoint.go

@@ -238,13 +238,8 @@ func (endpoint *Endpoint) Upload(stream pb.DRPCPiecestore_UploadStream) (err err
 		}
 	}()
 
-	// double verify that disk actually has sufficient capacity
-	status, err := endpoint.store.StorageStatus(ctx)
-	if err != nil {
-		return err
-	}
-	if status.DiskFree < limit.Limit {
-		return rpcstatus.Errorf(rpcstatus.Aborted, "not enough available disk space, have: %v, need: %v", status.DiskFree, limit.Limit)
+	if availableSpace < limit.Limit {
+		return rpcstatus.Errorf(rpcstatus.Aborted, "not enough available disk space, have: %v, need: %v", availableSpace, limit.Limit)
 	}
 
 	var pieceWriter *pieces.Writer

storagenode/piecestore/verification_test.go

@@ -107,7 +107,7 @@ func TestOrderLimitPutValidation(t *testing.T) {
 			orderExpiration: oneWeek,
 			limit:           10 * memory.KiB.Int64(),
 			availableSpace:  5 * memory.KiB.Int64(),
-			err:             "out of space",
+			err:             "not enough available disk space",
 		},
 	} {
 		tt := tt