diff --git a/satellite/audit/verifier.go b/satellite/audit/verifier.go index fe1cd4c21..3b6d887e3 100644 --- a/satellite/audit/verifier.go +++ b/satellite/audit/verifier.go @@ -227,7 +227,8 @@ func (verifier *Verifier) Verify(ctx context.Context, segment Segment, skip map[ Offlines: offlineNodes, Unknown: unknownNodes, } - return report, ErrNotEnoughShares.New("got %d, required %d", len(sharesToAudit), required) + return report, ErrNotEnoughShares.New("got: %d, required: %d, failed: %d, offline: %d, unknown: %d, contained: %d", + len(sharesToAudit), required, len(failedNodes), len(offlineNodes), len(unknownNodes), len(containedNodes)) } // ensure we get values, even if only zero values, so that redash can have an alert based on this mon.Counter("not_enough_shares_for_audit").Inc(0) diff --git a/satellite/repair/repairer/ec.go b/satellite/repair/repairer/ec.go index aacb2ef30..70299f2eb 100644 --- a/satellite/repair/repairer/ec.go +++ b/satellite/repair/repairer/ec.go @@ -7,6 +7,7 @@ import ( "bytes" "context" "errors" + "fmt" "io" "io/ioutil" "sort" @@ -83,6 +84,7 @@ func (ec *ECRepairer) Get(ctx context.Context, limits []*pb.AddressedOrderLimit, limiter := sync2.NewLimiter(es.RequiredCount()) cond := sync.NewCond(&sync.Mutex{}) + errChan := make(chan error, len(limits)) for currentLimitIndex, limit := range limits { if limit == nil { @@ -145,6 +147,7 @@ func (ec *ECRepairer) Get(ctx context.Context, limits []*pb.AddressedOrderLimit, ec.log.Debug("Failed to download pieces for repair", zap.Error(err)) } + errChan <- fmt.Errorf("node id: %s, error: %w", limit.GetLimit().StorageNodeId.String(), err) return } @@ -157,12 +160,20 @@ func (ec *ECRepairer) Get(ctx context.Context, limits []*pb.AddressedOrderLimit, } limiter.Wait() + close(errChan) + var errlist errs.Group if successfulPieces < es.RequiredCount() { mon.Meter("download_failed_not_enough_pieces_repair").Mark(1) //mon:locked + for err := range errChan { + if err != nil { + errlist.Add(err) + } + } return nil, failedPieces, &irreparableError{ piecesAvailable: int32(successfulPieces), piecesRequired: int32(es.RequiredCount()), + errlist: errlist, } } diff --git a/satellite/repair/repairer/segments.go b/satellite/repair/repairer/segments.go index 42c440801..462daef1d 100644 --- a/satellite/repair/repairer/segments.go +++ b/satellite/repair/repairer/segments.go @@ -42,6 +42,7 @@ var ( type irreparableError struct { piecesAvailable int32 piecesRequired int32 + errlist []error } func (ie *irreparableError) Error() string { @@ -303,6 +304,7 @@ func (repairer *SegmentRepairer) Repair(ctx context.Context, queueSegment *queue zap.Uint64("Position", queueSegment.Position.Encode()), zap.Int32("piecesAvailable", irreparableErr.piecesAvailable), zap.Int32("piecesRequired", irreparableErr.piecesRequired), + zap.Error(errs.Combine(irreparableErr.errlist...)), ) return false, nil }