2019-01-24 20:15:10 +00:00
|
|
|
// Copyright (C) 2019 Storj Labs, Inc.
|
2018-10-02 20:46:29 +01:00
|
|
|
// See LICENSE for copying information.
|
|
|
|
|
|
|
|
package checker
|
2018-10-03 19:35:56 +01:00
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2018-10-30 19:16:40 +00:00
|
|
|
"time"
|
2018-10-04 22:40:34 +01:00
|
|
|
|
2018-11-20 18:29:07 +00:00
|
|
|
"github.com/gogo/protobuf/proto"
|
2019-01-23 19:58:44 +00:00
|
|
|
"github.com/zeebo/errs"
|
2018-10-04 22:40:34 +01:00
|
|
|
"go.uber.org/zap"
|
2019-05-31 15:12:49 +01:00
|
|
|
"golang.org/x/sync/errgroup"
|
2019-01-23 19:58:44 +00:00
|
|
|
monkit "gopkg.in/spacemonkeygo/monkit.v2"
|
2018-10-04 22:40:34 +01:00
|
|
|
|
2019-02-11 21:06:39 +00:00
|
|
|
"storj.io/storj/internal/sync2"
|
2018-12-10 19:08:45 +00:00
|
|
|
"storj.io/storj/pkg/datarepair/irreparable"
|
2018-10-09 17:09:33 +01:00
|
|
|
"storj.io/storj/pkg/datarepair/queue"
|
2019-03-23 08:06:11 +00:00
|
|
|
"storj.io/storj/pkg/overlay"
|
2018-10-09 17:09:33 +01:00
|
|
|
"storj.io/storj/pkg/pb"
|
2018-11-29 18:39:27 +00:00
|
|
|
"storj.io/storj/pkg/storj"
|
2019-04-25 09:46:32 +01:00
|
|
|
"storj.io/storj/satellite/metainfo"
|
2018-10-09 17:09:33 +01:00
|
|
|
"storj.io/storj/storage"
|
2018-10-03 19:35:56 +01:00
|
|
|
)
|
|
|
|
|
2019-01-23 19:58:44 +00:00
|
|
|
// Error is a standard error class for this package.
|
|
|
|
var (
|
|
|
|
Error = errs.Class("checker error")
|
|
|
|
mon = monkit.Package()
|
|
|
|
)
|
|
|
|
|
|
|
|
// Config contains configurable values for checker
|
|
|
|
type Config struct {
|
2019-06-04 13:13:31 +01:00
|
|
|
Interval time.Duration `help:"how frequently checker should check for bad segments" releaseDefault:"30s" devDefault:"0h0m10s"`
|
2019-06-15 17:19:19 +01:00
|
|
|
IrreparableInterval time.Duration `help:"how frequently irrepairable checker should check for lost pieces" releaseDefault:"30m" devDefault:"0h0m5s"`
|
2019-07-08 23:04:35 +01:00
|
|
|
|
|
|
|
ReliabilityCacheStaleness time.Duration `help:"how stale reliable node cache can be" releaseDefault:"5m" devDefault:"5m"`
|
2019-05-30 16:18:20 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// durabilityStats remote segment information
|
|
|
|
type durabilityStats struct {
|
|
|
|
remoteFilesChecked int64
|
|
|
|
remoteSegmentsChecked int64
|
|
|
|
remoteSegmentsNeedingRepair int64
|
|
|
|
remoteSegmentsLost int64
|
|
|
|
remoteSegmentInfo []string
|
2019-01-23 19:58:44 +00:00
|
|
|
}
|
|
|
|
|
2018-10-09 17:09:33 +01:00
|
|
|
// Checker contains the information needed to do checks for missing pieces
|
2019-02-11 21:06:39 +00:00
|
|
|
type Checker struct {
|
2019-05-22 22:17:52 +01:00
|
|
|
metainfo *metainfo.Service
|
|
|
|
lastChecked string
|
|
|
|
repairQueue queue.RepairQueue
|
2019-07-08 23:04:35 +01:00
|
|
|
nodestate *ReliabilityCache
|
2019-05-22 22:17:52 +01:00
|
|
|
irrdb irreparable.DB
|
|
|
|
logger *zap.Logger
|
|
|
|
Loop sync2.Cycle
|
2019-05-30 16:18:20 +01:00
|
|
|
IrreparableLoop sync2.Cycle
|
2019-05-31 15:12:49 +01:00
|
|
|
monStats durabilityStats
|
2018-10-09 17:09:33 +01:00
|
|
|
}
|
|
|
|
|
2019-01-18 13:54:08 +00:00
|
|
|
// NewChecker creates a new instance of checker
|
2019-07-08 23:04:35 +01:00
|
|
|
func NewChecker(metainfo *metainfo.Service, repairQueue queue.RepairQueue, overlay *overlay.Cache, irrdb irreparable.DB, limit int, logger *zap.Logger, config Config) *Checker {
|
2019-01-18 13:54:08 +00:00
|
|
|
// TODO: reorder arguments
|
2019-07-08 23:04:35 +01:00
|
|
|
return &Checker{
|
2019-05-30 16:18:20 +01:00
|
|
|
metainfo: metainfo,
|
|
|
|
lastChecked: "",
|
|
|
|
repairQueue: repairQueue,
|
2019-07-08 23:04:35 +01:00
|
|
|
nodestate: NewReliabilityCache(overlay, config.ReliabilityCacheStaleness),
|
2019-05-30 16:18:20 +01:00
|
|
|
irrdb: irrdb,
|
|
|
|
logger: logger,
|
2019-07-08 23:04:35 +01:00
|
|
|
Loop: *sync2.NewCycle(config.Interval),
|
|
|
|
IrreparableLoop: *sync2.NewCycle(config.IrreparableInterval),
|
2019-05-31 15:12:49 +01:00
|
|
|
monStats: durabilityStats{},
|
2018-10-09 17:09:33 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-11-01 14:03:45 +00:00
|
|
|
// Run the checker loop
|
2019-02-11 21:06:39 +00:00
|
|
|
func (checker *Checker) Run(ctx context.Context) (err error) {
|
2018-11-01 14:03:45 +00:00
|
|
|
defer mon.Task()(&ctx)(&err)
|
|
|
|
|
2019-05-31 15:12:49 +01:00
|
|
|
group, ctx := errgroup.WithContext(ctx)
|
2019-05-30 16:18:20 +01:00
|
|
|
|
2019-05-31 15:12:49 +01:00
|
|
|
group.Go(func() error {
|
|
|
|
return checker.Loop.Run(ctx, checker.IdentifyInjuredSegments)
|
|
|
|
})
|
2019-05-30 16:18:20 +01:00
|
|
|
|
2019-05-31 15:12:49 +01:00
|
|
|
group.Go(func() error {
|
|
|
|
return checker.IrreparableLoop.Run(ctx, checker.IrreparableProcess)
|
|
|
|
})
|
2019-05-30 16:18:20 +01:00
|
|
|
|
2019-05-31 15:12:49 +01:00
|
|
|
return group.Wait()
|
2018-11-01 14:03:45 +00:00
|
|
|
}
|
|
|
|
|
2019-07-08 23:04:35 +01:00
|
|
|
// RefreshReliabilityCache forces refreshing node online status cache.
|
|
|
|
func (checker *Checker) RefreshReliabilityCache(ctx context.Context) error {
|
|
|
|
return checker.nodestate.Refresh(ctx)
|
|
|
|
}
|
|
|
|
|
2019-02-14 12:33:41 +00:00
|
|
|
// Close halts the Checker loop
|
|
|
|
func (checker *Checker) Close() error {
|
|
|
|
checker.Loop.Close()
|
|
|
|
return nil
|
|
|
|
}
|
2019-01-18 13:54:08 +00:00
|
|
|
|
2019-04-25 09:46:32 +01:00
|
|
|
// IdentifyInjuredSegments checks for missing pieces off of the metainfo and overlay cache
|
2019-02-11 21:06:39 +00:00
|
|
|
func (checker *Checker) IdentifyInjuredSegments(ctx context.Context) (err error) {
|
2018-10-09 17:09:33 +01:00
|
|
|
defer mon.Task()(&ctx)(&err)
|
|
|
|
|
2019-06-05 15:23:10 +01:00
|
|
|
err = checker.metainfo.Iterate(ctx, "", checker.lastChecked, true, false,
|
|
|
|
func(ctx context.Context, it storage.Iterator) error {
|
2018-10-09 17:09:33 +01:00
|
|
|
var item storage.ListItem
|
2019-05-08 18:59:50 +01:00
|
|
|
|
|
|
|
defer func() {
|
|
|
|
var nextItem storage.ListItem
|
2019-06-05 15:23:10 +01:00
|
|
|
it.Next(ctx, &nextItem)
|
2019-05-08 18:59:50 +01:00
|
|
|
// start at the next item in the next call
|
|
|
|
checker.lastChecked = nextItem.Key.String()
|
2019-05-23 17:14:08 +01:00
|
|
|
// if we have finished iterating, send and reset durability stats
|
|
|
|
if checker.lastChecked == "" {
|
2019-05-22 22:17:52 +01:00
|
|
|
// send durability stats
|
2019-05-31 15:12:49 +01:00
|
|
|
mon.IntVal("remote_files_checked").Observe(checker.monStats.remoteFilesChecked)
|
|
|
|
mon.IntVal("remote_segments_checked").Observe(checker.monStats.remoteSegmentsChecked)
|
|
|
|
mon.IntVal("remote_segments_needing_repair").Observe(checker.monStats.remoteSegmentsNeedingRepair)
|
|
|
|
mon.IntVal("remote_segments_lost").Observe(checker.monStats.remoteSegmentsLost)
|
|
|
|
mon.IntVal("remote_files_lost").Observe(int64(len(checker.monStats.remoteSegmentInfo)))
|
|
|
|
|
|
|
|
// reset durability stats for next iteration
|
|
|
|
checker.monStats = durabilityStats{}
|
2019-05-08 18:59:50 +01:00
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
2019-06-05 15:23:10 +01:00
|
|
|
for it.Next(ctx, &item) {
|
2018-10-09 17:09:33 +01:00
|
|
|
pointer := &pb.Pointer{}
|
2018-12-12 15:39:16 +00:00
|
|
|
|
2018-10-09 17:09:33 +01:00
|
|
|
err = proto.Unmarshal(item.Value, pointer)
|
|
|
|
if err != nil {
|
|
|
|
return Error.New("error unmarshalling pointer %s", err)
|
|
|
|
}
|
2019-07-08 23:04:35 +01:00
|
|
|
remote := pointer.GetRemote()
|
|
|
|
if remote == nil {
|
|
|
|
continue
|
|
|
|
}
|
2018-12-12 15:39:16 +00:00
|
|
|
|
2019-05-31 15:12:49 +01:00
|
|
|
err = checker.updateSegmentStatus(ctx, pointer, item.Key.String(), &checker.monStats)
|
2018-12-06 18:51:23 +00:00
|
|
|
if err != nil {
|
2019-05-30 16:18:20 +01:00
|
|
|
return err
|
2018-10-09 17:09:33 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
},
|
|
|
|
)
|
2019-02-26 15:17:51 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
2018-10-09 17:09:33 +01:00
|
|
|
}
|
|
|
|
|
2019-02-26 15:17:51 +00:00
|
|
|
// checks for a string in slice
|
|
|
|
func contains(a []string, x string) bool {
|
|
|
|
for _, n := range a {
|
|
|
|
if x == n {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
2019-05-30 16:18:20 +01:00
|
|
|
|
|
|
|
func (checker *Checker) updateSegmentStatus(ctx context.Context, pointer *pb.Pointer, path string, monStats *durabilityStats) (err error) {
|
2019-06-04 12:36:27 +01:00
|
|
|
defer mon.Task()(&ctx)(&err)
|
2019-05-30 16:18:20 +01:00
|
|
|
remote := pointer.GetRemote()
|
|
|
|
if remote == nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
pieces := remote.GetRemotePieces()
|
|
|
|
if pieces == nil {
|
|
|
|
checker.logger.Debug("no pieces on remote segment")
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2019-07-08 23:16:50 +01:00
|
|
|
missingPieces, err := checker.nodestate.MissingPieces(ctx, pointer.CreationDate, pieces)
|
2019-05-30 16:18:20 +01:00
|
|
|
if err != nil {
|
|
|
|
return Error.New("error getting missing pieces %s", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
monStats.remoteSegmentsChecked++
|
|
|
|
pathElements := storj.SplitPath(path)
|
|
|
|
if len(pathElements) >= 2 && pathElements[1] == "l" {
|
|
|
|
monStats.remoteFilesChecked++
|
|
|
|
}
|
|
|
|
|
|
|
|
numHealthy := int32(len(pieces) - len(missingPieces))
|
|
|
|
redundancy := pointer.Remote.Redundancy
|
2019-07-01 17:13:18 +01:00
|
|
|
// we repair when the number of healthy pieces is less than or equal to the repair threshold
|
2019-05-30 16:18:20 +01:00
|
|
|
// except for the case when the repair and success thresholds are the same (a case usually seen during testing)
|
2019-06-19 21:13:11 +01:00
|
|
|
if numHealthy > redundancy.MinReq && numHealthy <= redundancy.RepairThreshold && numHealthy < redundancy.SuccessThreshold {
|
2019-05-30 16:18:20 +01:00
|
|
|
if len(missingPieces) == 0 {
|
|
|
|
checker.logger.Warn("Missing pieces is zero in checker, but this should be impossible -- bad redundancy scheme.")
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
monStats.remoteSegmentsNeedingRepair++
|
|
|
|
err = checker.repairQueue.Insert(ctx, &pb.InjuredSegment{
|
|
|
|
Path: path,
|
|
|
|
LostPieces: missingPieces,
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return Error.New("error adding injured segment to queue %s", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// delete always returns nil when something was deleted and also when element didn't exists
|
|
|
|
err = checker.irrdb.Delete(ctx, []byte(path))
|
|
|
|
if err != nil {
|
|
|
|
checker.logger.Error("error deleting entry from irreparable db: ", zap.Error(err))
|
|
|
|
}
|
2019-06-14 10:16:31 +01:00
|
|
|
// we need one additional piece for error correction. If only the minimum is remaining the file can't be repaired and is lost.
|
|
|
|
// except for the case when minimum and repair thresholds are the same (a case usually seen during testing)
|
2019-06-19 21:13:11 +01:00
|
|
|
} else if numHealthy <= redundancy.MinReq && numHealthy < redundancy.RepairThreshold {
|
2019-05-30 16:18:20 +01:00
|
|
|
// check to make sure there are at least *4* path elements. the first three
|
|
|
|
// are project, segment, and bucket name, but we want to make sure we're talking
|
|
|
|
// about an actual object, and that there's an object name specified
|
|
|
|
if len(pathElements) >= 4 {
|
|
|
|
project, bucketName, segmentpath := pathElements[0], pathElements[2], pathElements[3]
|
|
|
|
lostSegInfo := storj.JoinPaths(project, bucketName, segmentpath)
|
|
|
|
if contains(monStats.remoteSegmentInfo, lostSegInfo) == false {
|
|
|
|
monStats.remoteSegmentInfo = append(monStats.remoteSegmentInfo, lostSegInfo)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
monStats.remoteSegmentsLost++
|
|
|
|
// make an entry in to the irreparable table
|
|
|
|
segmentInfo := &pb.IrreparableSegment{
|
|
|
|
Path: []byte(path),
|
|
|
|
SegmentDetail: pointer,
|
|
|
|
LostPieces: int32(len(missingPieces)),
|
|
|
|
LastRepairAttempt: time.Now().Unix(),
|
|
|
|
RepairAttemptCount: int64(1),
|
|
|
|
}
|
|
|
|
|
|
|
|
// add the entry if new or update attempt count if already exists
|
|
|
|
err := checker.irrdb.IncrementRepairAttempts(ctx, segmentInfo)
|
|
|
|
if err != nil {
|
|
|
|
return Error.New("error handling irreparable segment to queue %s", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2019-07-05 21:20:58 +01:00
|
|
|
// IrreparableProcess picks items from irreparabledb and add them to the repair
|
2019-07-01 17:13:18 +01:00
|
|
|
// worker queue if they, now, can be repaired.
|
2019-05-30 16:18:20 +01:00
|
|
|
func (checker *Checker) IrreparableProcess(ctx context.Context) (err error) {
|
|
|
|
defer mon.Task()(&ctx)(&err)
|
|
|
|
|
|
|
|
limit := 1
|
|
|
|
var offset int64
|
|
|
|
|
|
|
|
for {
|
|
|
|
seg, err := checker.irrdb.GetLimited(ctx, limit, offset)
|
|
|
|
if err != nil {
|
|
|
|
return Error.New("error reading segment from the queue %s", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// zero segments returned with nil err
|
|
|
|
if len(seg) == 0 {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
2019-05-31 15:12:49 +01:00
|
|
|
err = checker.updateSegmentStatus(ctx, seg[0].GetSegmentDetail(), string(seg[0].GetPath()), &durabilityStats{})
|
2019-05-30 16:18:20 +01:00
|
|
|
if err != nil {
|
|
|
|
checker.logger.Error("irrepair segment checker failed: ", zap.Error(err))
|
|
|
|
}
|
|
|
|
offset++
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|