storj/satellite/gracefulexit/chore.go

// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.

package gracefulexit

import (
	"context"
	"database/sql"
	"time"

	"github.com/zeebo/errs"
	"go.uber.org/zap"

	"storj.io/common/storj"
	"storj.io/common/sync2"
	"storj.io/storj/satellite/metabase/segmentloop"
	"storj.io/storj/satellite/overlay"
)

// Chore populates the graceful exit transfer queue.
//
// architecture: Chore
type Chore struct {
	log         *zap.Logger
	Loop        *sync2.Cycle
	db          DB
	config      Config
	overlay     overlay.DB
	segmentLoop *segmentloop.Service
}

// NewChore instantiates Chore.
func NewChore(log *zap.Logger, db DB, overlay overlay.DB, segmentLoop *segmentloop.Service, config Config) *Chore {
	return &Chore{
		log:         log,
		Loop:        sync2.NewCycle(config.ChoreInterval),
		db:          db,
		config:      config,
		overlay:     overlay,
		segmentLoop: segmentLoop,
	}
}

// Run starts the chore.
func (chore *Chore) Run(ctx context.Context) (err error) {
	defer mon.Task()(&ctx)(&err)
	return chore.Loop.Run(ctx, func(ctx context.Context) (err error) {
		defer mon.Task()(&ctx)(&err)

		exitingNodes, err := chore.overlay.GetExitingNodes(ctx)
		if err != nil {
			chore.log.Error("error retrieving nodes that have not finished exiting", zap.Error(err))
			return nil
		}

		nodeCount := len(exitingNodes)
		if nodeCount == 0 {
			return nil
		}
		chore.log.Debug("found exiting nodes", zap.Int("exitingNodes", nodeCount))

		exitingNodesLoopIncomplete := make(storj.NodeIDList, 0, nodeCount)
		for _, node := range exitingNodes {
			if node.ExitLoopCompletedAt == nil {
				exitingNodesLoopIncomplete = append(exitingNodesLoopIncomplete, node.NodeID)
				continue
			}

			progress, err := chore.db.GetProgress(ctx, node.NodeID)
			if err != nil && !errs.Is(err, sql.ErrNoRows) {
				chore.log.Error("error retrieving progress for node", zap.Stringer("Node ID", node.NodeID), zap.Error(err))
				continue
			}

			lastActivityTime := *node.ExitLoopCompletedAt
			if progress != nil {
				lastActivityTime = progress.UpdatedAt
			}

			// check inactive timeframe
			if lastActivityTime.Add(chore.config.MaxInactiveTimeFrame).Before(time.Now().UTC()) {
				exitStatusRequest := &overlay.ExitStatusRequest{
					NodeID:         node.NodeID,
					ExitSuccess:    false,
					ExitFinishedAt: time.Now().UTC(),
				}
				mon.Meter("graceful_exit_fail_inactive").Mark(1)
				_, err = chore.overlay.UpdateExitStatus(ctx, exitStatusRequest)
				if err != nil {
					chore.log.Error("error updating exit status", zap.Error(err))
					continue
				}

				// remove all items from the transfer queue
				err := chore.db.DeleteTransferQueueItems(ctx, node.NodeID)
				if err != nil {
					chore.log.Error("error deleting node from transfer queue", zap.Error(err))
				}
			}
		}

		// Populate transfer queue for nodes that have not completed the exit loop yet
		pathCollector := NewPathCollector(chore.db, exitingNodesLoopIncomplete, chore.log, chore.config.ChoreBatchSize)
		err = chore.segmentLoop.Join(ctx, pathCollector)
		if err != nil {
			chore.log.Error("error joining segment loop.", zap.Error(err))
			return nil
		}

		err = pathCollector.Flush(ctx)
		if err != nil {
			chore.log.Error("error flushing collector buffer.", zap.Error(err))
			return nil
		}

		now := time.Now().UTC()
		for _, nodeID := range exitingNodesLoopIncomplete {
			exitStatus := overlay.ExitStatusRequest{
				NodeID:              nodeID,
				ExitLoopCompletedAt: now,
			}
			_, err = chore.overlay.UpdateExitStatus(ctx, &exitStatus)
			if err != nil {
				chore.log.Error("error updating exit status.", zap.Error(err))
			}

			bytesToTransfer := pathCollector.nodeIDStorage[nodeID]
			mon.IntVal("graceful_exit_init_bytes_stored").Observe(bytesToTransfer)
		}
		return nil
	})
}

// Close closes chore.
func (chore *Chore) Close() error {
	chore.Loop.Close()
	return nil
}