b0d98b1c1a
For being able to use the segment metainfo loop, graceful exit transfers have to include the segment stream_id/position instead of the path. For this, we created a new table graceful_exit_segment_transfer_queue that will replace the graceful_exit_transfer_queue. The table has been created in a previous migration. This change gives access to this table. Graceful Exit doesn't use the table yet, this will be done in a next change. Change-Id: I6c09cff4cc45f0529813a8898ddb2d14aadb2cb8
86 lines
5.1 KiB
Go
86 lines
5.1 KiB
Go
// Copyright (C) 2019 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package gracefulexit
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"storj.io/common/storj"
|
|
"storj.io/common/uuid"
|
|
"storj.io/storj/satellite/metabase"
|
|
)
|
|
|
|
// Progress represents the persisted graceful exit progress record.
|
|
type Progress struct {
|
|
NodeID storj.NodeID
|
|
BytesTransferred int64
|
|
PiecesTransferred int64
|
|
PiecesFailed int64
|
|
UpdatedAt time.Time
|
|
UsesSegmentTransferQueue bool
|
|
}
|
|
|
|
// TransferQueueItem represents the persisted graceful exit queue record.
|
|
type TransferQueueItem struct {
|
|
NodeID storj.NodeID
|
|
Key metabase.SegmentKey
|
|
StreamID uuid.UUID
|
|
Position metabase.SegmentPosition
|
|
PieceNum int32
|
|
RootPieceID storj.PieceID
|
|
DurabilityRatio float64
|
|
QueuedAt time.Time
|
|
RequestedAt *time.Time
|
|
LastFailedAt *time.Time
|
|
LastFailedCode *int
|
|
FailedCount *int
|
|
FinishedAt *time.Time
|
|
OrderLimitSendCount int
|
|
}
|
|
|
|
// DB implements CRUD operations for graceful exit service.
|
|
//
|
|
// architecture: Database
|
|
type DB interface {
|
|
// IncrementProgress increments transfer stats for a node.
|
|
IncrementProgress(ctx context.Context, nodeID storj.NodeID, bytes int64, successfulTransfers int64, failedTransfers int64) error
|
|
// GetProgress gets a graceful exit progress entry.
|
|
GetProgress(ctx context.Context, nodeID storj.NodeID) (*Progress, error)
|
|
|
|
// Enqueue batch inserts graceful exit transfer queue entries it does not exist.
|
|
Enqueue(ctx context.Context, items []TransferQueueItem, batchSize int, usesSegmentTransferQueue bool) error
|
|
// UpdateTransferQueueItem creates a graceful exit transfer queue entry.
|
|
UpdateTransferQueueItem(ctx context.Context, item TransferQueueItem, usesSegmentTransferQueue bool) error
|
|
// DeleteTransferQueueItem deletes a graceful exit transfer queue entry.
|
|
DeleteTransferQueueItem(ctx context.Context, nodeID storj.NodeID, key metabase.SegmentKey, StreamID uuid.UUID, Position metabase.SegmentPosition, pieceNum int32) error
|
|
// DeleteTransferQueueItem deletes a graceful exit transfer queue entries by nodeID.
|
|
DeleteTransferQueueItems(ctx context.Context, nodeID storj.NodeID, usesSegmentTransferQueue bool) error
|
|
// DeleteFinishedTransferQueueItem deletes finished graceful exit transfer queue entries.
|
|
DeleteFinishedTransferQueueItems(ctx context.Context, nodeID storj.NodeID, usesSegmentTransferQueue bool) error
|
|
// DeleteAllFinishedTransferQueueItems deletes all graceful exit transfer
|
|
// queue items whose nodes have finished the exit before the indicated time
|
|
// returning the total number of deleted items.
|
|
DeleteAllFinishedTransferQueueItems(ctx context.Context, before time.Time, asOfSystemTimeInterval time.Duration, batchSize int) (count int64, err error)
|
|
// DeleteFinishedExitProgress deletes exit progress entries for nodes that
|
|
// finished exiting before the indicated time, returns number of deleted entries.
|
|
DeleteFinishedExitProgress(ctx context.Context, before time.Time, asOfSystemTimeInterval time.Duration) (count int64, err error)
|
|
// GetFinishedExitNodes gets nodes that are marked having finished graceful exit before a given time.
|
|
GetFinishedExitNodes(ctx context.Context, before time.Time, asOfSystemTimeInterval time.Duration) (finishedNodes []storj.NodeID, err error)
|
|
// GetTransferQueueItem gets a graceful exit transfer queue entry.
|
|
GetTransferQueueItem(ctx context.Context, nodeID storj.NodeID, key metabase.SegmentKey, StreamID uuid.UUID, Position metabase.SegmentPosition, pieceNum int32) (*TransferQueueItem, error)
|
|
// GetIncomplete gets incomplete graceful exit transfer queue entries ordered by durability ratio and queued date ascending.
|
|
GetIncomplete(ctx context.Context, nodeID storj.NodeID, limit int, offset int64, usesSegmentTransferQueue bool) ([]*TransferQueueItem, error)
|
|
// GetIncompleteNotFailed gets incomplete graceful exit transfer queue entries in the database ordered by durability ratio and queued date ascending.
|
|
GetIncompleteNotFailed(ctx context.Context, nodeID storj.NodeID, limit int, offset int64, usesSegmentTransferQueue bool) ([]*TransferQueueItem, error)
|
|
// GetIncompleteNotFailed gets incomplete graceful exit transfer queue entries that have failed <= maxFailures times, ordered by durability ratio and queued date ascending.
|
|
GetIncompleteFailed(ctx context.Context, nodeID storj.NodeID, maxFailures int, limit int, offset int64, usesSegmentTransferQueue bool) ([]*TransferQueueItem, error)
|
|
// IncrementOrderLimitSendCount increments the number of times a node has been sent an order limit for transferring.
|
|
IncrementOrderLimitSendCount(ctx context.Context, nodeID storj.NodeID, key metabase.SegmentKey, StreamID uuid.UUID, Position metabase.SegmentPosition, pieceNum int32) error
|
|
// CountFinishedTransferQueueItemsByNode return a map of the nodes which has
|
|
// finished the exit before the indicated time but there are at least one item
|
|
// left in the transfer queue.
|
|
CountFinishedTransferQueueItemsByNode(ctx context.Context, before time.Time, asOfSystemTimeInterval time.Duration) (map[storj.NodeID]int64, error)
|
|
}
|