JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/dbcleanup: remove expired serial chore

Browse Source 
Change-Id: Ib71d41eb6679d6435e5bc10b6244dac66380a74e
This commit is contained in:
Jessica Grebenschikov 2020-12-18 09:00:44 -08:00
parent 97a5e6c814
commit da0327c9b7
9 changed files with 0 additions and 342 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
private/testplanet/satellite.go
View File

@ -43,7 +43,6 @@ import (
"storj.io/storj/satellite/console/consoleauth"
"storj.io/storj/satellite/console/consoleweb"
"storj.io/storj/satellite/contact"
"storj.io/storj/satellite/dbcleanup"
"storj.io/storj/satellite/gc"
"storj.io/storj/satellite/gracefulexit"
"storj.io/storj/satellite/inspector"
@ -137,10 +136,6 @@ type Satellite struct {
Chore *expireddeletion.Chore
}
DBCleanup struct {
Chore *dbcleanup.Chore
}
Accounting struct {
Tally *tally.Service
Rollup *rollup.Service
@ -552,10 +547,6 @@ func (planet *Planet) newSatellite(ctx context.Context, prefix string, index int
Interval: defaultInterval,
Enabled: true,
},
DBCleanup: dbcleanup.Config{
SerialsInterval: defaultInterval,
BatchSize: 1000,
},
Tally: tally.Config{
Interval: defaultInterval,
},
@ -733,8 +724,6 @@ func createNewSystem(name string, log *zap.Logger, config satellite.Config, peer
system.ExpiredDeletion.Chore = peer.ExpiredDeletion.Chore
system.DBCleanup.Chore = peer.DBCleanup.Chore
system.Accounting.Tally = peer.Accounting.Tally
system.Accounting.Rollup = peer.Accounting.Rollup
system.Accounting.ProjectUsage = api.Accounting.ProjectUsage

16
satellite/core.go
View File

@ -31,7 +31,6 @@ import (
"storj.io/storj/satellite/accounting/tally"
"storj.io/storj/satellite/audit"
"storj.io/storj/satellite/contact"
"storj.io/storj/satellite/dbcleanup"
"storj.io/storj/satellite/gc"
"storj.io/storj/satellite/gracefulexit"
"storj.io/storj/satellite/metainfo"
@ -109,10 +108,6 @@ type Core struct {
Chore *expireddeletion.Chore
}
DBCleanup struct {
Chore *dbcleanup.Chore
}
Accounting struct {
Tally *tally.Service
Rollup *rollup.Service
@ -386,17 +381,6 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB,
debug.Cycle("Expired Segments Chore", peer.ExpiredDeletion.Chore.Loop))
}
{ // setup db cleanup
peer.DBCleanup.Chore = dbcleanup.NewChore(peer.Log.Named("dbcleanup"), peer.DB.Orders(), config.DBCleanup)
peer.Services.Add(lifecycle.Item{
Name: "dbcleanup",
Run: peer.DBCleanup.Chore.Run,
Close: peer.DBCleanup.Chore.Close,
})
peer.Debug.Server.Panel.Add(
debug.Cycle("DB Cleanup Serials", peer.DBCleanup.Chore.Serials))
}
{ // setup accounting
peer.Accounting.Tally = tally.New(peer.Log.Named("accounting:tally"), peer.DB.StoragenodeAccounting(), peer.DB.ProjectAccounting(), peer.LiveAccounting.Cache, peer.Metainfo.Loop, config.Tally.Interval)
peer.Services.Add(lifecycle.Item{

92
satellite/dbcleanup/chore.go
View File

@ -1,92 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package dbcleanup
import (
"context"
"time"
"github.com/spacemonkeygo/monkit/v3"
"github.com/zeebo/errs"
"go.uber.org/zap"
"storj.io/common/sync2"
"storj.io/storj/satellite/orders"
)
var (
// Error the default dbcleanup errs class.
Error = errs.Class("dbcleanup error")
mon = monkit.Package()
)
// Config defines configuration struct for dbcleanup chore.
type Config struct {
SerialsInterval time.Duration `help:"how often to delete expired serial numbers" default:"4h"`
BatchSize int `help:"number of records to delete per delete execution. 0 indicates no limit. applicable to cockroach DB only." default:"1000"`
}
// Chore for deleting DB entries that are no longer needed.
//
// architecture: Chore
type Chore struct {
log *zap.Logger
orders orders.DB
Serials *sync2.Cycle
config Config
serialsDeleteOptions orders.SerialDeleteOptions
}
// NewChore creates new chore for deleting DB entries.
func NewChore(log *zap.Logger, ordersDB orders.DB, config Config) *Chore {
return &Chore{
log: log,
orders: ordersDB,
Serials: sync2.NewCycle(config.SerialsInterval),
config: config,
serialsDeleteOptions: orders.SerialDeleteOptions{
BatchSize: config.BatchSize,
},
}
}
// Run starts the db cleanup chore.
func (chore *Chore) Run(ctx context.Context) (err error) {
defer mon.Task()(&ctx)(&err)
return chore.Serials.Run(ctx, chore.deleteExpiredSerials)
}
func (chore *Chore) deleteExpiredSerials(ctx context.Context) (err error) {
defer mon.Task()(&ctx)(&err)
chore.log.Debug("deleting expired serial numbers")
now := time.Now()
deleted, err := chore.orders.DeleteExpiredSerials(ctx, now, chore.serialsDeleteOptions)
if err != nil {
chore.log.Error("deleting expired serial numbers", zap.Error(err))
} else {
chore.log.Debug("expired serials deleted", zap.Int("items deleted", deleted))
}
deleted, err = chore.orders.DeleteExpiredConsumedSerials(ctx, now)
if err != nil {
chore.log.Error("deleting expired serial numbers", zap.Error(err))
} else {
chore.log.Debug("expired serials deleted", zap.Int("items deleted", deleted))
}
return nil
}
// Close stops the dbcleanup chore.
func (chore *Chore) Close() error {
chore.Serials.Close()
return nil
}

68
satellite/dbcleanup/chore_test.go
View File

@ -1,68 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package dbcleanup_test
import (
"testing"
"time"
"github.com/stretchr/testify/require"
"storj.io/common/storj"
"storj.io/common/testcontext"
"storj.io/storj/private/testplanet"
)
func TestDeleteExpiredSerials(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1, StorageNodeCount: 1, UplinkCount: 0,
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
satellite := planet.Satellites[0]
node := planet.StorageNodes[0].ID()
satellite.DBCleanup.Chore.Serials.Pause()
var expiredSerials []storj.SerialNumber
for i := 0; i < 5; i++ {
expiredSerials = append(expiredSerials, storj.SerialNumber{byte(i)})
}
var freshSerials []storj.SerialNumber
for i := 5; i < 10; i++ {
freshSerials = append(freshSerials, storj.SerialNumber{byte(i)})
}
yesterday := time.Now().Add(-24 * time.Hour)
for _, serial := range expiredSerials {
err := satellite.DB.Orders().CreateSerialInfo(ctx, serial, []byte("bucket"), yesterday)
require.NoError(t, err)
_, err = satellite.DB.Orders().UseSerialNumber(ctx, serial, node)
require.NoError(t, err)
}
tomorrow := yesterday.Add(48 * time.Hour)
for _, serial := range freshSerials {
err := satellite.DB.Orders().CreateSerialInfo(ctx, serial, []byte("bucket"), tomorrow)
require.NoError(t, err)
_, err = satellite.DB.Orders().UseSerialNumber(ctx, serial, node)
require.NoError(t, err)
}
// trigger expired serial number deletion
satellite.DBCleanup.Chore.Serials.TriggerWait()
// check expired serial numbers have been deleted from serial_numbers and used_serials
for _, serial := range expiredSerials {
_, err := satellite.DB.Orders().UseSerialNumber(ctx, serial, node)
require.EqualError(t, err, "serial number: serial number not found")
}
// check fresh serial numbers have not been deleted
for _, serial := range freshSerials {
_, err := satellite.DB.Orders().UseSerialNumber(ctx, serial, node)
require.EqualError(t, err, "serial number: serial number already used")
}
})
}

4
satellite/orders/endpoint.go
View File

@ -38,10 +38,6 @@ type DB interface {
UseSerialNumber(ctx context.Context, serialNumber storj.SerialNumber, storageNodeID storj.NodeID) ([]byte, error)
// UnuseSerialNumber removes pair serial number -> storage node id from database
UnuseSerialNumber(ctx context.Context, serialNumber storj.SerialNumber, storageNodeID storj.NodeID) error
// DeleteExpiredSerials deletes all expired serials in serial_number, used_serials, and consumed_serials table.
DeleteExpiredSerials(ctx context.Context, now time.Time, options SerialDeleteOptions) (_ int, err error)
// DeleteExpiredConsumedSerials deletes all expired serials in the consumed_serials table.
DeleteExpiredConsumedSerials(ctx context.Context, now time.Time) (_ int, err error)
// GetBucketIDFromSerialNumber returns the bucket ID associated with the serial number
GetBucketIDFromSerialNumber(ctx context.Context, serialNumber storj.SerialNumber) ([]byte, error)

82
satellite/orders/serials_test.go
View File

@ -1,82 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package orders_test
import (
"testing"
"time"
"github.com/stretchr/testify/require"
"storj.io/common/storj"
"storj.io/common/testcontext"
"storj.io/storj/satellite"
"storj.io/storj/satellite/orders"
"storj.io/storj/satellite/satellitedb/satellitedbtest"
)
func TestSerialNumbers(t *testing.T) {
satellitedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db satellite.DB) {
ordersDB := db.Orders()
expectedBucket := []byte("bucketID")
err := ordersDB.CreateSerialInfo(ctx, storj.SerialNumber{1}, expectedBucket, time.Now())
require.NoError(t, err)
bucketID, err := ordersDB.UseSerialNumber(ctx, storj.SerialNumber{1}, storj.NodeID{1})
require.NoError(t, err)
require.Equal(t, expectedBucket, bucketID)
// try to use used serial number
_, err = ordersDB.UseSerialNumber(ctx, storj.SerialNumber{1}, storj.NodeID{1})
require.Error(t, err)
require.True(t, orders.ErrUsingSerialNumber.Has(err))
err = ordersDB.UnuseSerialNumber(ctx, storj.SerialNumber{1}, storj.NodeID{1})
require.NoError(t, err)
bucketID, err = ordersDB.UseSerialNumber(ctx, storj.SerialNumber{1}, storj.NodeID{1})
require.NoError(t, err)
require.Equal(t, expectedBucket, bucketID)
// not existing serial number
bucketID, err = ordersDB.UseSerialNumber(ctx, storj.SerialNumber{99}, storj.NodeID{1})
require.Error(t, err)
require.True(t, orders.ErrUsingSerialNumber.Has(err))
require.Empty(t, bucketID)
deleted, err := ordersDB.DeleteExpiredSerials(ctx, time.Now(), orders.SerialDeleteOptions{})
require.NoError(t, err)
require.Equal(t, deleted, 1)
// check serial number has been deleted from serial_numbers and used_serials
_, err = ordersDB.UseSerialNumber(ctx, storj.SerialNumber{1}, storj.NodeID{1})
require.EqualError(t, err, "serial number: serial number not found")
})
}
func TestDeleteExpiredWithOptions(t *testing.T) {
satellitedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db satellite.DB) {
ordersDB := db.Orders()
err := ordersDB.CreateSerialInfo(ctx, storj.SerialNumber{1}, []byte("bucketID1"), time.Now())
require.NoError(t, err)
err = ordersDB.CreateSerialInfo(ctx, storj.SerialNumber{2}, []byte("bucketID2"), time.Now())
require.NoError(t, err)
deleted, err := ordersDB.DeleteExpiredSerials(ctx, time.Now(), orders.SerialDeleteOptions{
BatchSize: 1,
})
require.NoError(t, err)
require.Equal(t, 2, deleted)
// check serial number has been deleted from serial_numbers and used_serials
_, err = ordersDB.UseSerialNumber(ctx, storj.SerialNumber{1}, storj.NodeID{1})
require.EqualError(t, err, "serial number: serial number not found")
_, err = ordersDB.UseSerialNumber(ctx, storj.SerialNumber{2}, storj.NodeID{1})
require.EqualError(t, err, "serial number: serial number not found")
})
}

3
satellite/peer.go
View File

@ -26,7 +26,6 @@ import (
"storj.io/storj/satellite/console"
"storj.io/storj/satellite/console/consoleweb"
"storj.io/storj/satellite/contact"
"storj.io/storj/satellite/dbcleanup"
"storj.io/storj/satellite/gc"
"storj.io/storj/satellite/gracefulexit"
"storj.io/storj/satellite/mailservice"
@ -129,8 +128,6 @@ type Config struct {
ExpiredDeletion expireddeletion.Config
DBCleanup dbcleanup.Config
Tally tally.Config
Rollup rollup.Config
LiveAccounting live.Config

60
satellite/satellitedb/orders.go
View File

@ -58,66 +58,6 @@ func (db *ordersDB) CreateSerialInfo(ctx context.Context, serialNumber storj.Ser
)
}
// DeleteExpiredSerials deletes all expired serials in the serial_number table.
func (db *ordersDB) DeleteExpiredSerials(ctx context.Context, now time.Time, options orders.SerialDeleteOptions) (_ int, err error) {
defer mon.Task()(&ctx)(&err)
if db.db.implementation == dbutil.Cockroach && options.BatchSize > 0 {
var deleted int
for {
d, err := func() (_ int, err error) {
var r int
rs, err := db.db.Query(ctx, "DELETE FROM serial_numbers WHERE expires_at <= $1 ORDER BY expires_at DESC LIMIT $2 RETURNING expires_at;", now.UTC(), options.BatchSize)
if err != nil {
return 0, err
}
defer func() { err = errs.Combine(err, rs.Close()) }()
for rs.Next() {
err = rs.Scan(&now)
if err != nil {
return r, err
}
r++
}
if rs.Err() != nil {
return r, rs.Err()
}
return r, nil
}()
deleted += d
if err != nil {
return deleted, err
}
if d < options.BatchSize {
break
}
}
return deleted, err
}
count, err := db.db.Delete_SerialNumber_By_ExpiresAt_LessOrEqual(ctx, dbx.SerialNumber_ExpiresAt(now.UTC()))
if err != nil {
return 0, err
}
return int(count), nil
}
// DeleteExpiredConsumedSerials deletes all expired serials in the consumed_serials table.
func (db *ordersDB) DeleteExpiredConsumedSerials(ctx context.Context, now time.Time) (_ int, err error) {
defer mon.Task()(&ctx, now)(&err)
count, err := db.db.Delete_ConsumedSerial_By_ExpiresAt_LessOrEqual(ctx, dbx.ConsumedSerial_ExpiresAt(now))
if err != nil {
return 0, err
}
return int(count), nil
}
// UseSerialNumber creates a used serial number entry in database from an
// existing serial number.
// It returns the bucket ID associated to serialNumber.

6
scripts/testdata/satellite-config.yaml.lock vendored
View File

@ -160,12 +160,6 @@ contact.external-address: ""
# macaroon revocation cache expiration
# database-options.revocations-cache.expiration: 5m0s
# number of records to delete per delete execution. 0 indicates no limit. applicable to cockroach DB only.
# db-cleanup.batch-size: 1000
# how often to delete expired serial numbers
# db-cleanup.serials-interval: 4h0m0s
# Maximum Database Connection Lifetime, -1ns means the stdlib default
# db.conn_max_lifetime: 30m0s