satellite/dbcleanup: remove expired serial chore

Change-Id: Ib71d41eb6679d6435e5bc10b6244dac66380a74e

private/testplanet/satellite.go

@@ -43,7 +43,6 @@ import (
 	"storj.io/storj/satellite/console/consoleauth"
 	"storj.io/storj/satellite/console/consoleweb"
 	"storj.io/storj/satellite/contact"
-	"storj.io/storj/satellite/dbcleanup"
 	"storj.io/storj/satellite/gc"
 	"storj.io/storj/satellite/gracefulexit"
 	"storj.io/storj/satellite/inspector"
@@ -137,10 +136,6 @@ type Satellite struct {
 		Chore *expireddeletion.Chore
 	}
 
-	DBCleanup struct {
-		Chore *dbcleanup.Chore
-	}
-
 	Accounting struct {
 		Tally            *tally.Service
 		Rollup           *rollup.Service
@@ -552,10 +547,6 @@ func (planet *Planet) newSatellite(ctx context.Context, prefix string, index int
 			Interval: defaultInterval,
 			Enabled:  true,
 		},
-		DBCleanup: dbcleanup.Config{
-			SerialsInterval: defaultInterval,
-			BatchSize:       1000,
-		},
 		Tally: tally.Config{
 			Interval: defaultInterval,
 		},
@@ -733,8 +724,6 @@ func createNewSystem(name string, log *zap.Logger, config satellite.Config, peer
 
 	system.ExpiredDeletion.Chore = peer.ExpiredDeletion.Chore
 
-	system.DBCleanup.Chore = peer.DBCleanup.Chore
-
 	system.Accounting.Tally = peer.Accounting.Tally
 	system.Accounting.Rollup = peer.Accounting.Rollup
 	system.Accounting.ProjectUsage = api.Accounting.ProjectUsage

satellite/core.go

@@ -31,7 +31,6 @@ import (
 	"storj.io/storj/satellite/accounting/tally"
 	"storj.io/storj/satellite/audit"
 	"storj.io/storj/satellite/contact"
-	"storj.io/storj/satellite/dbcleanup"
 	"storj.io/storj/satellite/gc"
 	"storj.io/storj/satellite/gracefulexit"
 	"storj.io/storj/satellite/metainfo"
@@ -109,10 +108,6 @@ type Core struct {
 		Chore *expireddeletion.Chore
 	}
 
-	DBCleanup struct {
-		Chore *dbcleanup.Chore
-	}
-
 	Accounting struct {
 		Tally                 *tally.Service
 		Rollup                *rollup.Service
@@ -386,17 +381,6 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB,
 			debug.Cycle("Expired Segments Chore", peer.ExpiredDeletion.Chore.Loop))
 	}
 
-	{ // setup db cleanup
-		peer.DBCleanup.Chore = dbcleanup.NewChore(peer.Log.Named("dbcleanup"), peer.DB.Orders(), config.DBCleanup)
-		peer.Services.Add(lifecycle.Item{
-			Name:  "dbcleanup",
-			Run:   peer.DBCleanup.Chore.Run,
-			Close: peer.DBCleanup.Chore.Close,
-		})
-		peer.Debug.Server.Panel.Add(
-			debug.Cycle("DB Cleanup Serials", peer.DBCleanup.Chore.Serials))
-	}
-
 	{ // setup accounting
 		peer.Accounting.Tally = tally.New(peer.Log.Named("accounting:tally"), peer.DB.StoragenodeAccounting(), peer.DB.ProjectAccounting(), peer.LiveAccounting.Cache, peer.Metainfo.Loop, config.Tally.Interval)
 		peer.Services.Add(lifecycle.Item{

satellite/dbcleanup/chore.go

@@ -1,92 +0,0 @@
-// Copyright (C) 2019 Storj Labs, Inc.
-// See LICENSE for copying information.
-
-package dbcleanup
-
-import (
-	"context"
-	"time"
-
-	"github.com/spacemonkeygo/monkit/v3"
-	"github.com/zeebo/errs"
-	"go.uber.org/zap"
-
-	"storj.io/common/sync2"
-	"storj.io/storj/satellite/orders"
-)
-
-var (
-	// Error the default dbcleanup errs class.
-	Error = errs.Class("dbcleanup error")
-
-	mon = monkit.Package()
-)
-
-// Config defines configuration struct for dbcleanup chore.
-type Config struct {
-	SerialsInterval time.Duration `help:"how often to delete expired serial numbers" default:"4h"`
-	BatchSize       int           `help:"number of records to delete per delete execution. 0 indicates no limit. applicable to cockroach DB only." default:"1000"`
-}
-
-// Chore for deleting DB entries that are no longer needed.
-//
-// architecture: Chore
-type Chore struct {
-	log    *zap.Logger
-	orders orders.DB
-
-	Serials *sync2.Cycle
-	config  Config
-
-	serialsDeleteOptions orders.SerialDeleteOptions
-}
-
-// NewChore creates new chore for deleting DB entries.
-func NewChore(log *zap.Logger, ordersDB orders.DB, config Config) *Chore {
-	return &Chore{
-		log:    log,
-		orders: ordersDB,
-
-		Serials: sync2.NewCycle(config.SerialsInterval),
-		config:  config,
-
-		serialsDeleteOptions: orders.SerialDeleteOptions{
-			BatchSize: config.BatchSize,
-		},
-	}
-}
-
-// Run starts the db cleanup chore.
-func (chore *Chore) Run(ctx context.Context) (err error) {
-	defer mon.Task()(&ctx)(&err)
-	return chore.Serials.Run(ctx, chore.deleteExpiredSerials)
-}
-
-func (chore *Chore) deleteExpiredSerials(ctx context.Context) (err error) {
-	defer mon.Task()(&ctx)(&err)
-	chore.log.Debug("deleting expired serial numbers")
-
-	now := time.Now()
-
-	deleted, err := chore.orders.DeleteExpiredSerials(ctx, now, chore.serialsDeleteOptions)
-	if err != nil {
-		chore.log.Error("deleting expired serial numbers", zap.Error(err))
-	} else {
-		chore.log.Debug("expired serials deleted", zap.Int("items deleted", deleted))
-	}
-
-	deleted, err = chore.orders.DeleteExpiredConsumedSerials(ctx, now)
-	if err != nil {
-		chore.log.Error("deleting expired serial numbers", zap.Error(err))
-	} else {
-		chore.log.Debug("expired serials deleted", zap.Int("items deleted", deleted))
-	}
-
-	return nil
-}
-
-// Close stops the dbcleanup chore.
-func (chore *Chore) Close() error {
-	chore.Serials.Close()
-	return nil
-}

satellite/dbcleanup/chore_test.go

@@ -1,68 +0,0 @@
-// Copyright (C) 2019 Storj Labs, Inc.
-// See LICENSE for copying information.
-
-package dbcleanup_test
-
-import (
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-
-	"storj.io/common/storj"
-	"storj.io/common/testcontext"
-	"storj.io/storj/private/testplanet"
-)
-
-func TestDeleteExpiredSerials(t *testing.T) {
-	testplanet.Run(t, testplanet.Config{
-		SatelliteCount: 1, StorageNodeCount: 1, UplinkCount: 0,
-	}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
-		satellite := planet.Satellites[0]
-		node := planet.StorageNodes[0].ID()
-		satellite.DBCleanup.Chore.Serials.Pause()
-
-		var expiredSerials []storj.SerialNumber
-		for i := 0; i < 5; i++ {
-			expiredSerials = append(expiredSerials, storj.SerialNumber{byte(i)})
-		}
-
-		var freshSerials []storj.SerialNumber
-		for i := 5; i < 10; i++ {
-			freshSerials = append(freshSerials, storj.SerialNumber{byte(i)})
-		}
-
-		yesterday := time.Now().Add(-24 * time.Hour)
-		for _, serial := range expiredSerials {
-			err := satellite.DB.Orders().CreateSerialInfo(ctx, serial, []byte("bucket"), yesterday)
-			require.NoError(t, err)
-
-			_, err = satellite.DB.Orders().UseSerialNumber(ctx, serial, node)
-			require.NoError(t, err)
-		}
-
-		tomorrow := yesterday.Add(48 * time.Hour)
-		for _, serial := range freshSerials {
-			err := satellite.DB.Orders().CreateSerialInfo(ctx, serial, []byte("bucket"), tomorrow)
-			require.NoError(t, err)
-
-			_, err = satellite.DB.Orders().UseSerialNumber(ctx, serial, node)
-			require.NoError(t, err)
-		}
-
-		// trigger expired serial number deletion
-		satellite.DBCleanup.Chore.Serials.TriggerWait()
-
-		// check expired serial numbers have been deleted from serial_numbers and used_serials
-		for _, serial := range expiredSerials {
-			_, err := satellite.DB.Orders().UseSerialNumber(ctx, serial, node)
-			require.EqualError(t, err, "serial number: serial number not found")
-		}
-
-		// check fresh serial numbers have not been deleted
-		for _, serial := range freshSerials {
-			_, err := satellite.DB.Orders().UseSerialNumber(ctx, serial, node)
-			require.EqualError(t, err, "serial number: serial number already used")
-		}
-	})
-}

satellite/orders/endpoint.go

@@ -38,10 +38,6 @@ type DB interface {
 	UseSerialNumber(ctx context.Context, serialNumber storj.SerialNumber, storageNodeID storj.NodeID) ([]byte, error)
 	// UnuseSerialNumber removes pair serial number -> storage node id from database
 	UnuseSerialNumber(ctx context.Context, serialNumber storj.SerialNumber, storageNodeID storj.NodeID) error
-	// DeleteExpiredSerials deletes all expired serials in serial_number, used_serials, and consumed_serials table.
-	DeleteExpiredSerials(ctx context.Context, now time.Time, options SerialDeleteOptions) (_ int, err error)
-	// DeleteExpiredConsumedSerials deletes all expired serials in the consumed_serials table.
-	DeleteExpiredConsumedSerials(ctx context.Context, now time.Time) (_ int, err error)
 	// GetBucketIDFromSerialNumber returns the bucket ID associated with the serial number
 	GetBucketIDFromSerialNumber(ctx context.Context, serialNumber storj.SerialNumber) ([]byte, error)
 

satellite/orders/serials_test.go

@@ -1,82 +0,0 @@
-// Copyright (C) 2019 Storj Labs, Inc.
-// See LICENSE for copying information.
-
-package orders_test
-
-import (
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-
-	"storj.io/common/storj"
-	"storj.io/common/testcontext"
-	"storj.io/storj/satellite"
-	"storj.io/storj/satellite/orders"
-	"storj.io/storj/satellite/satellitedb/satellitedbtest"
-)
-
-func TestSerialNumbers(t *testing.T) {
-	satellitedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db satellite.DB) {
-		ordersDB := db.Orders()
-
-		expectedBucket := []byte("bucketID")
-		err := ordersDB.CreateSerialInfo(ctx, storj.SerialNumber{1}, expectedBucket, time.Now())
-		require.NoError(t, err)
-
-		bucketID, err := ordersDB.UseSerialNumber(ctx, storj.SerialNumber{1}, storj.NodeID{1})
-		require.NoError(t, err)
-		require.Equal(t, expectedBucket, bucketID)
-
-		// try to use used serial number
-		_, err = ordersDB.UseSerialNumber(ctx, storj.SerialNumber{1}, storj.NodeID{1})
-		require.Error(t, err)
-		require.True(t, orders.ErrUsingSerialNumber.Has(err))
-
-		err = ordersDB.UnuseSerialNumber(ctx, storj.SerialNumber{1}, storj.NodeID{1})
-		require.NoError(t, err)
-
-		bucketID, err = ordersDB.UseSerialNumber(ctx, storj.SerialNumber{1}, storj.NodeID{1})
-		require.NoError(t, err)
-		require.Equal(t, expectedBucket, bucketID)
-
-		// not existing serial number
-		bucketID, err = ordersDB.UseSerialNumber(ctx, storj.SerialNumber{99}, storj.NodeID{1})
-		require.Error(t, err)
-		require.True(t, orders.ErrUsingSerialNumber.Has(err))
-		require.Empty(t, bucketID)
-
-		deleted, err := ordersDB.DeleteExpiredSerials(ctx, time.Now(), orders.SerialDeleteOptions{})
-		require.NoError(t, err)
-		require.Equal(t, deleted, 1)
-
-		// check serial number has been deleted from serial_numbers and used_serials
-		_, err = ordersDB.UseSerialNumber(ctx, storj.SerialNumber{1}, storj.NodeID{1})
-		require.EqualError(t, err, "serial number: serial number not found")
-	})
-}
-
-func TestDeleteExpiredWithOptions(t *testing.T) {
-	satellitedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db satellite.DB) {
-		ordersDB := db.Orders()
-
-		err := ordersDB.CreateSerialInfo(ctx, storj.SerialNumber{1}, []byte("bucketID1"), time.Now())
-		require.NoError(t, err)
-
-		err = ordersDB.CreateSerialInfo(ctx, storj.SerialNumber{2}, []byte("bucketID2"), time.Now())
-		require.NoError(t, err)
-
-		deleted, err := ordersDB.DeleteExpiredSerials(ctx, time.Now(), orders.SerialDeleteOptions{
-			BatchSize: 1,
-		})
-		require.NoError(t, err)
-		require.Equal(t, 2, deleted)
-
-		// check serial number has been deleted from serial_numbers and used_serials
-		_, err = ordersDB.UseSerialNumber(ctx, storj.SerialNumber{1}, storj.NodeID{1})
-		require.EqualError(t, err, "serial number: serial number not found")
-
-		_, err = ordersDB.UseSerialNumber(ctx, storj.SerialNumber{2}, storj.NodeID{1})
-		require.EqualError(t, err, "serial number: serial number not found")
-	})
-}

satellite/peer.go

@@ -26,7 +26,6 @@ import (
 	"storj.io/storj/satellite/console"
 	"storj.io/storj/satellite/console/consoleweb"
 	"storj.io/storj/satellite/contact"
-	"storj.io/storj/satellite/dbcleanup"
 	"storj.io/storj/satellite/gc"
 	"storj.io/storj/satellite/gracefulexit"
 	"storj.io/storj/satellite/mailservice"
@@ -129,8 +128,6 @@ type Config struct {
 
 	ExpiredDeletion expireddeletion.Config
 
-	DBCleanup dbcleanup.Config
-
 	Tally            tally.Config
 	Rollup           rollup.Config
 	LiveAccounting   live.Config

satellite/satellitedb/orders.go

@@ -58,66 +58,6 @@ func (db *ordersDB) CreateSerialInfo(ctx context.Context, serialNumber storj.Ser
 	)
 }
 
-// DeleteExpiredSerials deletes all expired serials in the serial_number table.
-func (db *ordersDB) DeleteExpiredSerials(ctx context.Context, now time.Time, options orders.SerialDeleteOptions) (_ int, err error) {
-	defer mon.Task()(&ctx)(&err)
-
-	if db.db.implementation == dbutil.Cockroach && options.BatchSize > 0 {
-		var deleted int
-
-		for {
-			d, err := func() (_ int, err error) {
-				var r int
-				rs, err := db.db.Query(ctx, "DELETE FROM serial_numbers WHERE expires_at <= $1 ORDER BY expires_at DESC LIMIT $2 RETURNING expires_at;", now.UTC(), options.BatchSize)
-				if err != nil {
-					return 0, err
-				}
-				defer func() { err = errs.Combine(err, rs.Close()) }()
-
-				for rs.Next() {
-					err = rs.Scan(&now)
-					if err != nil {
-						return r, err
-					}
-					r++
-				}
-				if rs.Err() != nil {
-					return r, rs.Err()
-				}
-
-				return r, nil
-			}()
-			deleted += d
-			if err != nil {
-				return deleted, err
-			}
-
-			if d < options.BatchSize {
-				break
-			}
-		}
-
-		return deleted, err
-	}
-
-	count, err := db.db.Delete_SerialNumber_By_ExpiresAt_LessOrEqual(ctx, dbx.SerialNumber_ExpiresAt(now.UTC()))
-	if err != nil {
-		return 0, err
-	}
-	return int(count), nil
-}
-
-// DeleteExpiredConsumedSerials deletes all expired serials in the consumed_serials table.
-func (db *ordersDB) DeleteExpiredConsumedSerials(ctx context.Context, now time.Time) (_ int, err error) {
-	defer mon.Task()(&ctx, now)(&err)
-
-	count, err := db.db.Delete_ConsumedSerial_By_ExpiresAt_LessOrEqual(ctx, dbx.ConsumedSerial_ExpiresAt(now))
-	if err != nil {
-		return 0, err
-	}
-	return int(count), nil
-}
-
 // UseSerialNumber creates a used serial number entry in database from an
 // existing serial number.
 // It returns the bucket ID associated to serialNumber.

scripts/testdata/satellite-config.yaml.lock

@@ -160,12 +160,6 @@ contact.external-address: ""
 # macaroon revocation cache expiration
 # database-options.revocations-cache.expiration: 5m0s
 
-# number of records to delete per delete execution. 0 indicates no limit. applicable to cockroach DB only.
-# db-cleanup.batch-size: 1000
-
-# how often to delete expired serial numbers
-# db-cleanup.serials-interval: 4h0m0s
-
 # Maximum Database Connection Lifetime, -1ns means the stdlib default
 # db.conn_max_lifetime: 30m0s