2019-10-23 13:04:54 +01:00
|
|
|
// Copyright (C) 2019 Storj Labs, Inc.
|
|
|
|
// See LICENSE for copying information.
|
|
|
|
|
|
|
|
package stripecoinpayments
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/zeebo/errs"
|
|
|
|
"go.uber.org/zap"
|
2019-10-29 16:04:34 +00:00
|
|
|
"golang.org/x/sync/errgroup"
|
2019-10-23 13:04:54 +01:00
|
|
|
|
satellite/payments: chore to migrate big.Float values out of db
All code on known satellites at this moment in time should know how to
populate and use the new numeric columns on the
stripecoinpayments_tx_conversion_rates and coinpayments_transactions
tables in the satellite db. However, there are still gob-encoded
big.Float values in the database from before these columns existed. To
get rid of those values, so that we can excise the gob-decoding code
from the relevant sections, however, we need something to read the gob
bytestrings and convert them to numeric values, a few at a time, until
they're all gone.
To accomplish that, this change adds two chores to be run in the
satellite core process- one for the coinpayments_transactions table, and
one for the stripecoinpayments_tx_conversion_rates table. They should
run relatively infrequently, so that we do not impose any undue load on
processing resources or the db.
Both of these chores work without using explicit sql transactions, but
should still be concurrent-safe, since they work by way of
compare-and-swap type operations.
If the satellite core process needs to be restarted, both of these
chores will start scanning for migrateable rows from the beginning of
the id space again. This is not ideal, but shouldn't be a problem (as
far as I can tell, there are only a few thousand rows at most in either
of these tables on any production satellite).
Change-Id: I733b7cd96760d506a1cf52735f598c6c3aa19735
2021-10-29 14:32:59 +01:00
|
|
|
"storj.io/common/errs2"
|
2019-12-27 11:48:47 +00:00
|
|
|
"storj.io/common/sync2"
|
2019-10-23 13:04:54 +01:00
|
|
|
)
|
|
|
|
|
2019-10-29 16:04:34 +00:00
|
|
|
// ErrChore is stripecoinpayments clearing loop chore error class.
|
2021-04-28 09:06:17 +01:00
|
|
|
var ErrChore = errs.Class("stripecoinpayments chore")
|
2019-10-23 13:04:54 +01:00
|
|
|
|
2019-10-29 16:04:34 +00:00
|
|
|
// Chore runs clearing process of reconciling transactions deposits,
|
2019-10-23 13:04:54 +01:00
|
|
|
// customer balance, invoices and usages.
|
2019-10-29 16:04:34 +00:00
|
|
|
//
|
|
|
|
// architecture: Chore
|
|
|
|
type Chore struct {
|
|
|
|
log *zap.Logger
|
|
|
|
service *Service
|
2020-01-30 13:06:43 +00:00
|
|
|
TransactionCycle *sync2.Cycle
|
|
|
|
AccountBalanceCycle *sync2.Cycle
|
satellite/payments: chore to migrate big.Float values out of db
All code on known satellites at this moment in time should know how to
populate and use the new numeric columns on the
stripecoinpayments_tx_conversion_rates and coinpayments_transactions
tables in the satellite db. However, there are still gob-encoded
big.Float values in the database from before these columns existed. To
get rid of those values, so that we can excise the gob-decoding code
from the relevant sections, however, we need something to read the gob
bytestrings and convert them to numeric values, a few at a time, until
they're all gone.
To accomplish that, this change adds two chores to be run in the
satellite core process- one for the coinpayments_transactions table, and
one for the stripecoinpayments_tx_conversion_rates table. They should
run relatively infrequently, so that we do not impose any undue load on
processing resources or the db.
Both of these chores work without using explicit sql transactions, but
should still be concurrent-safe, since they work by way of
compare-and-swap type operations.
If the satellite core process needs to be restarted, both of these
chores will start scanning for migrateable rows from the beginning of
the id space again. This is not ideal, but shouldn't be a problem (as
far as I can tell, there are only a few thousand rows at most in either
of these tables on any production satellite).
Change-Id: I733b7cd96760d506a1cf52735f598c6c3aa19735
2021-10-29 14:32:59 +01:00
|
|
|
|
|
|
|
// temporary! remove once all gob-encoded big.Float records are gone from DBs on all satellites:
|
|
|
|
TransactionMigrationCycle *sync2.Cycle
|
|
|
|
ConversionRateMigrationCycle *sync2.Cycle
|
|
|
|
migrationBatchSize int
|
2019-10-23 13:04:54 +01:00
|
|
|
}
|
|
|
|
|
2019-10-29 16:04:34 +00:00
|
|
|
// NewChore creates new clearing loop chore.
|
2019-11-26 17:58:51 +00:00
|
|
|
// TODO: uncomment new interval when coupons will be finished.
|
satellite/payments: chore to migrate big.Float values out of db
All code on known satellites at this moment in time should know how to
populate and use the new numeric columns on the
stripecoinpayments_tx_conversion_rates and coinpayments_transactions
tables in the satellite db. However, there are still gob-encoded
big.Float values in the database from before these columns existed. To
get rid of those values, so that we can excise the gob-decoding code
from the relevant sections, however, we need something to read the gob
bytestrings and convert them to numeric values, a few at a time, until
they're all gone.
To accomplish that, this change adds two chores to be run in the
satellite core process- one for the coinpayments_transactions table, and
one for the stripecoinpayments_tx_conversion_rates table. They should
run relatively infrequently, so that we do not impose any undue load on
processing resources or the db.
Both of these chores work without using explicit sql transactions, but
should still be concurrent-safe, since they work by way of
compare-and-swap type operations.
If the satellite core process needs to be restarted, both of these
chores will start scanning for migrateable rows from the beginning of
the id space again. This is not ideal, but shouldn't be a problem (as
far as I can tell, there are only a few thousand rows at most in either
of these tables on any production satellite).
Change-Id: I733b7cd96760d506a1cf52735f598c6c3aa19735
2021-10-29 14:32:59 +01:00
|
|
|
func NewChore(log *zap.Logger, service *Service, txInterval, accBalanceInterval, migrationBatchInterval time.Duration, migrationBatchSize int) *Chore {
|
2019-10-29 16:04:34 +00:00
|
|
|
return &Chore{
|
satellite/payments: chore to migrate big.Float values out of db
All code on known satellites at this moment in time should know how to
populate and use the new numeric columns on the
stripecoinpayments_tx_conversion_rates and coinpayments_transactions
tables in the satellite db. However, there are still gob-encoded
big.Float values in the database from before these columns existed. To
get rid of those values, so that we can excise the gob-decoding code
from the relevant sections, however, we need something to read the gob
bytestrings and convert them to numeric values, a few at a time, until
they're all gone.
To accomplish that, this change adds two chores to be run in the
satellite core process- one for the coinpayments_transactions table, and
one for the stripecoinpayments_tx_conversion_rates table. They should
run relatively infrequently, so that we do not impose any undue load on
processing resources or the db.
Both of these chores work without using explicit sql transactions, but
should still be concurrent-safe, since they work by way of
compare-and-swap type operations.
If the satellite core process needs to be restarted, both of these
chores will start scanning for migrateable rows from the beginning of
the id space again. This is not ideal, but shouldn't be a problem (as
far as I can tell, there are only a few thousand rows at most in either
of these tables on any production satellite).
Change-Id: I733b7cd96760d506a1cf52735f598c6c3aa19735
2021-10-29 14:32:59 +01:00
|
|
|
log: log,
|
|
|
|
service: service,
|
|
|
|
TransactionCycle: sync2.NewCycle(txInterval),
|
|
|
|
AccountBalanceCycle: sync2.NewCycle(accBalanceInterval),
|
|
|
|
TransactionMigrationCycle: sync2.NewCycle(migrationBatchInterval),
|
|
|
|
ConversionRateMigrationCycle: sync2.NewCycle(migrationBatchInterval),
|
|
|
|
migrationBatchSize: migrationBatchSize,
|
2019-10-23 13:04:54 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Run runs all clearing related cycles.
|
2019-10-29 16:04:34 +00:00
|
|
|
func (chore *Chore) Run(ctx context.Context) (err error) {
|
2019-10-23 13:04:54 +01:00
|
|
|
defer mon.Task()(&ctx)(&err)
|
|
|
|
|
2019-10-29 16:04:34 +00:00
|
|
|
var group errgroup.Group
|
|
|
|
|
|
|
|
chore.TransactionCycle.Start(ctx, &group,
|
|
|
|
func(ctx context.Context) error {
|
|
|
|
chore.log.Info("running transactions update cycle")
|
|
|
|
|
|
|
|
if err := chore.service.updateTransactionsLoop(ctx); err != nil {
|
|
|
|
chore.log.Error("transaction update cycle failed", zap.Error(ErrChore.Wrap(err)))
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
},
|
|
|
|
)
|
|
|
|
chore.AccountBalanceCycle.Start(ctx, &group,
|
2019-10-23 13:04:54 +01:00
|
|
|
func(ctx context.Context) error {
|
2019-10-29 16:04:34 +00:00
|
|
|
chore.log.Info("running account balance update cycle")
|
2019-10-23 13:04:54 +01:00
|
|
|
|
2019-10-29 16:04:34 +00:00
|
|
|
if err := chore.service.updateAccountBalanceLoop(ctx); err != nil {
|
|
|
|
chore.log.Error("account balance update cycle failed", zap.Error(ErrChore.Wrap(err)))
|
2019-10-23 13:04:54 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
},
|
|
|
|
)
|
|
|
|
|
satellite/payments: chore to migrate big.Float values out of db
All code on known satellites at this moment in time should know how to
populate and use the new numeric columns on the
stripecoinpayments_tx_conversion_rates and coinpayments_transactions
tables in the satellite db. However, there are still gob-encoded
big.Float values in the database from before these columns existed. To
get rid of those values, so that we can excise the gob-decoding code
from the relevant sections, however, we need something to read the gob
bytestrings and convert them to numeric values, a few at a time, until
they're all gone.
To accomplish that, this change adds two chores to be run in the
satellite core process- one for the coinpayments_transactions table, and
one for the stripecoinpayments_tx_conversion_rates table. They should
run relatively infrequently, so that we do not impose any undue load on
processing resources or the db.
Both of these chores work without using explicit sql transactions, but
should still be concurrent-safe, since they work by way of
compare-and-swap type operations.
If the satellite core process needs to be restarted, both of these
chores will start scanning for migrateable rows from the beginning of
the id space again. This is not ideal, but shouldn't be a problem (as
far as I can tell, there are only a few thousand rows at most in either
of these tables on any production satellite).
Change-Id: I733b7cd96760d506a1cf52735f598c6c3aa19735
2021-10-29 14:32:59 +01:00
|
|
|
var transactionMigrationNextRange string
|
|
|
|
var transactionMigrationDone bool
|
|
|
|
chore.TransactionMigrationCycle.Start(ctx, &group,
|
|
|
|
func(ctx context.Context) (err error) {
|
|
|
|
if transactionMigrationDone {
|
|
|
|
mon.Event("coinpayments_transactions_gob_encoded_big_float_migration_done")
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
var migrated int
|
|
|
|
migrated, transactionMigrationNextRange, err = chore.service.db.Transactions().MigrateGobFloatTransactionRecords(ctx, transactionMigrationNextRange, chore.migrationBatchSize)
|
|
|
|
mon.Meter("coinpayments_transactions_gob_encoded_big_float_rows_migrated").Mark(migrated)
|
|
|
|
if transactionMigrationNextRange == "" {
|
|
|
|
transactionMigrationDone = true
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
if !errs2.IsCanceled(err) {
|
|
|
|
chore.log.Error("gob-encoded big.Float transaction migration chore failed", zap.Error(ErrChore.Wrap(err)))
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
},
|
|
|
|
)
|
|
|
|
|
|
|
|
var conversionRateMigrationNextRange string
|
|
|
|
var conversionRateMigrationDone bool
|
|
|
|
chore.ConversionRateMigrationCycle.Start(ctx, &group,
|
|
|
|
func(ctx context.Context) (err error) {
|
|
|
|
if conversionRateMigrationDone {
|
|
|
|
mon.Event("stripecoinpayments_tx_conversion_rates_gob_encoded_big_float_migration_done")
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
var migrated int
|
|
|
|
migrated, conversionRateMigrationNextRange, err = chore.service.db.Transactions().MigrateGobFloatConversionRateRecords(ctx, conversionRateMigrationNextRange, chore.migrationBatchSize)
|
|
|
|
mon.Meter("stripecoinpayments_tx_conversion_rates_gob_encoded_big_float_rows_migrated").Mark(migrated)
|
|
|
|
if conversionRateMigrationNextRange == "" {
|
|
|
|
conversionRateMigrationDone = true
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
if !errs2.IsCanceled(err) {
|
|
|
|
chore.log.Error("gob-encoded big.Float conversion rate migration chore failed", zap.Error(ErrChore.Wrap(err)))
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
},
|
|
|
|
)
|
|
|
|
|
2019-10-29 16:04:34 +00:00
|
|
|
return ErrChore.Wrap(group.Wait())
|
2019-10-23 13:04:54 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// Close closes all underlying resources.
|
2019-10-29 16:04:34 +00:00
|
|
|
func (chore *Chore) Close() (err error) {
|
2019-10-23 13:04:54 +01:00
|
|
|
defer mon.Task()(nil)(&err)
|
|
|
|
|
2019-10-29 16:04:34 +00:00
|
|
|
chore.TransactionCycle.Close()
|
|
|
|
chore.AccountBalanceCycle.Close()
|
satellite/payments: chore to migrate big.Float values out of db
All code on known satellites at this moment in time should know how to
populate and use the new numeric columns on the
stripecoinpayments_tx_conversion_rates and coinpayments_transactions
tables in the satellite db. However, there are still gob-encoded
big.Float values in the database from before these columns existed. To
get rid of those values, so that we can excise the gob-decoding code
from the relevant sections, however, we need something to read the gob
bytestrings and convert them to numeric values, a few at a time, until
they're all gone.
To accomplish that, this change adds two chores to be run in the
satellite core process- one for the coinpayments_transactions table, and
one for the stripecoinpayments_tx_conversion_rates table. They should
run relatively infrequently, so that we do not impose any undue load on
processing resources or the db.
Both of these chores work without using explicit sql transactions, but
should still be concurrent-safe, since they work by way of
compare-and-swap type operations.
If the satellite core process needs to be restarted, both of these
chores will start scanning for migrateable rows from the beginning of
the id space again. This is not ideal, but shouldn't be a problem (as
far as I can tell, there are only a few thousand rows at most in either
of these tables on any production satellite).
Change-Id: I733b7cd96760d506a1cf52735f598c6c3aa19735
2021-10-29 14:32:59 +01:00
|
|
|
chore.TransactionMigrationCycle.Close()
|
|
|
|
chore.ConversionRateMigrationCycle.Close()
|
2019-10-23 13:04:54 +01:00
|
|
|
return nil
|
|
|
|
}
|