cmd/tools: remove migration tool code for generating missing salt
Remove generate-missing-project-salt migration tool code and related tests. This migration has already been run and this code is no longer needed. Issue https://github.com/storj/storj-private/issues/163 Change-Id: I4e36dcd95a07c5305c597113a7fd08148e100ccc
This commit is contained in:
Lizzy Thomson
Storj Robot
parent
0559d19db5
commit
7e2d98988b
@ -1,93 +0,0 @@
|
||||
// Copyright (C) 2023 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
pgx "github.com/jackc/pgx/v4"
|
||||
"github.com/spacemonkeygo/monkit/v3"
|
||||
"github.com/spf13/cobra"
|
||||
flag "github.com/spf13/pflag"
|
||||
"github.com/zeebo/errs"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"storj.io/private/process"
|
||||
)
|
||||
|
||||
var mon = monkit.Package()
|
||||
|
||||
var (
|
||||
rootCmd = &cobra.Command{
|
||||
Use: "generate-missing-project-salt",
|
||||
Short: "generate-missing-project-salt",
|
||||
}
|
||||
|
||||
runCmd = &cobra.Command{
|
||||
Use: "run",
|
||||
Short: "run generate-missing-project-salt",
|
||||
RunE: run,
|
||||
}
|
||||
|
||||
config Config
|
||||
)
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(runCmd)
|
||||
|
||||
config.BindFlags(runCmd.Flags())
|
||||
}
|
||||
|
||||
// Config defines configuration for migration.
|
||||
type Config struct {
|
||||
SatelliteDB string
|
||||
Limit int
|
||||
MaxUpdates int
|
||||
}
|
||||
|
||||
// BindFlags adds bench flags to the the flagset.
|
||||
func (config *Config) BindFlags(flag *flag.FlagSet) {
|
||||
flag.StringVar(&config.SatelliteDB, "satellitedb", "", "connection URL for satelliteDB")
|
||||
flag.IntVar(&config.Limit, "limit", 1000, "number of updates to perform at once")
|
||||
flag.IntVar(&config.MaxUpdates, "max-updates", 0, "max number of updates to perform on each table")
|
||||
}
|
||||
|
||||
// VerifyFlags verifies whether the values provided are valid.
|
||||
func (config *Config) VerifyFlags() error {
|
||||
var errlist errs.Group
|
||||
if config.SatelliteDB == "" {
|
||||
errlist.Add(errors.New("flag '--satellitedb' is not set"))
|
||||
}
|
||||
return errlist.Err()
|
||||
}
|
||||
|
||||
func run(cmd *cobra.Command, args []string) error {
|
||||
if err := config.VerifyFlags(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, _ := process.Ctx(cmd)
|
||||
log := zap.L()
|
||||
return GenerateMissingProjectSalt(ctx, log, config)
|
||||
}
|
||||
|
||||
func main() {
|
||||
process.Exec(rootCmd)
|
||||
}
|
||||
|
||||
// GenerateMissingProjectSalt updates projects salt column where salt is null.
|
||||
func GenerateMissingProjectSalt(ctx context.Context, log *zap.Logger, config Config) (err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
|
||||
conn, err := pgx.Connect(ctx, config.SatelliteDB)
|
||||
if err != nil {
|
||||
return errs.New("unable to connect %q: %w", config.SatelliteDB, err)
|
||||
}
|
||||
defer func() {
|
||||
err = errs.Combine(err, conn.Close(ctx))
|
||||
}()
|
||||
|
||||
return GenerateMissingSalt(ctx, log, conn, config)
|
||||
}
|
||||
@ -1,130 +0,0 @@
|
||||
// Copyright (C) 2023 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package main_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
pgx "github.com/jackc/pgx/v4"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zaptest"
|
||||
|
||||
"storj.io/common/testcontext"
|
||||
"storj.io/common/testrand"
|
||||
"storj.io/common/uuid"
|
||||
"storj.io/private/dbutil"
|
||||
"storj.io/private/dbutil/tempdb"
|
||||
migrator "storj.io/storj/cmd/tools/generate-missing-project-salt"
|
||||
"storj.io/storj/satellite"
|
||||
"storj.io/storj/satellite/console"
|
||||
"storj.io/storj/satellite/satellitedb/satellitedbtest"
|
||||
)
|
||||
|
||||
// Test salt column is updated correctly.
|
||||
func TestGenerateMissingSaltTest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
prepare := func(t *testing.T, ctx *testcontext.Context, rawDB *dbutil.TempDatabase, db satellite.DB, conn *pgx.Conn, log *zap.Logger) (projectsIDs []uuid.UUID, shouldUpdate int) {
|
||||
myProject1, err := db.Console().Projects().Insert(ctx, &console.Project{
|
||||
Name: "test1",
|
||||
Description: "test1",
|
||||
OwnerID: testrand.UUID(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
projectsIDs = append(projectsIDs, myProject1.ID)
|
||||
err = db.Console().Projects().TestNullifySalt(ctx, myProject1.ID)
|
||||
require.NoError(t, err)
|
||||
salt1, err := db.Console().Projects().TestGetSalt(ctx, myProject1.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, len(salt1), 0)
|
||||
shouldUpdate++
|
||||
|
||||
// Project "test2" should have a populated salt column and should not get updated.
|
||||
myProject2, err := db.Console().Projects().Insert(ctx, &console.Project{
|
||||
Name: "test2",
|
||||
Description: "test2",
|
||||
OwnerID: testrand.UUID(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
projectsIDs = append(projectsIDs, myProject2.ID)
|
||||
salt2, err := db.Console().Projects().TestGetSalt(ctx, myProject2.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, salt2)
|
||||
|
||||
return projectsIDs, shouldUpdate
|
||||
}
|
||||
|
||||
check := func(t *testing.T, ctx context.Context, db satellite.DB, projectsIDs []uuid.UUID, shouldUpdate int) {
|
||||
var updated int
|
||||
var notUpdated int
|
||||
|
||||
for _, p := range projectsIDs {
|
||||
saltdb, err := db.Console().Projects().TestGetSalt(ctx, p)
|
||||
require.NoError(t, err)
|
||||
idHash := sha256.Sum256(p[:])
|
||||
salt := idHash[:]
|
||||
// if the salt column is the hashed project ID, it means we migrated that row
|
||||
if bytes.Equal(salt, saltdb) {
|
||||
updated++
|
||||
} else {
|
||||
notUpdated++
|
||||
}
|
||||
}
|
||||
require.Equal(t, shouldUpdate, updated)
|
||||
require.Equal(t, len(projectsIDs)-shouldUpdate, notUpdated)
|
||||
}
|
||||
|
||||
test(t, prepare, migrator.GenerateMissingSalt, check, &migrator.Config{
|
||||
Limit: 2,
|
||||
})
|
||||
}
|
||||
|
||||
func test(t *testing.T,
|
||||
prepare func(t *testing.T, ctx *testcontext.Context, rawDB *dbutil.TempDatabase, db satellite.DB, conn *pgx.Conn, log *zap.Logger) (projectsIDs []uuid.UUID, shouldUpdate int),
|
||||
migrate func(ctx context.Context, log *zap.Logger, conn *pgx.Conn, config migrator.Config) (err error),
|
||||
check func(t *testing.T, ctx context.Context, db satellite.DB, projectsIDs []uuid.UUID, shouldUpdate int), config *migrator.Config) {
|
||||
|
||||
log := zaptest.NewLogger(t)
|
||||
|
||||
for _, satelliteDB := range satellitedbtest.Databases() {
|
||||
satelliteDB := satelliteDB
|
||||
t.Run(satelliteDB.Name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := testcontext.New(t)
|
||||
|
||||
schemaSuffix := satellitedbtest.SchemaSuffix()
|
||||
schema := satellitedbtest.SchemaName(t.Name(), "category", 0, schemaSuffix)
|
||||
|
||||
tempDB, err := tempdb.OpenUnique(ctx, satelliteDB.MasterDB.URL, schema)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := satellitedbtest.CreateMasterDBOnTopOf(ctx, log, tempDB, "generate-missing-project-salt")
|
||||
require.NoError(t, err)
|
||||
defer ctx.Check(db.Close)
|
||||
|
||||
err = db.Testing().TestMigrateToLatest(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
mConnStr := strings.Replace(tempDB.ConnStr, "cockroach", "postgres", 1)
|
||||
|
||||
conn, err := pgx.Connect(ctx, mConnStr)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
require.NoError(t, conn.Close(ctx))
|
||||
}()
|
||||
|
||||
projectsIDs, shouldUpdate := prepare(t, ctx, tempDB, db, conn, log)
|
||||
|
||||
err = migrate(ctx, log, conn, *config)
|
||||
require.NoError(t, err)
|
||||
|
||||
check(t, ctx, db, projectsIDs, shouldUpdate)
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -1,102 +0,0 @@
|
||||
// Copyright (C) 2023 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
|
||||
pgx "github.com/jackc/pgx/v4"
|
||||
"github.com/zeebo/errs"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"storj.io/private/dbutil/cockroachutil"
|
||||
"storj.io/private/dbutil/pgutil"
|
||||
)
|
||||
|
||||
// GenerateMissingSalt updates the salt column in projects table to add SHA-256 hash of the project ID where salt is NULL.
|
||||
func GenerateMissingSalt(ctx context.Context, log *zap.Logger, conn *pgx.Conn, config Config) (err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
|
||||
lastID := []byte{}
|
||||
var total int
|
||||
var rowsFound bool
|
||||
|
||||
for {
|
||||
rowsFound = false
|
||||
idsToUpdate := struct {
|
||||
ids [][]byte
|
||||
salt [][]byte
|
||||
}{}
|
||||
err = func() error {
|
||||
rows, err := conn.Query(ctx, `
|
||||
SELECT id FROM projects
|
||||
WHERE salt IS NULL
|
||||
AND id > $1
|
||||
ORDER BY id
|
||||
LIMIT $2
|
||||
`, lastID, config.Limit)
|
||||
if err != nil {
|
||||
return errs.New("error select ids for update: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
rowsFound = true
|
||||
var id []byte
|
||||
err = rows.Scan(&id)
|
||||
if err != nil {
|
||||
return errs.New("error scanning results from select: %w", err)
|
||||
}
|
||||
|
||||
idHash := sha256.Sum256(id)
|
||||
salt := idHash[:]
|
||||
idsToUpdate.salt = append(idsToUpdate.salt, salt)
|
||||
idsToUpdate.ids = append(idsToUpdate.ids, id)
|
||||
lastID = id
|
||||
}
|
||||
|
||||
return rows.Err()
|
||||
}()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !rowsFound {
|
||||
break
|
||||
}
|
||||
|
||||
var updated int
|
||||
for {
|
||||
row := conn.QueryRow(ctx,
|
||||
`WITH to_update AS (
|
||||
SELECT unnest($1::bytea[]) as id,
|
||||
unnest($2::bytea[]) as salt
|
||||
),
|
||||
updated as (
|
||||
UPDATE projects
|
||||
SET salt = to_update.salt
|
||||
FROM to_update
|
||||
WHERE projects.id = to_update.id
|
||||
RETURNING 1
|
||||
)
|
||||
SELECT count(*)
|
||||
FROM updated;
|
||||
`, pgutil.ByteaArray(idsToUpdate.ids), pgutil.ByteaArray(idsToUpdate.salt))
|
||||
err := row.Scan(&updated)
|
||||
if err != nil {
|
||||
if cockroachutil.NeedsRetry(err) {
|
||||
continue
|
||||
} else if errs.Is(err, pgx.ErrNoRows) {
|
||||
break
|
||||
}
|
||||
return errs.New("error updating projects %w", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
total += updated
|
||||
log.Info("batch update complete", zap.Int("rows updated", updated), zap.Binary("last id", lastID))
|
||||
}
|
||||
log.Info("projects migration complete", zap.Int("total rows updated", total))
|
||||
return nil
|
||||
}
|
||||
@ -54,13 +54,6 @@ type Projects interface {
|
||||
|
||||
// UpdateUsageLimits is a method for updating project's usage limits.
|
||||
UpdateUsageLimits(ctx context.Context, id uuid.UUID, limits UsageLimits) error
|
||||
|
||||
// TestGetSalt returns the value (whether null or hashed project ID) from project's salt column in db.
|
||||
TestGetSalt(ctx context.Context, id uuid.UUID) ([]byte, error)
|
||||
|
||||
// TestNullifySalt is a temporary method for nullifying the salt column
|
||||
// for testing a migration tool (TODO lizzy delete after migration).
|
||||
TestNullifySalt(ctx context.Context, id uuid.UUID) error
|
||||
}
|
||||
|
||||
// UsageLimitsConfig is a configuration struct for default per-project usage limits.
|
||||
|
||||
@ -103,20 +103,6 @@ func (projects *projects) GetSalt(ctx context.Context, id uuid.UUID) (salt []byt
|
||||
return salt, nil
|
||||
}
|
||||
|
||||
// TestGetSalt returns the project's salt column value from the db.
|
||||
func (projects *projects) TestGetSalt(ctx context.Context, id uuid.UUID) (salt []byte, err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
|
||||
res, err := projects.db.Get_Project_Salt_By_Id(ctx, dbx.Project_Id(id[:]))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
salt = res.Salt
|
||||
|
||||
return salt, nil
|
||||
}
|
||||
|
||||
// GetByPublicID is a method for querying project from the database by public_id.
|
||||
func (projects *projects) GetByPublicID(ctx context.Context, publicID uuid.UUID) (_ *console.Project, err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
@ -472,18 +458,3 @@ func (projects *projects) UpdateUsageLimits(ctx context.Context, id uuid.UUID, l
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// TestNullifySalt is a temporary method for nullifying the salt column
|
||||
// for testing a migration tool (TODO lizzy delete after migration).
|
||||
func (projects *projects) TestNullifySalt(ctx context.Context, id uuid.UUID) (err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
|
||||
var salt []byte
|
||||
row := projects.sdb.QueryRow(ctx, `UPDATE projects SET salt = NULL WHERE id = $1 RETURNING salt`, id)
|
||||
err = row.Scan(&salt)
|
||||
if err != nil {
|
||||
return errs.New("error scanning results: %w", err)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user