cmd/tools: remove migration tool code for generating missing salt
Remove generate-missing-project-salt migration tool code and related tests. This migration has already been run and this code is no longer needed. Issue https://github.com/storj/storj-private/issues/163 Change-Id: I4e36dcd95a07c5305c597113a7fd08148e100ccc
This commit is contained in:
Lizzy Thomson
Storj Robot
parent
0559d19db5
commit
7e2d98988b
@ -1,93 +0,0 @@
|
|||||||
// Copyright (C) 2023 Storj Labs, Inc.
|
|
||||||
// See LICENSE for copying information.
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
|
|
||||||
pgx "github.com/jackc/pgx/v4"
|
|
||||||
"github.com/spacemonkeygo/monkit/v3"
|
|
||||||
"github.com/spf13/cobra"
|
|
||||||
flag "github.com/spf13/pflag"
|
|
||||||
"github.com/zeebo/errs"
|
|
||||||
"go.uber.org/zap"
|
|
||||||
|
|
||||||
"storj.io/private/process"
|
|
||||||
)
|
|
||||||
|
|
||||||
var mon = monkit.Package()
|
|
||||||
|
|
||||||
var (
|
|
||||||
rootCmd = &cobra.Command{
|
|
||||||
Use: "generate-missing-project-salt",
|
|
||||||
Short: "generate-missing-project-salt",
|
|
||||||
}
|
|
||||||
|
|
||||||
runCmd = &cobra.Command{
|
|
||||||
Use: "run",
|
|
||||||
Short: "run generate-missing-project-salt",
|
|
||||||
RunE: run,
|
|
||||||
}
|
|
||||||
|
|
||||||
config Config
|
|
||||||
)
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
rootCmd.AddCommand(runCmd)
|
|
||||||
|
|
||||||
config.BindFlags(runCmd.Flags())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Config defines configuration for migration.
|
|
||||||
type Config struct {
|
|
||||||
SatelliteDB string
|
|
||||||
Limit int
|
|
||||||
MaxUpdates int
|
|
||||||
}
|
|
||||||
|
|
||||||
// BindFlags adds bench flags to the the flagset.
|
|
||||||
func (config *Config) BindFlags(flag *flag.FlagSet) {
|
|
||||||
flag.StringVar(&config.SatelliteDB, "satellitedb", "", "connection URL for satelliteDB")
|
|
||||||
flag.IntVar(&config.Limit, "limit", 1000, "number of updates to perform at once")
|
|
||||||
flag.IntVar(&config.MaxUpdates, "max-updates", 0, "max number of updates to perform on each table")
|
|
||||||
}
|
|
||||||
|
|
||||||
// VerifyFlags verifies whether the values provided are valid.
|
|
||||||
func (config *Config) VerifyFlags() error {
|
|
||||||
var errlist errs.Group
|
|
||||||
if config.SatelliteDB == "" {
|
|
||||||
errlist.Add(errors.New("flag '--satellitedb' is not set"))
|
|
||||||
}
|
|
||||||
return errlist.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
func run(cmd *cobra.Command, args []string) error {
|
|
||||||
if err := config.VerifyFlags(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, _ := process.Ctx(cmd)
|
|
||||||
log := zap.L()
|
|
||||||
return GenerateMissingProjectSalt(ctx, log, config)
|
|
||||||
}
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
process.Exec(rootCmd)
|
|
||||||
}
|
|
||||||
|
|
||||||
// GenerateMissingProjectSalt updates projects salt column where salt is null.
|
|
||||||
func GenerateMissingProjectSalt(ctx context.Context, log *zap.Logger, config Config) (err error) {
|
|
||||||
defer mon.Task()(&ctx)(&err)
|
|
||||||
|
|
||||||
conn, err := pgx.Connect(ctx, config.SatelliteDB)
|
|
||||||
if err != nil {
|
|
||||||
return errs.New("unable to connect %q: %w", config.SatelliteDB, err)
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
err = errs.Combine(err, conn.Close(ctx))
|
|
||||||
}()
|
|
||||||
|
|
||||||
return GenerateMissingSalt(ctx, log, conn, config)
|
|
||||||
}
|
|
||||||
@ -1,130 +0,0 @@
|
|||||||
// Copyright (C) 2023 Storj Labs, Inc.
|
|
||||||
// See LICENSE for copying information.
|
|
||||||
|
|
||||||
package main_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
pgx "github.com/jackc/pgx/v4"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"go.uber.org/zap"
|
|
||||||
"go.uber.org/zap/zaptest"
|
|
||||||
|
|
||||||
"storj.io/common/testcontext"
|
|
||||||
"storj.io/common/testrand"
|
|
||||||
"storj.io/common/uuid"
|
|
||||||
"storj.io/private/dbutil"
|
|
||||||
"storj.io/private/dbutil/tempdb"
|
|
||||||
migrator "storj.io/storj/cmd/tools/generate-missing-project-salt"
|
|
||||||
"storj.io/storj/satellite"
|
|
||||||
"storj.io/storj/satellite/console"
|
|
||||||
"storj.io/storj/satellite/satellitedb/satellitedbtest"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Test salt column is updated correctly.
|
|
||||||
func TestGenerateMissingSaltTest(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
prepare := func(t *testing.T, ctx *testcontext.Context, rawDB *dbutil.TempDatabase, db satellite.DB, conn *pgx.Conn, log *zap.Logger) (projectsIDs []uuid.UUID, shouldUpdate int) {
|
|
||||||
myProject1, err := db.Console().Projects().Insert(ctx, &console.Project{
|
|
||||||
Name: "test1",
|
|
||||||
Description: "test1",
|
|
||||||
OwnerID: testrand.UUID(),
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
projectsIDs = append(projectsIDs, myProject1.ID)
|
|
||||||
err = db.Console().Projects().TestNullifySalt(ctx, myProject1.ID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
salt1, err := db.Console().Projects().TestGetSalt(ctx, myProject1.ID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, len(salt1), 0)
|
|
||||||
shouldUpdate++
|
|
||||||
|
|
||||||
// Project "test2" should have a populated salt column and should not get updated.
|
|
||||||
myProject2, err := db.Console().Projects().Insert(ctx, &console.Project{
|
|
||||||
Name: "test2",
|
|
||||||
Description: "test2",
|
|
||||||
OwnerID: testrand.UUID(),
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
projectsIDs = append(projectsIDs, myProject2.ID)
|
|
||||||
salt2, err := db.Console().Projects().TestGetSalt(ctx, myProject2.ID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, salt2)
|
|
||||||
|
|
||||||
return projectsIDs, shouldUpdate
|
|
||||||
}
|
|
||||||
|
|
||||||
check := func(t *testing.T, ctx context.Context, db satellite.DB, projectsIDs []uuid.UUID, shouldUpdate int) {
|
|
||||||
var updated int
|
|
||||||
var notUpdated int
|
|
||||||
|
|
||||||
for _, p := range projectsIDs {
|
|
||||||
saltdb, err := db.Console().Projects().TestGetSalt(ctx, p)
|
|
||||||
require.NoError(t, err)
|
|
||||||
idHash := sha256.Sum256(p[:])
|
|
||||||
salt := idHash[:]
|
|
||||||
// if the salt column is the hashed project ID, it means we migrated that row
|
|
||||||
if bytes.Equal(salt, saltdb) {
|
|
||||||
updated++
|
|
||||||
} else {
|
|
||||||
notUpdated++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
require.Equal(t, shouldUpdate, updated)
|
|
||||||
require.Equal(t, len(projectsIDs)-shouldUpdate, notUpdated)
|
|
||||||
}
|
|
||||||
|
|
||||||
test(t, prepare, migrator.GenerateMissingSalt, check, &migrator.Config{
|
|
||||||
Limit: 2,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func test(t *testing.T,
|
|
||||||
prepare func(t *testing.T, ctx *testcontext.Context, rawDB *dbutil.TempDatabase, db satellite.DB, conn *pgx.Conn, log *zap.Logger) (projectsIDs []uuid.UUID, shouldUpdate int),
|
|
||||||
migrate func(ctx context.Context, log *zap.Logger, conn *pgx.Conn, config migrator.Config) (err error),
|
|
||||||
check func(t *testing.T, ctx context.Context, db satellite.DB, projectsIDs []uuid.UUID, shouldUpdate int), config *migrator.Config) {
|
|
||||||
|
|
||||||
log := zaptest.NewLogger(t)
|
|
||||||
|
|
||||||
for _, satelliteDB := range satellitedbtest.Databases() {
|
|
||||||
satelliteDB := satelliteDB
|
|
||||||
t.Run(satelliteDB.Name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
ctx := testcontext.New(t)
|
|
||||||
|
|
||||||
schemaSuffix := satellitedbtest.SchemaSuffix()
|
|
||||||
schema := satellitedbtest.SchemaName(t.Name(), "category", 0, schemaSuffix)
|
|
||||||
|
|
||||||
tempDB, err := tempdb.OpenUnique(ctx, satelliteDB.MasterDB.URL, schema)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
db, err := satellitedbtest.CreateMasterDBOnTopOf(ctx, log, tempDB, "generate-missing-project-salt")
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer ctx.Check(db.Close)
|
|
||||||
|
|
||||||
err = db.Testing().TestMigrateToLatest(ctx)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
mConnStr := strings.Replace(tempDB.ConnStr, "cockroach", "postgres", 1)
|
|
||||||
|
|
||||||
conn, err := pgx.Connect(ctx, mConnStr)
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer func() {
|
|
||||||
require.NoError(t, conn.Close(ctx))
|
|
||||||
}()
|
|
||||||
|
|
||||||
projectsIDs, shouldUpdate := prepare(t, ctx, tempDB, db, conn, log)
|
|
||||||
|
|
||||||
err = migrate(ctx, log, conn, *config)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
check(t, ctx, db, projectsIDs, shouldUpdate)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,102 +0,0 @@
|
|||||||
// Copyright (C) 2023 Storj Labs, Inc.
|
|
||||||
// See LICENSE for copying information.
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
|
||||||
|
|
||||||
pgx "github.com/jackc/pgx/v4"
|
|
||||||
"github.com/zeebo/errs"
|
|
||||||
"go.uber.org/zap"
|
|
||||||
|
|
||||||
"storj.io/private/dbutil/cockroachutil"
|
|
||||||
"storj.io/private/dbutil/pgutil"
|
|
||||||
)
|
|
||||||
|
|
||||||
// GenerateMissingSalt updates the salt column in projects table to add SHA-256 hash of the project ID where salt is NULL.
|
|
||||||
func GenerateMissingSalt(ctx context.Context, log *zap.Logger, conn *pgx.Conn, config Config) (err error) {
|
|
||||||
defer mon.Task()(&ctx)(&err)
|
|
||||||
|
|
||||||
lastID := []byte{}
|
|
||||||
var total int
|
|
||||||
var rowsFound bool
|
|
||||||
|
|
||||||
for {
|
|
||||||
rowsFound = false
|
|
||||||
idsToUpdate := struct {
|
|
||||||
ids [][]byte
|
|
||||||
salt [][]byte
|
|
||||||
}{}
|
|
||||||
err = func() error {
|
|
||||||
rows, err := conn.Query(ctx, `
|
|
||||||
SELECT id FROM projects
|
|
||||||
WHERE salt IS NULL
|
|
||||||
AND id > $1
|
|
||||||
ORDER BY id
|
|
||||||
LIMIT $2
|
|
||||||
`, lastID, config.Limit)
|
|
||||||
if err != nil {
|
|
||||||
return errs.New("error select ids for update: %w", err)
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
|
|
||||||
for rows.Next() {
|
|
||||||
rowsFound = true
|
|
||||||
var id []byte
|
|
||||||
err = rows.Scan(&id)
|
|
||||||
if err != nil {
|
|
||||||
return errs.New("error scanning results from select: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
idHash := sha256.Sum256(id)
|
|
||||||
salt := idHash[:]
|
|
||||||
idsToUpdate.salt = append(idsToUpdate.salt, salt)
|
|
||||||
idsToUpdate.ids = append(idsToUpdate.ids, id)
|
|
||||||
lastID = id
|
|
||||||
}
|
|
||||||
|
|
||||||
return rows.Err()
|
|
||||||
}()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !rowsFound {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
var updated int
|
|
||||||
for {
|
|
||||||
row := conn.QueryRow(ctx,
|
|
||||||
`WITH to_update AS (
|
|
||||||
SELECT unnest($1::bytea[]) as id,
|
|
||||||
unnest($2::bytea[]) as salt
|
|
||||||
),
|
|
||||||
updated as (
|
|
||||||
UPDATE projects
|
|
||||||
SET salt = to_update.salt
|
|
||||||
FROM to_update
|
|
||||||
WHERE projects.id = to_update.id
|
|
||||||
RETURNING 1
|
|
||||||
)
|
|
||||||
SELECT count(*)
|
|
||||||
FROM updated;
|
|
||||||
`, pgutil.ByteaArray(idsToUpdate.ids), pgutil.ByteaArray(idsToUpdate.salt))
|
|
||||||
err := row.Scan(&updated)
|
|
||||||
if err != nil {
|
|
||||||
if cockroachutil.NeedsRetry(err) {
|
|
||||||
continue
|
|
||||||
} else if errs.Is(err, pgx.ErrNoRows) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
return errs.New("error updating projects %w", err)
|
|
||||||
}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
total += updated
|
|
||||||
log.Info("batch update complete", zap.Int("rows updated", updated), zap.Binary("last id", lastID))
|
|
||||||
}
|
|
||||||
log.Info("projects migration complete", zap.Int("total rows updated", total))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@ -54,13 +54,6 @@ type Projects interface {
|
|||||||
|
|
||||||
// UpdateUsageLimits is a method for updating project's usage limits.
|
// UpdateUsageLimits is a method for updating project's usage limits.
|
||||||
UpdateUsageLimits(ctx context.Context, id uuid.UUID, limits UsageLimits) error
|
UpdateUsageLimits(ctx context.Context, id uuid.UUID, limits UsageLimits) error
|
||||||
|
|
||||||
// TestGetSalt returns the value (whether null or hashed project ID) from project's salt column in db.
|
|
||||||
TestGetSalt(ctx context.Context, id uuid.UUID) ([]byte, error)
|
|
||||||
|
|
||||||
// TestNullifySalt is a temporary method for nullifying the salt column
|
|
||||||
// for testing a migration tool (TODO lizzy delete after migration).
|
|
||||||
TestNullifySalt(ctx context.Context, id uuid.UUID) error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// UsageLimitsConfig is a configuration struct for default per-project usage limits.
|
// UsageLimitsConfig is a configuration struct for default per-project usage limits.
|
||||||
|
|||||||
@ -103,20 +103,6 @@ func (projects *projects) GetSalt(ctx context.Context, id uuid.UUID) (salt []byt
|
|||||||
return salt, nil
|
return salt, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestGetSalt returns the project's salt column value from the db.
|
|
||||||
func (projects *projects) TestGetSalt(ctx context.Context, id uuid.UUID) (salt []byte, err error) {
|
|
||||||
defer mon.Task()(&ctx)(&err)
|
|
||||||
|
|
||||||
res, err := projects.db.Get_Project_Salt_By_Id(ctx, dbx.Project_Id(id[:]))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
salt = res.Salt
|
|
||||||
|
|
||||||
return salt, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetByPublicID is a method for querying project from the database by public_id.
|
// GetByPublicID is a method for querying project from the database by public_id.
|
||||||
func (projects *projects) GetByPublicID(ctx context.Context, publicID uuid.UUID) (_ *console.Project, err error) {
|
func (projects *projects) GetByPublicID(ctx context.Context, publicID uuid.UUID) (_ *console.Project, err error) {
|
||||||
defer mon.Task()(&ctx)(&err)
|
defer mon.Task()(&ctx)(&err)
|
||||||
@ -472,18 +458,3 @@ func (projects *projects) UpdateUsageLimits(ctx context.Context, id uuid.UUID, l
|
|||||||
)
|
)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestNullifySalt is a temporary method for nullifying the salt column
|
|
||||||
// for testing a migration tool (TODO lizzy delete after migration).
|
|
||||||
func (projects *projects) TestNullifySalt(ctx context.Context, id uuid.UUID) (err error) {
|
|
||||||
defer mon.Task()(&ctx)(&err)
|
|
||||||
|
|
||||||
var salt []byte
|
|
||||||
row := projects.sdb.QueryRow(ctx, `UPDATE projects SET salt = NULL WHERE id = $1 RETURNING salt`, id)
|
|
||||||
err = row.Scan(&salt)
|
|
||||||
if err != nil {
|
|
||||||
return errs.New("error scanning results: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user