diff --git a/cmd/tools/generate-missing-project-salt/main.go b/cmd/tools/generate-missing-project-salt/main.go deleted file mode 100644 index f10e72f29..000000000 --- a/cmd/tools/generate-missing-project-salt/main.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (C) 2023 Storj Labs, Inc. -// See LICENSE for copying information. - -package main - -import ( - "context" - "errors" - - pgx "github.com/jackc/pgx/v4" - "github.com/spacemonkeygo/monkit/v3" - "github.com/spf13/cobra" - flag "github.com/spf13/pflag" - "github.com/zeebo/errs" - "go.uber.org/zap" - - "storj.io/private/process" -) - -var mon = monkit.Package() - -var ( - rootCmd = &cobra.Command{ - Use: "generate-missing-project-salt", - Short: "generate-missing-project-salt", - } - - runCmd = &cobra.Command{ - Use: "run", - Short: "run generate-missing-project-salt", - RunE: run, - } - - config Config -) - -func init() { - rootCmd.AddCommand(runCmd) - - config.BindFlags(runCmd.Flags()) -} - -// Config defines configuration for migration. -type Config struct { - SatelliteDB string - Limit int - MaxUpdates int -} - -// BindFlags adds bench flags to the the flagset. -func (config *Config) BindFlags(flag *flag.FlagSet) { - flag.StringVar(&config.SatelliteDB, "satellitedb", "", "connection URL for satelliteDB") - flag.IntVar(&config.Limit, "limit", 1000, "number of updates to perform at once") - flag.IntVar(&config.MaxUpdates, "max-updates", 0, "max number of updates to perform on each table") -} - -// VerifyFlags verifies whether the values provided are valid. -func (config *Config) VerifyFlags() error { - var errlist errs.Group - if config.SatelliteDB == "" { - errlist.Add(errors.New("flag '--satellitedb' is not set")) - } - return errlist.Err() -} - -func run(cmd *cobra.Command, args []string) error { - if err := config.VerifyFlags(); err != nil { - return err - } - - ctx, _ := process.Ctx(cmd) - log := zap.L() - return GenerateMissingProjectSalt(ctx, log, config) -} - -func main() { - process.Exec(rootCmd) -} - -// GenerateMissingProjectSalt updates projects salt column where salt is null. -func GenerateMissingProjectSalt(ctx context.Context, log *zap.Logger, config Config) (err error) { - defer mon.Task()(&ctx)(&err) - - conn, err := pgx.Connect(ctx, config.SatelliteDB) - if err != nil { - return errs.New("unable to connect %q: %w", config.SatelliteDB, err) - } - defer func() { - err = errs.Combine(err, conn.Close(ctx)) - }() - - return GenerateMissingSalt(ctx, log, conn, config) -} diff --git a/cmd/tools/generate-missing-project-salt/main_test.go b/cmd/tools/generate-missing-project-salt/main_test.go deleted file mode 100644 index 7386ae587..000000000 --- a/cmd/tools/generate-missing-project-salt/main_test.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (C) 2023 Storj Labs, Inc. -// See LICENSE for copying information. - -package main_test - -import ( - "bytes" - "context" - "crypto/sha256" - "strings" - "testing" - - pgx "github.com/jackc/pgx/v4" - "github.com/stretchr/testify/require" - "go.uber.org/zap" - "go.uber.org/zap/zaptest" - - "storj.io/common/testcontext" - "storj.io/common/testrand" - "storj.io/common/uuid" - "storj.io/private/dbutil" - "storj.io/private/dbutil/tempdb" - migrator "storj.io/storj/cmd/tools/generate-missing-project-salt" - "storj.io/storj/satellite" - "storj.io/storj/satellite/console" - "storj.io/storj/satellite/satellitedb/satellitedbtest" -) - -// Test salt column is updated correctly. -func TestGenerateMissingSaltTest(t *testing.T) { - t.Parallel() - - prepare := func(t *testing.T, ctx *testcontext.Context, rawDB *dbutil.TempDatabase, db satellite.DB, conn *pgx.Conn, log *zap.Logger) (projectsIDs []uuid.UUID, shouldUpdate int) { - myProject1, err := db.Console().Projects().Insert(ctx, &console.Project{ - Name: "test1", - Description: "test1", - OwnerID: testrand.UUID(), - }) - require.NoError(t, err) - projectsIDs = append(projectsIDs, myProject1.ID) - err = db.Console().Projects().TestNullifySalt(ctx, myProject1.ID) - require.NoError(t, err) - salt1, err := db.Console().Projects().TestGetSalt(ctx, myProject1.ID) - require.NoError(t, err) - require.Equal(t, len(salt1), 0) - shouldUpdate++ - - // Project "test2" should have a populated salt column and should not get updated. - myProject2, err := db.Console().Projects().Insert(ctx, &console.Project{ - Name: "test2", - Description: "test2", - OwnerID: testrand.UUID(), - }) - require.NoError(t, err) - projectsIDs = append(projectsIDs, myProject2.ID) - salt2, err := db.Console().Projects().TestGetSalt(ctx, myProject2.ID) - require.NoError(t, err) - require.NotNil(t, salt2) - - return projectsIDs, shouldUpdate - } - - check := func(t *testing.T, ctx context.Context, db satellite.DB, projectsIDs []uuid.UUID, shouldUpdate int) { - var updated int - var notUpdated int - - for _, p := range projectsIDs { - saltdb, err := db.Console().Projects().TestGetSalt(ctx, p) - require.NoError(t, err) - idHash := sha256.Sum256(p[:]) - salt := idHash[:] - // if the salt column is the hashed project ID, it means we migrated that row - if bytes.Equal(salt, saltdb) { - updated++ - } else { - notUpdated++ - } - } - require.Equal(t, shouldUpdate, updated) - require.Equal(t, len(projectsIDs)-shouldUpdate, notUpdated) - } - - test(t, prepare, migrator.GenerateMissingSalt, check, &migrator.Config{ - Limit: 2, - }) -} - -func test(t *testing.T, - prepare func(t *testing.T, ctx *testcontext.Context, rawDB *dbutil.TempDatabase, db satellite.DB, conn *pgx.Conn, log *zap.Logger) (projectsIDs []uuid.UUID, shouldUpdate int), - migrate func(ctx context.Context, log *zap.Logger, conn *pgx.Conn, config migrator.Config) (err error), - check func(t *testing.T, ctx context.Context, db satellite.DB, projectsIDs []uuid.UUID, shouldUpdate int), config *migrator.Config) { - - log := zaptest.NewLogger(t) - - for _, satelliteDB := range satellitedbtest.Databases() { - satelliteDB := satelliteDB - t.Run(satelliteDB.Name, func(t *testing.T) { - t.Parallel() - ctx := testcontext.New(t) - - schemaSuffix := satellitedbtest.SchemaSuffix() - schema := satellitedbtest.SchemaName(t.Name(), "category", 0, schemaSuffix) - - tempDB, err := tempdb.OpenUnique(ctx, satelliteDB.MasterDB.URL, schema) - require.NoError(t, err) - - db, err := satellitedbtest.CreateMasterDBOnTopOf(ctx, log, tempDB, "generate-missing-project-salt") - require.NoError(t, err) - defer ctx.Check(db.Close) - - err = db.Testing().TestMigrateToLatest(ctx) - require.NoError(t, err) - - mConnStr := strings.Replace(tempDB.ConnStr, "cockroach", "postgres", 1) - - conn, err := pgx.Connect(ctx, mConnStr) - require.NoError(t, err) - defer func() { - require.NoError(t, conn.Close(ctx)) - }() - - projectsIDs, shouldUpdate := prepare(t, ctx, tempDB, db, conn, log) - - err = migrate(ctx, log, conn, *config) - require.NoError(t, err) - - check(t, ctx, db, projectsIDs, shouldUpdate) - }) - } -} diff --git a/cmd/tools/generate-missing-project-salt/migrations.go b/cmd/tools/generate-missing-project-salt/migrations.go deleted file mode 100644 index 6d67bb2a4..000000000 --- a/cmd/tools/generate-missing-project-salt/migrations.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (C) 2023 Storj Labs, Inc. -// See LICENSE for copying information. - -package main - -import ( - "context" - "crypto/sha256" - - pgx "github.com/jackc/pgx/v4" - "github.com/zeebo/errs" - "go.uber.org/zap" - - "storj.io/private/dbutil/cockroachutil" - "storj.io/private/dbutil/pgutil" -) - -// GenerateMissingSalt updates the salt column in projects table to add SHA-256 hash of the project ID where salt is NULL. -func GenerateMissingSalt(ctx context.Context, log *zap.Logger, conn *pgx.Conn, config Config) (err error) { - defer mon.Task()(&ctx)(&err) - - lastID := []byte{} - var total int - var rowsFound bool - - for { - rowsFound = false - idsToUpdate := struct { - ids [][]byte - salt [][]byte - }{} - err = func() error { - rows, err := conn.Query(ctx, ` - SELECT id FROM projects - WHERE salt IS NULL - AND id > $1 - ORDER BY id - LIMIT $2 - `, lastID, config.Limit) - if err != nil { - return errs.New("error select ids for update: %w", err) - } - defer rows.Close() - - for rows.Next() { - rowsFound = true - var id []byte - err = rows.Scan(&id) - if err != nil { - return errs.New("error scanning results from select: %w", err) - } - - idHash := sha256.Sum256(id) - salt := idHash[:] - idsToUpdate.salt = append(idsToUpdate.salt, salt) - idsToUpdate.ids = append(idsToUpdate.ids, id) - lastID = id - } - - return rows.Err() - }() - if err != nil { - return err - } - if !rowsFound { - break - } - - var updated int - for { - row := conn.QueryRow(ctx, - `WITH to_update AS ( - SELECT unnest($1::bytea[]) as id, - unnest($2::bytea[]) as salt - ), - updated as ( - UPDATE projects - SET salt = to_update.salt - FROM to_update - WHERE projects.id = to_update.id - RETURNING 1 - ) - SELECT count(*) - FROM updated; - `, pgutil.ByteaArray(idsToUpdate.ids), pgutil.ByteaArray(idsToUpdate.salt)) - err := row.Scan(&updated) - if err != nil { - if cockroachutil.NeedsRetry(err) { - continue - } else if errs.Is(err, pgx.ErrNoRows) { - break - } - return errs.New("error updating projects %w", err) - } - break - } - total += updated - log.Info("batch update complete", zap.Int("rows updated", updated), zap.Binary("last id", lastID)) - } - log.Info("projects migration complete", zap.Int("total rows updated", total)) - return nil -} diff --git a/satellite/console/projects.go b/satellite/console/projects.go index 32262df0c..b756420d9 100644 --- a/satellite/console/projects.go +++ b/satellite/console/projects.go @@ -54,13 +54,6 @@ type Projects interface { // UpdateUsageLimits is a method for updating project's usage limits. UpdateUsageLimits(ctx context.Context, id uuid.UUID, limits UsageLimits) error - - // TestGetSalt returns the value (whether null or hashed project ID) from project's salt column in db. - TestGetSalt(ctx context.Context, id uuid.UUID) ([]byte, error) - - // TestNullifySalt is a temporary method for nullifying the salt column - // for testing a migration tool (TODO lizzy delete after migration). - TestNullifySalt(ctx context.Context, id uuid.UUID) error } // UsageLimitsConfig is a configuration struct for default per-project usage limits. diff --git a/satellite/satellitedb/projects.go b/satellite/satellitedb/projects.go index db1372aca..36a3d0477 100644 --- a/satellite/satellitedb/projects.go +++ b/satellite/satellitedb/projects.go @@ -103,20 +103,6 @@ func (projects *projects) GetSalt(ctx context.Context, id uuid.UUID) (salt []byt return salt, nil } -// TestGetSalt returns the project's salt column value from the db. -func (projects *projects) TestGetSalt(ctx context.Context, id uuid.UUID) (salt []byte, err error) { - defer mon.Task()(&ctx)(&err) - - res, err := projects.db.Get_Project_Salt_By_Id(ctx, dbx.Project_Id(id[:])) - if err != nil { - return nil, err - } - - salt = res.Salt - - return salt, nil -} - // GetByPublicID is a method for querying project from the database by public_id. func (projects *projects) GetByPublicID(ctx context.Context, publicID uuid.UUID) (_ *console.Project, err error) { defer mon.Task()(&ctx)(&err) @@ -472,18 +458,3 @@ func (projects *projects) UpdateUsageLimits(ctx context.Context, id uuid.UUID, l ) return err } - -// TestNullifySalt is a temporary method for nullifying the salt column -// for testing a migration tool (TODO lizzy delete after migration). -func (projects *projects) TestNullifySalt(ctx context.Context, id uuid.UUID) (err error) { - defer mon.Task()(&ctx)(&err) - - var salt []byte - row := projects.sdb.QueryRow(ctx, `UPDATE projects SET salt = NULL WHERE id = $1 RETURNING salt`, id) - err = row.Scan(&salt) - if err != nil { - return errs.New("error scanning results: %w", err) - } - - return err -}