certificate/authorization,cmd/certificates: remove gob code

Now that we've migrated data we can remove gob handling.

Change-Id: I55f772b7a0dda71c51db0683bad3db66b89867ac

certificate/authorization/authorizations.go

@@ -6,11 +6,7 @@ package authorization
 import (
 	"bytes"
 	"context"
-	"crypto/ecdsa"
-	"crypto/elliptic"
 	"crypto/rand"
-	"crypto/rsa"
-	"encoding/gob"
 	"fmt"
 	"strconv"
 	"strings"
@@ -80,12 +76,6 @@ type Claim struct {
 	SignedChainBytes [][]byte
 }
 
-func init() {
-	gob.Register(&ecdsa.PublicKey{})
-	gob.Register(&rsa.PublicKey{})
-	gob.Register(elliptic.P256())
-}
-
 // NewAuthorization creates a new, unclaimed authorization with a random token value.
 func NewAuthorization(userID string) (*Authorization, error) {
 	token := Token{UserID: userID}
@@ -127,20 +117,8 @@ func ParseToken(tokenString string) (*Token, error) {
 	return t, nil
 }
 
-func isGobEncoded(data []byte) bool {
-	return bytes.HasPrefix(data, []byte{0x14, 0xff, 0xb3, 0x2, 0x1, 0x1, 0x5, 0x47, 0x72})
-}
-
 // Unmarshal deserializes a set of authorizations.
 func (group *Group) Unmarshal(data []byte) error {
-	if isGobEncoded(data) {
-		decoder := gob.NewDecoder(bytes.NewBuffer(data))
-		if err := decoder.Decode(group); err != nil {
-			return Error.Wrap(err)
-		}
-		return nil
-	}
-
 	msg := &certificatepb.AuthorizationGroup{}
 	if err := pb.Unmarshal(data, msg); err != nil {
 		return Error.Wrap(err)

certificate/authorization/db.go

@@ -296,39 +296,3 @@ func (authDB *DB) put(ctx context.Context, userID string, auths Group) (err erro
 	}
 	return nil
 }
-
-// MigrateGob migrates gob encoded Group to protobuf encoded Group.
-func (authDB *DB) MigrateGob(ctx context.Context, progress func(userID string)) (err error) {
-	defer mon.Task()(&ctx)(&err)
-	err = authDB.db.IterateUnordered(ctx, func(ctx context.Context, it storage.Iterator) error {
-		var item storage.ListItem
-
-		for it.Next(ctx, &item) {
-			if !isGobEncoded(item.Value) {
-				continue
-			}
-			if progress != nil {
-				progress(string(item.Key))
-			}
-
-			var group Group
-			if err := group.Unmarshal(item.Value); err != nil {
-				return ErrDBInternal.New("unmarshal failed key=%q: %w", item.Key, err)
-			}
-
-			newValue, err := group.Marshal()
-			if err != nil {
-				return ErrDBInternal.New("re-marshal failed key=%q: %w", item.Key, err)
-			}
-
-			err = authDB.db.CompareAndSwap(ctx, item.Key, item.Value, newValue)
-			if err != nil {
-				return ErrDBInternal.New("updating %q failed: %w", item.Key, err)
-			}
-		}
-
-		return nil
-	})
-
-	return ErrDBInternal.Wrap(err)
-}

certificate/authorization/db_test.go

@@ -20,7 +20,6 @@ import (
 	"storj.io/common/rpc/rpcpeer"
 	"storj.io/common/testcontext"
 	"storj.io/storj/certificate/certificatepb"
-	"storj.io/storj/private/testredis"
 	"storj.io/storj/storage"
 )
 
@@ -415,39 +414,3 @@ func newTestAuthDB(t *testing.T, ctx *testcontext.Context) *DB {
 	require.NoError(t, err)
 	return db
 }
-
-func TestMigrateGob_Redis(t *testing.T) {
-	ctx := testcontext.New(t)
-
-	server, err := testredis.Start(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer ctx.Check(server.Close)
-
-	db, err := OpenDB(ctx, "redis://"+server.Addr()+"?db=1", true)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer ctx.Check(db.Close)
-
-	require.NoError(t, db.db.Put(ctx, storage.Key("gob"), expectedGroupDataGob))
-	require.NoError(t, db.db.Put(ctx, storage.Key("pb"), expectedGroupDataProto))
-
-	count := 0
-	err = db.MigrateGob(ctx, func(userID string) {
-		count++
-		t.Log("migrating", userID)
-	})
-	require.NoError(t, err)
-	require.Equal(t, 1, count)
-
-	data, err := db.db.Get(ctx, storage.Key("gob"))
-	require.NoError(t, err)
-	require.False(t, isGobEncoded(data))
-	require.Equal(t, expectedGroupDataProto, []byte(data))
-
-	data, err = db.db.Get(ctx, storage.Key("pb"))
-	require.NoError(t, err)
-	require.Equal(t, expectedGroupDataProto, []byte(data))
-}

cmd/certificates/main.go

@@ -99,7 +99,6 @@ func main() {
 
 	rootCmd.AddCommand(authCmd)
 	rootCmd.AddCommand(runCmd)
-	rootCmd.AddCommand(migrateCmd)
 	rootCmd.AddCommand(setupCmd)
 	rootCmd.AddCommand(signCmd)
 	rootCmd.AddCommand(verifyCmd)
@@ -114,7 +113,6 @@ func main() {
 	process.Bind(authInfoCmd, &authCfg, defaults, cfgstruct.ConfDir(confDir))
 	process.Bind(authExportCmd, &authCfg, defaults, cfgstruct.ConfDir(confDir))
 	process.Bind(runCmd, &runCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
-	process.Bind(migrateCmd, &runCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
 	process.Bind(setupCmd, &setupCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir), cfgstruct.SetupMode())
 	process.Bind(signCmd, &signCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
 	process.Bind(verifyCmd, &verifyCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))

cmd/certificates/migrate.go

@@ -1,42 +0,0 @@
-// Copyright (C) 2023 Storj Labs, Inc.
-// See LICENSE for copying information.
-
-package main
-
-import (
-	"github.com/spf13/cobra"
-	"github.com/zeebo/errs"
-	"go.uber.org/zap"
-
-	"storj.io/private/process"
-	"storj.io/storj/certificate/authorization"
-)
-
-var (
-	migrateCmd = &cobra.Command{
-		Use:   "migrate-gob",
-		Short: "Migrate from gob encoding to protobuf encoding",
-		RunE:  cmdMigrate,
-	}
-)
-
-func cmdMigrate(cmd *cobra.Command, args []string) error {
-	ctx, _ := process.Ctx(cmd)
-
-	authorizationDB, err := authorization.OpenDBFromCfg(ctx, runCfg.AuthorizationDB)
-	if err != nil {
-		return errs.New("error opening authorizations database: %+v", err)
-	}
-	defer func() {
-		err = errs.Combine(err, authorizationDB.Close())
-	}()
-
-	log := zap.L()
-	count := 0
-	return authorizationDB.MigrateGob(ctx, func(userID string) {
-		if count%100 == 0 {
-			log.Info("progress", zap.String("last", userID), zap.Int("total-processed-count", count))
-		}
-		count++
-	})
-}

storage/boltdb/client.go

@@ -230,13 +230,6 @@ func (client *Client) Iterate(ctx context.Context, opts storage.IterateOptions,
 	return client.IterateWithoutLookupLimit(ctx, opts, fn)
 }
 
-// IterateUnordered iterates over all data, however, does not guarantee ordering.
-// It only guarantees all items are iterated at least once.
-func (client *Client) IterateUnordered(ctx context.Context, fn func(context.Context, storage.Iterator) error) (err error) {
-	defer mon.Task()(&ctx)(&err)
-	return client.Iterate(ctx, storage.IterateOptions{}, fn)
-}
-
 // IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
 func (client *Client) IterateWithoutLookupLimit(ctx context.Context, opts storage.IterateOptions, fn func(context.Context, storage.Iterator) error) (err error) {
 	defer mon.Task()(&ctx)(&err)

storage/common.go

@@ -72,9 +72,6 @@ type KeyValueStore interface {
 	List(ctx context.Context, start Key, limit int) (Keys, error)
 	// Iterate iterates over items based on opts.
 	Iterate(ctx context.Context, opts IterateOptions, fn func(context.Context, Iterator) error) error
-	// IterateUnordered iterates over all data, however, does not guarantee ordering.
-	// It only guarantees all items are iterated at least once.
-	IterateUnordered(ctx context.Context, fn func(context.Context, Iterator) error) error
 	// IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
 	IterateWithoutLookupLimit(ctx context.Context, opts IterateOptions, fn func(context.Context, Iterator) error) error
 	// CompareAndSwap atomically compares and swaps oldValue with newValue.

storage/redis/client.go

@@ -193,17 +193,6 @@ func (client *Client) Iterate(ctx context.Context, opts storage.IterateOptions,
 	return client.IterateWithoutLookupLimit(ctx, opts, fn)
 }
 
-// IterateUnordered iterates over all data, however, does not guarantee ordering.
-// It only guarantees all items are iterated at least once.
-func (client *Client) IterateUnordered(ctx context.Context, fn func(context.Context, storage.Iterator) error) (err error) {
-	defer mon.Task()(&ctx)(&err)
-
-	return fn(ctx, &ScanIterator{
-		db: client.db,
-		it: client.db.Scan(ctx, 0, "*", 0).Iterator(),
-	})
-}
-
 // IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
 func (client *Client) IterateWithoutLookupLimit(ctx context.Context, opts storage.IterateOptions, fn func(context.Context, storage.Iterator) error) (err error) {
 	defer mon.Task()(&ctx)(&err)

storage/storelogger/logger.go

@@ -100,26 +100,6 @@ func (store *Logger) Iterate(ctx context.Context, opts storage.IterateOptions, f
 	})
 }
 
-// IterateUnordered iterates over all data, however, does not guarantee ordering.
-// It only guarantees all items are iterated at least once.
-func (store *Logger) IterateUnordered(ctx context.Context, fn func(context.Context, storage.Iterator) error) (err error) {
-	defer mon.Task()(&ctx)(&err)
-	store.log.Debug("IterateUnordered")
-	return store.store.IterateUnordered(ctx, func(ctx context.Context, it storage.Iterator) error {
-		return fn(ctx, storage.IteratorFunc(func(ctx context.Context, item *storage.ListItem) bool {
-			ok := it.Next(ctx, item)
-			if ok {
-				store.log.Debug("  ",
-					zap.ByteString("key", item.Key),
-					zap.Int("value length", len(item.Value)),
-					zap.Binary("truncated value", truncate(item.Value)),
-				)
-			}
-			return ok
-		}))
-	})
-}
-
 // IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
 func (store *Logger) IterateWithoutLookupLimit(ctx context.Context, opts storage.IterateOptions, fn func(context.Context, storage.Iterator) error) (err error) {
 	defer mon.Task()(&ctx)(&err)

storage/teststore/store.go

@@ -237,13 +237,6 @@ func (store *Client) Iterate(ctx context.Context, opts storage.IterateOptions, f
 	return store.IterateWithoutLookupLimit(ctx, opts, fn)
 }
 
-// IterateUnordered iterates over all data, however, does not guarantee ordering.
-// It only guarantees all items are iterated at least once.
-func (store *Client) IterateUnordered(ctx context.Context, fn func(context.Context, storage.Iterator) error) (err error) {
-	defer mon.Task()(&ctx)(&err)
-	return store.IterateWithoutLookupLimit(ctx, storage.IterateOptions{}, fn)
-}
-
 // IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
 func (store *Client) IterateWithoutLookupLimit(ctx context.Context, opts storage.IterateOptions, fn func(context.Context, storage.Iterator) error) (err error) {
 	defer mon.Task()(&ctx)(&err)