JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

certificate/authorization,cmd/certificates: remove gob code

Browse Source 
Now that we've migrated data we can remove gob handling.

Change-Id: I55f772b7a0dda71c51db0683bad3db66b89867ac
This commit is contained in:
Egon Elbre 2023-01-27 15:06:31 +02:00
parent 252c437b0e
commit 0e00c7b8da
11 changed files with 4 additions and 197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
certificate/authorization/authorizations.go
View File

@ -6,11 +6,7 @@ package authorization
import (
"bytes"
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/rsa"
"encoding/gob"
"fmt"
"strconv"
"strings"
@ -80,12 +76,6 @@ type Claim struct {
SignedChainBytes [][]byte
}
func init() {
gob.Register(&ecdsa.PublicKey{})
gob.Register(&rsa.PublicKey{})
gob.Register(elliptic.P256())
}
// NewAuthorization creates a new, unclaimed authorization with a random token value.
func NewAuthorization(userID string) (*Authorization, error) {
token := Token{UserID: userID}
@ -127,20 +117,8 @@ func ParseToken(tokenString string) (*Token, error) {
return t, nil
}
func isGobEncoded(data []byte) bool {
return bytes.HasPrefix(data, []byte{0x14, 0xff, 0xb3, 0x2, 0x1, 0x1, 0x5, 0x47, 0x72})
}
// Unmarshal deserializes a set of authorizations.
func (group *Group) Unmarshal(data []byte) error {
if isGobEncoded(data) {
decoder := gob.NewDecoder(bytes.NewBuffer(data))
if err := decoder.Decode(group); err != nil {
return Error.Wrap(err)
}
return nil
}
msg := &certificatepb.AuthorizationGroup{}
if err := pb.Unmarshal(data, msg); err != nil {
return Error.Wrap(err)

14
certificate/authorization/authorizations_test.go
View File

File diff suppressed because one or more lines are too long

36
certificate/authorization/db.go
View File

@ -296,39 +296,3 @@ func (authDB *DB) put(ctx context.Context, userID string, auths Group) (err erro
}
return nil
}
// MigrateGob migrates gob encoded Group to protobuf encoded Group.
func (authDB *DB) MigrateGob(ctx context.Context, progress func(userID string)) (err error) {
defer mon.Task()(&ctx)(&err)
err = authDB.db.IterateUnordered(ctx, func(ctx context.Context, it storage.Iterator) error {
var item storage.ListItem
for it.Next(ctx, &item) {
if !isGobEncoded(item.Value) {
continue
}
if progress != nil {
progress(string(item.Key))
}
var group Group
if err := group.Unmarshal(item.Value); err != nil {
return ErrDBInternal.New("unmarshal failed key=%q: %w", item.Key, err)
}
newValue, err := group.Marshal()
if err != nil {
return ErrDBInternal.New("re-marshal failed key=%q: %w", item.Key, err)
}
err = authDB.db.CompareAndSwap(ctx, item.Key, item.Value, newValue)
if err != nil {
return ErrDBInternal.New("updating %q failed: %w", item.Key, err)
}
}
return nil
})
return ErrDBInternal.Wrap(err)
}

37
certificate/authorization/db_test.go
View File

@ -20,7 +20,6 @@ import (
"storj.io/common/rpc/rpcpeer"
"storj.io/common/testcontext"
"storj.io/storj/certificate/certificatepb"
"storj.io/storj/private/testredis"
"storj.io/storj/storage"
)
@ -415,39 +414,3 @@ func newTestAuthDB(t *testing.T, ctx *testcontext.Context) *DB {
require.NoError(t, err)
return db
}
func TestMigrateGob_Redis(t *testing.T) {
ctx := testcontext.New(t)
server, err := testredis.Start(ctx)
if err != nil {
t.Fatal(err)
}
defer ctx.Check(server.Close)
db, err := OpenDB(ctx, "redis://"+server.Addr()+"?db=1", true)
if err != nil {
t.Fatal(err)
}
defer ctx.Check(db.Close)
require.NoError(t, db.db.Put(ctx, storage.Key("gob"), expectedGroupDataGob))
require.NoError(t, db.db.Put(ctx, storage.Key("pb"), expectedGroupDataProto))
count := 0
err = db.MigrateGob(ctx, func(userID string) {
count++
t.Log("migrating", userID)
})
require.NoError(t, err)
require.Equal(t, 1, count)
data, err := db.db.Get(ctx, storage.Key("gob"))
require.NoError(t, err)
require.False(t, isGobEncoded(data))
require.Equal(t, expectedGroupDataProto, []byte(data))
data, err = db.db.Get(ctx, storage.Key("pb"))
require.NoError(t, err)
require.Equal(t, expectedGroupDataProto, []byte(data))
}

2
cmd/certificates/main.go
View File

@ -99,7 +99,6 @@ func main() {
rootCmd.AddCommand(authCmd)
rootCmd.AddCommand(runCmd)
rootCmd.AddCommand(migrateCmd)
rootCmd.AddCommand(setupCmd)
rootCmd.AddCommand(signCmd)
rootCmd.AddCommand(verifyCmd)
@ -114,7 +113,6 @@ func main() {
process.Bind(authInfoCmd, &authCfg, defaults, cfgstruct.ConfDir(confDir))
process.Bind(authExportCmd, &authCfg, defaults, cfgstruct.ConfDir(confDir))
process.Bind(runCmd, &runCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
process.Bind(migrateCmd, &runCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
process.Bind(setupCmd, &setupCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir), cfgstruct.SetupMode())
process.Bind(signCmd, &signCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
process.Bind(verifyCmd, &verifyCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))

42
cmd/certificates/migrate.go
View File

@ -1,42 +0,0 @@
// Copyright (C) 2023 Storj Labs, Inc.
// See LICENSE for copying information.
package main
import (
"github.com/spf13/cobra"
"github.com/zeebo/errs"
"go.uber.org/zap"
"storj.io/private/process"
"storj.io/storj/certificate/authorization"
)
var (
migrateCmd = &cobra.Command{
Use: "migrate-gob",
Short: "Migrate from gob encoding to protobuf encoding",
RunE: cmdMigrate,
}
)
func cmdMigrate(cmd *cobra.Command, args []string) error {
ctx, _ := process.Ctx(cmd)
authorizationDB, err := authorization.OpenDBFromCfg(ctx, runCfg.AuthorizationDB)
if err != nil {
return errs.New("error opening authorizations database: %+v", err)
}
defer func() {
err = errs.Combine(err, authorizationDB.Close())
}()
log := zap.L()
count := 0
return authorizationDB.MigrateGob(ctx, func(userID string) {
if count%100 == 0 {
log.Info("progress", zap.String("last", userID), zap.Int("total-processed-count", count))
}
count++
})
}

7
storage/boltdb/client.go
View File

@ -230,13 +230,6 @@ func (client *Client) Iterate(ctx context.Context, opts storage.IterateOptions,
return client.IterateWithoutLookupLimit(ctx, opts, fn)
}
// IterateUnordered iterates over all data, however, does not guarantee ordering.
// It only guarantees all items are iterated at least once.
func (client *Client) IterateUnordered(ctx context.Context, fn func(context.Context, storage.Iterator) error) (err error) {
defer mon.Task()(&ctx)(&err)
return client.Iterate(ctx, storage.IterateOptions{}, fn)
}
// IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
func (client *Client) IterateWithoutLookupLimit(ctx context.Context, opts storage.IterateOptions, fn func(context.Context, storage.Iterator) error) (err error) {
defer mon.Task()(&ctx)(&err)

3
storage/common.go
View File

@ -72,9 +72,6 @@ type KeyValueStore interface {
List(ctx context.Context, start Key, limit int) (Keys, error)
// Iterate iterates over items based on opts.
Iterate(ctx context.Context, opts IterateOptions, fn func(context.Context, Iterator) error) error
// IterateUnordered iterates over all data, however, does not guarantee ordering.
// It only guarantees all items are iterated at least once.
IterateUnordered(ctx context.Context, fn func(context.Context, Iterator) error) error
// IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
IterateWithoutLookupLimit(ctx context.Context, opts IterateOptions, fn func(context.Context, Iterator) error) error
// CompareAndSwap atomically compares and swaps oldValue with newValue.

11
storage/redis/client.go
View File

@ -193,17 +193,6 @@ func (client *Client) Iterate(ctx context.Context, opts storage.IterateOptions,
return client.IterateWithoutLookupLimit(ctx, opts, fn)
}
// IterateUnordered iterates over all data, however, does not guarantee ordering.
// It only guarantees all items are iterated at least once.
func (client *Client) IterateUnordered(ctx context.Context, fn func(context.Context, storage.Iterator) error) (err error) {
defer mon.Task()(&ctx)(&err)
return fn(ctx, &ScanIterator{
db: client.db,
it: client.db.Scan(ctx, 0, "*", 0).Iterator(),
})
}
// IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
func (client *Client) IterateWithoutLookupLimit(ctx context.Context, opts storage.IterateOptions, fn func(context.Context, storage.Iterator) error) (err error) {
defer mon.Task()(&ctx)(&err)

20
storage/storelogger/logger.go
View File

@ -100,26 +100,6 @@ func (store *Logger) Iterate(ctx context.Context, opts storage.IterateOptions, f
})
}
// IterateUnordered iterates over all data, however, does not guarantee ordering.
// It only guarantees all items are iterated at least once.
func (store *Logger) IterateUnordered(ctx context.Context, fn func(context.Context, storage.Iterator) error) (err error) {
defer mon.Task()(&ctx)(&err)
store.log.Debug("IterateUnordered")
return store.store.IterateUnordered(ctx, func(ctx context.Context, it storage.Iterator) error {
return fn(ctx, storage.IteratorFunc(func(ctx context.Context, item *storage.ListItem) bool {
ok := it.Next(ctx, item)
if ok {
store.log.Debug(" ",
zap.ByteString("key", item.Key),
zap.Int("value length", len(item.Value)),
zap.Binary("truncated value", truncate(item.Value)),
)
}
return ok
}))
})
}
// IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
func (store *Logger) IterateWithoutLookupLimit(ctx context.Context, opts storage.IterateOptions, fn func(context.Context, storage.Iterator) error) (err error) {
defer mon.Task()(&ctx)(&err)

7
storage/teststore/store.go
View File

@ -237,13 +237,6 @@ func (store *Client) Iterate(ctx context.Context, opts storage.IterateOptions, f
return store.IterateWithoutLookupLimit(ctx, opts, fn)
}
// IterateUnordered iterates over all data, however, does not guarantee ordering.
// It only guarantees all items are iterated at least once.
func (store *Client) IterateUnordered(ctx context.Context, fn func(context.Context, storage.Iterator) error) (err error) {
defer mon.Task()(&ctx)(&err)
return store.IterateWithoutLookupLimit(ctx, storage.IterateOptions{}, fn)
}
// IterateWithoutLookupLimit calls the callback with an iterator over the keys, but doesn't enforce default limit on opts.
func (store *Client) IterateWithoutLookupLimit(ctx context.Context, opts storage.IterateOptions, fn func(context.Context, storage.Iterator) error) (err error) {
defer mon.Task()(&ctx)(&err)