From d0686648dbd1956a0bffa5390ba87ef93b985385 Mon Sep 17 00:00:00 2001 From: Egon Elbre Date: Mon, 23 Jan 2023 18:05:43 +0200 Subject: [PATCH] certificate/authorization: implement gob to pb migration Change-Id: I6f33f5802e3f0a3c8a5f0c3cff61ef836a645c41 --- certificate/authorization/authorizations.go | 6 ++- certificate/authorization/db.go | 38 +++++++++++++++++++ certificate/authorization/db_test.go | 37 ++++++++++++++++++ cmd/certificates/main.go | 2 + cmd/certificates/migrate.go | 42 +++++++++++++++++++++ 5 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 cmd/certificates/migrate.go diff --git a/certificate/authorization/authorizations.go b/certificate/authorization/authorizations.go index 3a6e2d763..31d32c987 100644 --- a/certificate/authorization/authorizations.go +++ b/certificate/authorization/authorizations.go @@ -127,9 +127,13 @@ func ParseToken(tokenString string) (*Token, error) { return t, nil } +func isGobEncoded(data []byte) bool { + return bytes.HasPrefix(data, []byte{0x14, 0xff, 0xb3, 0x2, 0x1, 0x1, 0x5, 0x47, 0x72}) +} + // Unmarshal deserializes a set of authorizations. func (group *Group) Unmarshal(data []byte) error { - if bytes.HasPrefix(data, []byte{0x14, 0xff, 0xb3, 0x2, 0x1, 0x1, 0x5, 0x47, 0x72}) { + if isGobEncoded(data) { decoder := gob.NewDecoder(bytes.NewBuffer(data)) if err := decoder.Decode(group); err != nil { return Error.Wrap(err) diff --git a/certificate/authorization/db.go b/certificate/authorization/db.go index 540b2efd6..21054dbd5 100644 --- a/certificate/authorization/db.go +++ b/certificate/authorization/db.go @@ -296,3 +296,41 @@ func (authDB *DB) put(ctx context.Context, userID string, auths Group) (err erro } return nil } + +// MigrateGob migrates gob encoded Group to protobuf encoded Group. +func (authDB *DB) MigrateGob(ctx context.Context, progress func(userID string)) (err error) { + defer mon.Task()(&ctx)(&err) + err = authDB.db.Iterate(ctx, storage.IterateOptions{ + Recurse: true, + }, func(ctx context.Context, it storage.Iterator) error { + var item storage.ListItem + + for it.Next(ctx, &item) { + if !isGobEncoded(item.Value) { + continue + } + if progress != nil { + progress(string(item.Key)) + } + + var group Group + if err := group.Unmarshal(item.Value); err != nil { + return ErrDBInternal.New("unmarshal failed key=%q: %w", item.Key, err) + } + + newValue, err := group.Marshal() + if err != nil { + return ErrDBInternal.New("re-marshal failed key=%q: %w", item.Key, err) + } + + err = authDB.db.CompareAndSwap(ctx, item.Key, item.Value, newValue) + if err != nil { + return ErrDBInternal.New("updating %q failed: %w", item.Key, err) + } + } + + return nil + }) + + return ErrDBInternal.Wrap(err) +} diff --git a/certificate/authorization/db_test.go b/certificate/authorization/db_test.go index d250a5757..000d52dea 100644 --- a/certificate/authorization/db_test.go +++ b/certificate/authorization/db_test.go @@ -20,6 +20,7 @@ import ( "storj.io/common/rpc/rpcpeer" "storj.io/common/testcontext" "storj.io/storj/certificate/certificatepb" + "storj.io/storj/private/testredis" "storj.io/storj/storage" ) @@ -414,3 +415,39 @@ func newTestAuthDB(t *testing.T, ctx *testcontext.Context) *DB { require.NoError(t, err) return db } + +func TestMigrateGob_Redis(t *testing.T) { + ctx := testcontext.New(t) + + server, err := testredis.Start(ctx) + if err != nil { + t.Fatal(err) + } + defer ctx.Check(server.Close) + + db, err := OpenDB(ctx, "redis://"+server.Addr()+"?db=1", true) + if err != nil { + t.Fatal(err) + } + defer ctx.Check(db.Close) + + require.NoError(t, db.db.Put(ctx, storage.Key("gob"), expectedGroupDataGob)) + require.NoError(t, db.db.Put(ctx, storage.Key("pb"), expectedGroupDataProto)) + + count := 0 + err = db.MigrateGob(ctx, func(userID string) { + count++ + t.Log("migrating", userID) + }) + require.NoError(t, err) + require.Equal(t, 1, count) + + data, err := db.db.Get(ctx, storage.Key("gob")) + require.NoError(t, err) + require.False(t, isGobEncoded(data)) + require.Equal(t, expectedGroupDataProto, []byte(data)) + + data, err = db.db.Get(ctx, storage.Key("pb")) + require.NoError(t, err) + require.Equal(t, expectedGroupDataProto, []byte(data)) +} diff --git a/cmd/certificates/main.go b/cmd/certificates/main.go index d774aaba8..1a4efeb89 100644 --- a/cmd/certificates/main.go +++ b/cmd/certificates/main.go @@ -99,6 +99,7 @@ func main() { rootCmd.AddCommand(authCmd) rootCmd.AddCommand(runCmd) + rootCmd.AddCommand(migrateCmd) rootCmd.AddCommand(setupCmd) rootCmd.AddCommand(signCmd) rootCmd.AddCommand(verifyCmd) @@ -113,6 +114,7 @@ func main() { process.Bind(authInfoCmd, &authCfg, defaults, cfgstruct.ConfDir(confDir)) process.Bind(authExportCmd, &authCfg, defaults, cfgstruct.ConfDir(confDir)) process.Bind(runCmd, &runCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir)) + process.Bind(migrateCmd, &runCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir)) process.Bind(setupCmd, &setupCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir), cfgstruct.SetupMode()) process.Bind(signCmd, &signCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir)) process.Bind(verifyCmd, &verifyCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir)) diff --git a/cmd/certificates/migrate.go b/cmd/certificates/migrate.go new file mode 100644 index 000000000..c4e6d990e --- /dev/null +++ b/cmd/certificates/migrate.go @@ -0,0 +1,42 @@ +// Copyright (C) 2023 Storj Labs, Inc. +// See LICENSE for copying information. + +package main + +import ( + "github.com/spf13/cobra" + "github.com/zeebo/errs" + "go.uber.org/zap" + + "storj.io/private/process" + "storj.io/storj/certificate/authorization" +) + +var ( + migrateCmd = &cobra.Command{ + Use: "migrate-gob", + Short: "Migrate from gob encoding to protobuf encoding", + RunE: cmdMigrate, + } +) + +func cmdMigrate(cmd *cobra.Command, args []string) error { + ctx, _ := process.Ctx(cmd) + + authorizationDB, err := authorization.OpenDBFromCfg(ctx, runCfg.AuthorizationDB) + if err != nil { + return errs.New("error opening authorizations database: %+v", err) + } + defer func() { + err = errs.Combine(err, authorizationDB.Close()) + }() + + log := zap.L() + count := 0 + return authorizationDB.MigrateGob(ctx, func(userID string) { + if count%100 == 0 { + log.Info("progress", zap.String("user", userID), zap.Int("count", count)) + } + count++ + }) +}