storj/satellite/satellitedb/migrate_postgres_test.go

180 lines
4.8 KiB
Go
Raw Normal View History

2019-02-14 21:55:21 +00:00
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package satellitedb_test
import (
"fmt"
"io/ioutil"
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
"storj.io/storj/internal/dbutil/dbschema"
"storj.io/storj/internal/dbutil/pgutil"
2019-04-26 14:39:11 +01:00
"storj.io/storj/internal/dbutil/pgutil/pgtest"
2019-02-14 21:55:21 +00:00
"storj.io/storj/satellite/satellitedb"
)
// loadSnapshots loads all the dbschemas from testdata/postgres.* caching the result
func loadSnapshots(connstr string) (*dbschema.Snapshots, error) {
snapshots := &dbschema.Snapshots{}
// find all postgres sql files
matches, err := filepath.Glob("testdata/postgres.*")
if err != nil {
return nil, err
}
for _, match := range matches {
versionStr := match[19 : len(match)-4] // hack to avoid trim issues with path differences in windows/linux
version, err := strconv.Atoi(versionStr)
if err != nil {
return nil, err
}
scriptData, err := ioutil.ReadFile(match)
if err != nil {
return nil, err
}
snapshot, err := pgutil.LoadSnapshotFromSQL(connstr, string(scriptData))
if err != nil {
return nil, fmt.Errorf("Version %d error: %+v", version, err)
2019-02-14 21:55:21 +00:00
}
snapshot.Version = version
snapshots.Add(snapshot)
}
snapshots.Sort()
return snapshots, nil
}
const newDataSeparator = `-- NEW DATA --`
func newData(snap *dbschema.Snapshot) string {
tokens := strings.SplitN(snap.Script, newDataSeparator, 2)
if len(tokens) != 2 {
return ""
}
return tokens[1]
}
var (
dbxschema struct {
sync.Once
*dbschema.Schema
err error
}
)
// loadDBXSChema loads dbxscript schema only once and caches it,
// it shouldn't change during the test
func loadDBXSchema(connstr, dbxscript string) (*dbschema.Schema, error) {
dbxschema.Do(func() {
dbxschema.Schema, dbxschema.err = pgutil.LoadSchemaFromSQL(connstr, dbxscript)
})
return dbxschema.Schema, dbxschema.err
}
const (
minBaseVersion = 0
maxBaseVersion = 4
)
func TestMigratePostgres(t *testing.T) {
2019-04-26 14:39:11 +01:00
if *pgtest.ConnStr == "" {
t.Skip("Postgres flag missing, example: -postgres-test-db=" + pgtest.DefaultConnStr)
2019-02-14 21:55:21 +00:00
}
2019-04-26 14:39:11 +01:00
snapshots, err := loadSnapshots(*pgtest.ConnStr)
2019-02-14 21:55:21 +00:00
require.NoError(t, err)
for _, base := range snapshots.List {
// versions 0 to 4 can be a starting point
if base.Version < minBaseVersion || maxBaseVersion < base.Version {
continue
}
t.Run(strconv.Itoa(base.Version), func(t *testing.T) {
log := zaptest.NewLogger(t)
schemaName := "migrate/satellite/" + strconv.Itoa(base.Version) + pgutil.CreateRandomTestingSchemaName(8)
2019-04-26 14:39:11 +01:00
connstr := pgutil.ConnstrWithSchema(*pgtest.ConnStr, schemaName)
2019-02-14 21:55:21 +00:00
// create a new satellitedb connection
db, err := satellitedb.New(log, connstr)
require.NoError(t, err)
defer func() { require.NoError(t, db.Close()) }()
// setup our own schema to avoid collisions
require.NoError(t, db.CreateSchema(schemaName))
defer func() { require.NoError(t, db.DropSchema(schemaName)) }()
// we need raw database access unfortunately
rawdb := db.(*satellitedb.DB).TestDBAccess()
// insert the base data into postgres
_, err = rawdb.Exec(base.Script)
require.NoError(t, err)
var finalSchema *dbschema.Schema
// get migration for this database
migrations := db.(*satellitedb.DB).PostgresMigration()
for i, step := range migrations.Steps {
// the schema is different when migration step is before the step, cannot test the layout
if step.Version < base.Version {
continue
}
tag := fmt.Sprintf("#%d - v%d", i, step.Version)
// run migration up to a specific version
err := migrations.TargetVersion(step.Version).Run(log.Named("migrate"), rawdb)
require.NoError(t, err, tag)
// find the matching expected version
expected, ok := snapshots.FindVersion(step.Version)
require.True(t, ok, "Missing snapshot v%d. Did you forget to add a snapshot for the new migration?", step.Version)
2019-02-14 21:55:21 +00:00
// insert data for new tables
if newdata := newData(expected); newdata != "" && step.Version > base.Version {
_, err = rawdb.Exec(newdata)
require.NoError(t, err, tag)
}
// load schema from database
currentSchema, err := pgutil.QuerySchema(rawdb)
require.NoError(t, err, tag)
// we don't care changes in versions table
currentSchema.DropTable("versions")
// load data from database
currentData, err := pgutil.QueryData(rawdb, currentSchema)
require.NoError(t, err, tag)
// verify schema and data
require.Equal(t, expected.Schema, currentSchema, tag)
require.Equal(t, expected.Data, currentData, tag)
// keep the last version around
finalSchema = currentSchema
}
// verify that we also match the dbx version
2019-04-26 14:39:11 +01:00
dbxschema, err := loadDBXSchema(*pgtest.ConnStr, rawdb.Schema())
2019-02-14 21:55:21 +00:00
require.NoError(t, err)
require.Equal(t, dbxschema, finalSchema, "dbx")
})
}
}