Network Wipe (Pre Beta) (#2566)
This commit is contained in:
Stefan Benten
littleskunk
parent
d044613679
commit
de300e9235
@ -40,18 +40,18 @@ func NewDir(path string) (*Dir, error) {
|
||||
}
|
||||
|
||||
return dir, errs.Combine(
|
||||
os.MkdirAll(dir.blobdir(), dirPermission),
|
||||
os.MkdirAll(dir.blobsdir(), dirPermission),
|
||||
os.MkdirAll(dir.tempdir(), dirPermission),
|
||||
os.MkdirAll(dir.trashdir(), dirPermission),
|
||||
os.MkdirAll(dir.garbagedir(), dirPermission),
|
||||
)
|
||||
}
|
||||
|
||||
// Path returns the directory path
|
||||
func (dir *Dir) Path() string { return dir.path }
|
||||
|
||||
func (dir *Dir) blobdir() string { return filepath.Join(dir.path, "blob") }
|
||||
func (dir *Dir) tempdir() string { return filepath.Join(dir.path, "tmp") }
|
||||
func (dir *Dir) trashdir() string { return filepath.Join(dir.path, "trash") }
|
||||
func (dir *Dir) blobsdir() string { return filepath.Join(dir.path, "blobs") }
|
||||
func (dir *Dir) tempdir() string { return filepath.Join(dir.path, "temp") }
|
||||
func (dir *Dir) garbagedir() string { return filepath.Join(dir.path, "garbage") }
|
||||
|
||||
// CreateTemporaryFile creates a preallocated temporary file in the temp directory
|
||||
// prealloc preallocates file to make writing faster
|
||||
@ -93,16 +93,16 @@ func (dir *Dir) blobToPath(ref storage.BlobRef) (string, error) {
|
||||
// ensure we always have at least
|
||||
key = "11" + key
|
||||
}
|
||||
return filepath.Join(dir.blobdir(), namespace, key[:2], key[2:]), nil
|
||||
return filepath.Join(dir.blobsdir(), namespace, key[:2], key[2:]), nil
|
||||
}
|
||||
|
||||
// blobToTrashPath converts blob reference to a filepath in transient storage
|
||||
// the files in trash are deleted in an interval (in case the initial deletion didn't work for some reason)
|
||||
func (dir *Dir) blobToTrashPath(ref storage.BlobRef) string {
|
||||
name := []byte{}
|
||||
var name []byte
|
||||
name = append(name, ref.Namespace...)
|
||||
name = append(name, ref.Key...)
|
||||
return filepath.Join(dir.trashdir(), pathEncoding.EncodeToString(name))
|
||||
return filepath.Join(dir.garbagedir(), pathEncoding.EncodeToString(name))
|
||||
}
|
||||
|
||||
// Commit commits temporary file to the permanent storage
|
||||
@ -234,7 +234,7 @@ func (dir *Dir) GarbageCollect(ctx context.Context) (err error) {
|
||||
}
|
||||
|
||||
// remove anything left in the trashdir
|
||||
_ = removeAllContent(ctx, dir.trashdir())
|
||||
_ = removeAllContent(ctx, dir.garbagedir())
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@ -29,6 +29,7 @@ type InfoDB struct {
|
||||
db utcDB
|
||||
bandwidthdb bandwidthdb
|
||||
pieceinfo pieceinfo
|
||||
location string
|
||||
}
|
||||
|
||||
// newInfo creates or opens InfoDB at the specified path.
|
||||
@ -47,6 +48,7 @@ func newInfo(path string) (*InfoDB, error) {
|
||||
infoDb := &InfoDB{db: utcDB{db}}
|
||||
infoDb.pieceinfo = pieceinfo{InfoDB: infoDb, space: spaceUsed{used: 0, once: sync.Once{}}}
|
||||
infoDb.bandwidthdb = bandwidthdb{InfoDB: infoDb, bandwidth: bandwidthUsed{used: 0, mu: sync.RWMutex{}, usedSince: time.Time{}}, loop: sync2.NewCycle(time.Hour)}
|
||||
infoDb.location = path
|
||||
|
||||
return infoDb, nil
|
||||
}
|
||||
@ -283,6 +285,82 @@ func (db *InfoDB) Migration() *migrate.Migration {
|
||||
)`,
|
||||
},
|
||||
},
|
||||
{
|
||||
Description: "Clear Tables from Alpha data",
|
||||
Version: 12,
|
||||
Action: migrate.SQL{
|
||||
`DROP TABLE pieceinfo`,
|
||||
`DROP TABLE used_serial`,
|
||||
`DROP TABLE order_archive`,
|
||||
`CREATE TABLE pieceinfo_ (
|
||||
satellite_id BLOB NOT NULL,
|
||||
piece_id BLOB NOT NULL,
|
||||
piece_size BIGINT NOT NULL,
|
||||
piece_expiration TIMESTAMP,
|
||||
|
||||
order_limit BLOB NOT NULL,
|
||||
uplink_piece_hash BLOB NOT NULL,
|
||||
uplink_cert_id INTEGER NOT NULL,
|
||||
|
||||
deletion_failed_at TIMESTAMP,
|
||||
piece_creation TIMESTAMP NOT NULL,
|
||||
|
||||
FOREIGN KEY(uplink_cert_id) REFERENCES certificate(cert_id)
|
||||
)`,
|
||||
`CREATE UNIQUE INDEX pk_pieceinfo_ ON pieceinfo_(satellite_id, piece_id)`,
|
||||
`CREATE INDEX idx_pieceinfo__expiration ON pieceinfo_(piece_expiration) WHERE piece_expiration IS NOT NULL`,
|
||||
`CREATE TABLE used_serial_ (
|
||||
satellite_id BLOB NOT NULL,
|
||||
serial_number BLOB NOT NULL,
|
||||
expiration TIMESTAMP NOT NULL
|
||||
)`,
|
||||
`CREATE UNIQUE INDEX pk_used_serial_ ON used_serial_(satellite_id, serial_number)`,
|
||||
`CREATE INDEX idx_used_serial_ ON used_serial_(expiration)`,
|
||||
`CREATE TABLE order_archive_ (
|
||||
satellite_id BLOB NOT NULL,
|
||||
serial_number BLOB NOT NULL,
|
||||
|
||||
order_limit_serialized BLOB NOT NULL,
|
||||
order_serialized BLOB NOT NULL,
|
||||
|
||||
uplink_cert_id INTEGER NOT NULL,
|
||||
|
||||
status INTEGER NOT NULL,
|
||||
archived_at TIMESTAMP NOT NULL,
|
||||
|
||||
FOREIGN KEY(uplink_cert_id) REFERENCES certificate(cert_id)
|
||||
)`,
|
||||
},
|
||||
},
|
||||
{
|
||||
Description: "Free Storagenodes from trash data",
|
||||
Version: 13,
|
||||
Action: migrate.Func(func(log *zap.Logger, mgdb migrate.DB, tx *sql.Tx) error {
|
||||
// When using inmemory DB, skip deletion process
|
||||
if db.location == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := os.RemoveAll(filepath.Join(filepath.Dir(db.location), "blob/ukfu6bhbboxilvt7jrwlqk7y2tapb5d2r2tsmj2sjxvw5qaaaaaa")) // us-central1
|
||||
if err != nil {
|
||||
log.Sugar().Debug(err)
|
||||
}
|
||||
err = os.RemoveAll(filepath.Join(filepath.Dir(db.location), "blob/v4weeab67sbgvnbwd5z7tweqsqqun7qox2agpbxy44mqqaaaaaaa")) // europe-west1
|
||||
if err != nil {
|
||||
log.Sugar().Debug(err)
|
||||
}
|
||||
err = os.RemoveAll(filepath.Join(filepath.Dir(db.location), "blob/qstuylguhrn2ozjv4h2c6xpxykd622gtgurhql2k7k75wqaaaaaa")) // asia-east1
|
||||
if err != nil {
|
||||
log.Sugar().Debug(err)
|
||||
}
|
||||
err = os.RemoveAll(filepath.Join(filepath.Dir(db.location), "blob/abforhuxbzyd35blusvrifvdwmfx4hmocsva4vmpp3rgqaaaaaaa")) // "tothemoon (stefan)"
|
||||
if err != nil {
|
||||
log.Sugar().Debug(err)
|
||||
}
|
||||
// To prevent the node from starting up, we just log errors and return nil
|
||||
return nil
|
||||
}),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@ -150,7 +150,7 @@ func (db *ordersdb) Archive(ctx context.Context, satellite storj.NodeID, serial
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
|
||||
result, err := db.db.Exec(`
|
||||
INSERT INTO order_archive (
|
||||
INSERT INTO order_archive_ (
|
||||
satellite_id, serial_number,
|
||||
order_limit_serialized, order_serialized,
|
||||
uplink_cert_id,
|
||||
@ -187,7 +187,7 @@ func (db *ordersdb) ListArchived(ctx context.Context, limit int) (_ []*orders.Ar
|
||||
|
||||
rows, err := db.db.Query(`
|
||||
SELECT order_limit_serialized, order_serialized, status, archived_at
|
||||
FROM order_archive
|
||||
FROM order_archive_
|
||||
LIMIT ?
|
||||
`, limit)
|
||||
if err != nil {
|
||||
|
||||
@ -58,7 +58,7 @@ func (db *pieceinfo) Add(ctx context.Context, info *pieces.Info) (err error) {
|
||||
// TODO remove `uplink_cert_id` from DB
|
||||
_, err = db.db.ExecContext(ctx, db.Rebind(`
|
||||
INSERT INTO
|
||||
pieceinfo(satellite_id, piece_id, piece_size, piece_creation, piece_expiration, order_limit, uplink_piece_hash, uplink_cert_id)
|
||||
pieceinfo_(satellite_id, piece_id, piece_size, piece_creation, piece_expiration, order_limit, uplink_piece_hash, uplink_cert_id)
|
||||
VALUES (?,?,?,?,?,?,?,?)
|
||||
`), info.SatelliteID, info.PieceID, info.PieceSize, info.PieceCreation.UTC(), pieceExpiration, orderLimit, uplinkPieceHash, 0)
|
||||
|
||||
@ -75,7 +75,7 @@ func (db *pieceinfo) GetPieceIDs(ctx context.Context, satelliteID storj.NodeID,
|
||||
|
||||
rows, err := db.db.QueryContext(ctx, db.Rebind(`
|
||||
SELECT piece_id
|
||||
FROM pieceinfo
|
||||
FROM pieceinfo_
|
||||
WHERE satellite_id = ? AND datetime(piece_creation) < datetime(?)
|
||||
ORDER BY piece_id
|
||||
LIMIT ? OFFSET ?
|
||||
@ -107,7 +107,7 @@ func (db *pieceinfo) Get(ctx context.Context, satelliteID storj.NodeID, pieceID
|
||||
|
||||
err = db.db.QueryRowContext(ctx, db.Rebind(`
|
||||
SELECT piece_size, piece_creation, piece_expiration, order_limit, uplink_piece_hash
|
||||
FROM pieceinfo
|
||||
FROM pieceinfo_
|
||||
WHERE satellite_id = ? AND piece_id = ?
|
||||
`), satelliteID, pieceID).Scan(&info.PieceSize, &info.PieceCreation, &info.PieceExpiration, &orderLimit, &uplinkPieceHash)
|
||||
if err != nil {
|
||||
@ -136,7 +136,7 @@ func (db *pieceinfo) Delete(ctx context.Context, satelliteID storj.NodeID, piece
|
||||
var pieceSize int64
|
||||
err = db.db.QueryRowContext(ctx, db.Rebind(`
|
||||
SELECT piece_size
|
||||
FROM pieceinfo
|
||||
FROM pieceinfo_
|
||||
WHERE satellite_id = ? AND piece_id = ?
|
||||
`), satelliteID, pieceID).Scan(&pieceSize)
|
||||
// Ignore no rows found errors
|
||||
@ -144,7 +144,7 @@ func (db *pieceinfo) Delete(ctx context.Context, satelliteID storj.NodeID, piece
|
||||
return ErrInfo.Wrap(err)
|
||||
}
|
||||
_, err = db.db.ExecContext(ctx, db.Rebind(`
|
||||
DELETE FROM pieceinfo
|
||||
DELETE FROM pieceinfo_
|
||||
WHERE satellite_id = ?
|
||||
AND piece_id = ?
|
||||
`), satelliteID, pieceID)
|
||||
@ -163,7 +163,7 @@ func (db *pieceinfo) DeleteFailed(ctx context.Context, satelliteID storj.NodeID,
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
|
||||
_, err = db.db.ExecContext(ctx, db.Rebind(`
|
||||
UPDATE pieceinfo
|
||||
UPDATE pieceinfo_
|
||||
SET deletion_failed_at = ?
|
||||
WHERE satellite_id = ?
|
||||
AND piece_id = ?
|
||||
@ -178,7 +178,7 @@ func (db *pieceinfo) GetExpired(ctx context.Context, expiredAt time.Time, limit
|
||||
|
||||
rows, err := db.db.QueryContext(ctx, db.Rebind(`
|
||||
SELECT satellite_id, piece_id, piece_size
|
||||
FROM pieceinfo
|
||||
FROM pieceinfo_
|
||||
WHERE piece_expiration IS NOT NULL
|
||||
AND piece_expiration < ?
|
||||
AND ((deletion_failed_at IS NULL) OR deletion_failed_at <> ?)
|
||||
@ -222,7 +222,7 @@ func (db *pieceinfo) CalculatedSpaceUsed(ctx context.Context) (_ int64, err erro
|
||||
var sum sql.NullInt64
|
||||
err = db.db.QueryRowContext(ctx, db.Rebind(`
|
||||
SELECT SUM(piece_size)
|
||||
FROM pieceinfo
|
||||
FROM pieceinfo_
|
||||
`)).Scan(&sum)
|
||||
|
||||
if err == sql.ErrNoRows || !sum.Valid {
|
||||
@ -238,7 +238,7 @@ func (db *pieceinfo) SpaceUsedBySatellite(ctx context.Context, satelliteID storj
|
||||
var sum sql.NullInt64
|
||||
err = db.db.QueryRowContext(ctx, db.Rebind(`
|
||||
SELECT SUM(piece_size)
|
||||
FROM pieceinfo
|
||||
FROM pieceinfo_
|
||||
WHERE satellite_id = ?
|
||||
`), satelliteID).Scan(&sum)
|
||||
|
||||
|
||||
140
storagenode/storagenodedb/testdata/sqlite.v12.sql
vendored
Normal file
140
storagenode/storagenodedb/testdata/sqlite.v12.sql
vendored
Normal file
@ -0,0 +1,140 @@
|
||||
-- table for keeping serials that need to be verified against
|
||||
CREATE TABLE used_serial_ (
|
||||
satellite_id BLOB NOT NULL,
|
||||
serial_number BLOB NOT NULL,
|
||||
expiration TIMESTAMP NOT NULL
|
||||
);
|
||||
-- primary key on satellite id and serial number
|
||||
CREATE UNIQUE INDEX pk_used_serial_ ON used_serial_(satellite_id, serial_number);
|
||||
-- expiration index to allow fast deletion
|
||||
CREATE INDEX idx_used_serial_ ON used_serial_(expiration);
|
||||
|
||||
-- certificate table for storing uplink/satellite certificates
|
||||
CREATE TABLE certificate (
|
||||
cert_id INTEGER
|
||||
);
|
||||
|
||||
-- table for storing piece meta info
|
||||
CREATE TABLE pieceinfo_ (
|
||||
satellite_id BLOB NOT NULL,
|
||||
piece_id BLOB NOT NULL,
|
||||
piece_size BIGINT NOT NULL,
|
||||
piece_expiration TIMESTAMP,
|
||||
|
||||
order_limit BLOB NOT NULL,
|
||||
uplink_piece_hash BLOB NOT NULL,
|
||||
uplink_cert_id INTEGER NOT NULL,
|
||||
|
||||
deletion_failed_at TIMESTAMP,
|
||||
piece_creation TIMESTAMP NOT NULL,
|
||||
|
||||
FOREIGN KEY(uplink_cert_id) REFERENCES certificate(cert_id)
|
||||
);
|
||||
-- primary key by satellite id and piece id
|
||||
CREATE UNIQUE INDEX pk_pieceinfo_ ON pieceinfo_(satellite_id, piece_id);
|
||||
-- fast queries for expiration for pieces that have one
|
||||
CREATE INDEX idx_pieceinfo__expiration ON pieceinfo_(piece_expiration) WHERE piece_expiration IS NOT NULL;
|
||||
|
||||
-- table for storing bandwidth usage
|
||||
CREATE TABLE bandwidth_usage (
|
||||
satellite_id BLOB NOT NULL,
|
||||
action INTEGER NOT NULL,
|
||||
amount BIGINT NOT NULL,
|
||||
created_at TIMESTAMP NOT NULL
|
||||
);
|
||||
CREATE INDEX idx_bandwidth_usage_satellite ON bandwidth_usage(satellite_id);
|
||||
CREATE INDEX idx_bandwidth_usage_created ON bandwidth_usage(created_at);
|
||||
|
||||
-- table for storing all unsent orders
|
||||
CREATE TABLE unsent_order (
|
||||
satellite_id BLOB NOT NULL,
|
||||
serial_number BLOB NOT NULL,
|
||||
|
||||
order_limit_serialized BLOB NOT NULL,
|
||||
order_serialized BLOB NOT NULL,
|
||||
order_limit_expiration TIMESTAMP NOT NULL,
|
||||
|
||||
uplink_cert_id INTEGER NOT NULL,
|
||||
|
||||
FOREIGN KEY(uplink_cert_id) REFERENCES certificate(cert_id)
|
||||
);
|
||||
CREATE UNIQUE INDEX idx_orders ON unsent_order(satellite_id, serial_number);
|
||||
|
||||
-- table for storing all sent orders
|
||||
CREATE TABLE order_archive_ (
|
||||
satellite_id BLOB NOT NULL,
|
||||
serial_number BLOB NOT NULL,
|
||||
|
||||
order_limit_serialized BLOB NOT NULL,
|
||||
order_serialized BLOB NOT NULL,
|
||||
|
||||
uplink_cert_id INTEGER NOT NULL,
|
||||
|
||||
status INTEGER NOT NULL,
|
||||
archived_at TIMESTAMP NOT NULL,
|
||||
|
||||
FOREIGN KEY(uplink_cert_id) REFERENCES certificate(cert_id)
|
||||
);
|
||||
|
||||
-- table for storing vouchers
|
||||
CREATE TABLE vouchers (
|
||||
satellite_id BLOB PRIMARY KEY NOT NULL,
|
||||
voucher_serialized BLOB NOT NULL,
|
||||
expiration TIMESTAMP NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE bandwidth_usage_rollups (
|
||||
interval_start TIMESTAMP NOT NULL,
|
||||
satellite_id BLOB NOT NULL,
|
||||
action INTEGER NOT NULL,
|
||||
amount BIGINT NOT NULL,
|
||||
PRIMARY KEY ( interval_start, satellite_id, action )
|
||||
);
|
||||
|
||||
INSERT INTO unsent_order VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',X'1eddef484b4c03f01332279032796972',X'0a101eddef484b4c03f0133227903279697212202b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf410001a201968996e7ef170a402fdfd88b6753df792c063c07c555905ffac9cd3cbd1c00022200ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac30002a20d00cf14f3c68b56321ace04902dec0484eb6f9098b22b31c6b3f82db249f191630643802420c08dfeb88e50510a8c1a5b9034a0c08dfeb88e50510a8c1a5b9035246304402204df59dc6f5d1bb7217105efbc9b3604d19189af37a81efbf16258e5d7db5549e02203bb4ead16e6e7f10f658558c22b59c3339911841e8dbaae6e2dea821f7326894',X'0a101eddef484b4c03f0133227903279697210321a47304502206d4c106ddec88140414bac5979c95bdea7de2e0ecc5be766e08f7d5ea36641a7022100e932ff858f15885ffa52d07e260c2c25d3861810ea6157956c1793ad0c906284','2019-04-01 16:01:35.9254586+00:00',1);
|
||||
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',0,0,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',0,0,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',1,1,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',1,1,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',2,2,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',2,2,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',3,3,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',3,3,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',4,4,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',4,4,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',5,5,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',5,5,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',6,6,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',6,6,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',1,1,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',1,1,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',2,2,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',2,2,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',3,3,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',3,3,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',4,4,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',4,4,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',5,5,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',5,5,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',6,6,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',6,6,'2019-04-01 20:51:24.1074772+00:00');
|
||||
|
||||
INSERT INTO vouchers VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000', X'd5e757fd8d207d1c46583fb58330f803dc961b71147308ff75ff1e72a0df6b0b', '2019-07-04 00:00:00.000000+00:00');
|
||||
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',0,0);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',0,0);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',1,1);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',1,1);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',2,2);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',2,2);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',3,3);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',3,3);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',4,4);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',4,4);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',5,5);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',5,5);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',6,6);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',6,6);
|
||||
|
||||
-- NEW DATA --
|
||||
140
storagenode/storagenodedb/testdata/sqlite.v13.sql
vendored
Normal file
140
storagenode/storagenodedb/testdata/sqlite.v13.sql
vendored
Normal file
@ -0,0 +1,140 @@
|
||||
-- table for keeping serials that need to be verified against
|
||||
CREATE TABLE used_serial_ (
|
||||
satellite_id BLOB NOT NULL,
|
||||
serial_number BLOB NOT NULL,
|
||||
expiration TIMESTAMP NOT NULL
|
||||
);
|
||||
-- primary key on satellite id and serial number
|
||||
CREATE UNIQUE INDEX pk_used_serial_ ON used_serial_(satellite_id, serial_number);
|
||||
-- expiration index to allow fast deletion
|
||||
CREATE INDEX idx_used_serial_ ON used_serial_(expiration);
|
||||
|
||||
-- certificate table for storing uplink/satellite certificates
|
||||
CREATE TABLE certificate (
|
||||
cert_id INTEGER
|
||||
);
|
||||
|
||||
-- table for storing piece meta info
|
||||
CREATE TABLE pieceinfo_ (
|
||||
satellite_id BLOB NOT NULL,
|
||||
piece_id BLOB NOT NULL,
|
||||
piece_size BIGINT NOT NULL,
|
||||
piece_expiration TIMESTAMP,
|
||||
|
||||
order_limit BLOB NOT NULL,
|
||||
uplink_piece_hash BLOB NOT NULL,
|
||||
uplink_cert_id INTEGER NOT NULL,
|
||||
|
||||
deletion_failed_at TIMESTAMP,
|
||||
piece_creation TIMESTAMP NOT NULL,
|
||||
|
||||
FOREIGN KEY(uplink_cert_id) REFERENCES certificate(cert_id)
|
||||
);
|
||||
-- primary key by satellite id and piece id
|
||||
CREATE UNIQUE INDEX pk_pieceinfo_ ON pieceinfo_(satellite_id, piece_id);
|
||||
-- fast queries for expiration for pieces that have one
|
||||
CREATE INDEX idx_pieceinfo__expiration ON pieceinfo_(piece_expiration) WHERE piece_expiration IS NOT NULL;
|
||||
|
||||
-- table for storing bandwidth usage
|
||||
CREATE TABLE bandwidth_usage (
|
||||
satellite_id BLOB NOT NULL,
|
||||
action INTEGER NOT NULL,
|
||||
amount BIGINT NOT NULL,
|
||||
created_at TIMESTAMP NOT NULL
|
||||
);
|
||||
CREATE INDEX idx_bandwidth_usage_satellite ON bandwidth_usage(satellite_id);
|
||||
CREATE INDEX idx_bandwidth_usage_created ON bandwidth_usage(created_at);
|
||||
|
||||
-- table for storing all unsent orders
|
||||
CREATE TABLE unsent_order (
|
||||
satellite_id BLOB NOT NULL,
|
||||
serial_number BLOB NOT NULL,
|
||||
|
||||
order_limit_serialized BLOB NOT NULL,
|
||||
order_serialized BLOB NOT NULL,
|
||||
order_limit_expiration TIMESTAMP NOT NULL,
|
||||
|
||||
uplink_cert_id INTEGER NOT NULL,
|
||||
|
||||
FOREIGN KEY(uplink_cert_id) REFERENCES certificate(cert_id)
|
||||
);
|
||||
CREATE UNIQUE INDEX idx_orders ON unsent_order(satellite_id, serial_number);
|
||||
|
||||
-- table for storing all sent orders
|
||||
CREATE TABLE order_archive_ (
|
||||
satellite_id BLOB NOT NULL,
|
||||
serial_number BLOB NOT NULL,
|
||||
|
||||
order_limit_serialized BLOB NOT NULL,
|
||||
order_serialized BLOB NOT NULL,
|
||||
|
||||
uplink_cert_id INTEGER NOT NULL,
|
||||
|
||||
status INTEGER NOT NULL,
|
||||
archived_at TIMESTAMP NOT NULL,
|
||||
|
||||
FOREIGN KEY(uplink_cert_id) REFERENCES certificate(cert_id)
|
||||
);
|
||||
|
||||
-- table for storing vouchers
|
||||
CREATE TABLE vouchers (
|
||||
satellite_id BLOB PRIMARY KEY NOT NULL,
|
||||
voucher_serialized BLOB NOT NULL,
|
||||
expiration TIMESTAMP NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE bandwidth_usage_rollups (
|
||||
interval_start TIMESTAMP NOT NULL,
|
||||
satellite_id BLOB NOT NULL,
|
||||
action INTEGER NOT NULL,
|
||||
amount BIGINT NOT NULL,
|
||||
PRIMARY KEY ( interval_start, satellite_id, action )
|
||||
);
|
||||
|
||||
INSERT INTO unsent_order VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',X'1eddef484b4c03f01332279032796972',X'0a101eddef484b4c03f0133227903279697212202b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf410001a201968996e7ef170a402fdfd88b6753df792c063c07c555905ffac9cd3cbd1c00022200ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac30002a20d00cf14f3c68b56321ace04902dec0484eb6f9098b22b31c6b3f82db249f191630643802420c08dfeb88e50510a8c1a5b9034a0c08dfeb88e50510a8c1a5b9035246304402204df59dc6f5d1bb7217105efbc9b3604d19189af37a81efbf16258e5d7db5549e02203bb4ead16e6e7f10f658558c22b59c3339911841e8dbaae6e2dea821f7326894',X'0a101eddef484b4c03f0133227903279697210321a47304502206d4c106ddec88140414bac5979c95bdea7de2e0ecc5be766e08f7d5ea36641a7022100e932ff858f15885ffa52d07e260c2c25d3861810ea6157956c1793ad0c906284','2019-04-01 16:01:35.9254586+00:00',1);
|
||||
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',0,0,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',0,0,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',1,1,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',1,1,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',2,2,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',2,2,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',3,3,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',3,3,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',4,4,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',4,4,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',5,5,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',5,5,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',6,6,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',6,6,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',1,1,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',1,1,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',2,2,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',2,2,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',3,3,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',3,3,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',4,4,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',4,4,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',5,5,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',5,5,'2019-04-01 20:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',6,6,'2019-04-01 18:51:24.1074772+00:00');
|
||||
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',6,6,'2019-04-01 20:51:24.1074772+00:00');
|
||||
|
||||
INSERT INTO vouchers VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000', X'd5e757fd8d207d1c46583fb58330f803dc961b71147308ff75ff1e72a0df6b0b', '2019-07-04 00:00:00.000000+00:00');
|
||||
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',0,0);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',0,0);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',1,1);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',1,1);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',2,2);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',2,2);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',3,3);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',3,3);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',4,4);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',4,4);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',5,5);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',5,5);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',6,6);
|
||||
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',6,6);
|
||||
|
||||
-- NEW DATA --
|
||||
@ -29,7 +29,7 @@ func (db *usedSerials) Add(ctx context.Context, satelliteID storj.NodeID, serial
|
||||
|
||||
_, err = db.db.Exec(`
|
||||
INSERT INTO
|
||||
used_serial(satellite_id, serial_number, expiration)
|
||||
used_serial_(satellite_id, serial_number, expiration)
|
||||
VALUES(?, ?, ?)`, satelliteID, serialNumber, expiration.UTC())
|
||||
|
||||
return ErrInfo.Wrap(err)
|
||||
@ -39,7 +39,7 @@ func (db *usedSerials) Add(ctx context.Context, satelliteID storj.NodeID, serial
|
||||
func (db *usedSerials) DeleteExpired(ctx context.Context, now time.Time) (err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
|
||||
_, err = db.db.Exec(`DELETE FROM used_serial WHERE expiration < ?`, now.UTC())
|
||||
_, err = db.db.Exec(`DELETE FROM used_serial_ WHERE expiration < ?`, now.UTC())
|
||||
return ErrInfo.Wrap(err)
|
||||
}
|
||||
|
||||
@ -48,7 +48,7 @@ func (db *usedSerials) DeleteExpired(ctx context.Context, now time.Time) (err er
|
||||
func (db *usedSerials) IterateAll(ctx context.Context, fn piecestore.SerialNumberFn) (err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
|
||||
rows, err := db.db.Query(`SELECT satellite_id, serial_number, expiration FROM used_serial`)
|
||||
rows, err := db.db.Query(`SELECT satellite_id, serial_number, expiration FROM used_serial_`)
|
||||
if err != nil {
|
||||
return ErrInfo.Wrap(err)
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user