From 1b5d953fd9956f5533a942a2797f509d4841b08d Mon Sep 17 00:00:00 2001 From: paul cannon Date: Tue, 1 Nov 2022 12:45:41 -0500 Subject: [PATCH] satellite/satellitedb: add queue for verifications Since the auditor will be moving to a different process from the metainfo loop, we need a way of communicating which segments have been chosen for audit. This queue will be that communication, for now. Contrast this with the queue for _re_verifications in commit 9c67f62f. Refs: https://github.com/storj/storj/issues/5251 Change-Id: I9a269c7ef21e6c5e9c6e5e1f3db298fe159a8a79 --- satellite/audit/queue.go | 10 + satellite/peer.go | 2 + satellite/satellitedb/database.go | 5 + satellite/satellitedb/dbx/satellitedb.dbx | 10 + satellite/satellitedb/dbx/satellitedb.dbx.go | 164 ++++ .../satellitedb/dbx/satellitedb.dbx.pgx.sql | 8 + .../dbx/satellitedb.dbx.pgxcockroach.sql | 8 + satellite/satellitedb/migrate.go | 15 + satellite/satellitedb/migratez.go | 10 +- .../satellitedb/testdata/postgres.v215.sql | 787 ++++++++++++++++++ satellite/satellitedb/verifyqueue.go | 105 +++ satellite/satellitedb/verifyqueue_test.go | 105 +++ 12 files changed, 1228 insertions(+), 1 deletion(-) create mode 100644 satellite/satellitedb/testdata/postgres.v215.sql create mode 100644 satellite/satellitedb/verifyqueue.go create mode 100644 satellite/satellitedb/verifyqueue_test.go diff --git a/satellite/audit/queue.go b/satellite/audit/queue.go index ac404526c..42fb09af1 100644 --- a/satellite/audit/queue.go +++ b/satellite/audit/queue.go @@ -130,6 +130,16 @@ func (queues *Queues) WaitForSwap(ctx context.Context) error { return ctx.Err() } +// VerifyQueue controls manipulation of a database-based queue of segments to be +// verified; that is, segments chosen at random from all segments on the +// satellite, for which workers should perform audits. We will try to download a +// stripe of data across all pieces in the segment and ensure that all pieces +// conform to the same polynomial. +type VerifyQueue interface { + Push(ctx context.Context, segments []Segment) (err error) + Next(ctx context.Context) (Segment, error) +} + // ReverifyQueue controls manipulation of a queue of pieces to be _re_verified; // that is, a node timed out when we requested an audit of the piece, and now // we need to follow up with that node until we get a proper answer to the diff --git a/satellite/peer.go b/satellite/peer.go index 6d2ebf496..933221a8f 100644 --- a/satellite/peer.go +++ b/satellite/peer.go @@ -100,6 +100,8 @@ type DB interface { ProjectAccounting() accounting.ProjectAccounting // RepairQueue returns queue for segments that need repairing RepairQueue() queue.RepairQueue + // VerifyQueue returns queue for segments chosen for verification + VerifyQueue() audit.VerifyQueue // ReverifyQueue returns queue for pieces that need audit reverification ReverifyQueue() audit.ReverifyQueue // Console returns database for satellite console diff --git a/satellite/satellitedb/database.go b/satellite/satellitedb/database.go index 6f080ec87..ed5c94795 100644 --- a/satellite/satellitedb/database.go +++ b/satellite/satellitedb/database.go @@ -212,6 +212,11 @@ func (dbc *satelliteDBCollection) RepairQueue() queue.RepairQueue { return &repairQueue{db: dbc.getByName("repairqueue")} } +// VerifyQueue is a getter for VerifyQueue database. +func (dbc *satelliteDBCollection) VerifyQueue() audit.VerifyQueue { + return &verifyQueue{db: dbc.getByName("verifyqueue")} +} + // ReverifyQueue is a getter for ReverifyQueue database. func (dbc *satelliteDBCollection) ReverifyQueue() audit.ReverifyQueue { return &reverifyQueue{db: dbc.getByName("reverifyqueue")} diff --git a/satellite/satellitedb/dbx/satellitedb.dbx b/satellite/satellitedb/dbx/satellitedb.dbx index 030d31140..1d9dcf107 100644 --- a/satellite/satellitedb/dbx/satellitedb.dbx +++ b/satellite/satellitedb/dbx/satellitedb.dbx @@ -69,6 +69,16 @@ read one ( where segment_pending_audits.node_id = ? ) +model verification_audits ( + key inserted_at stream_id position + + field inserted_at timestamp ( default current_timestamp ) + field stream_id blob + field position uint64 + field expires_at timestamp (nullable) + field encrypted_size int +) + model reverification_audits ( key node_id stream_id position diff --git a/satellite/satellitedb/dbx/satellitedb.dbx.go b/satellite/satellitedb/dbx/satellitedb.dbx.go index 7bcf5883e..29e7f9661 100644 --- a/satellite/satellitedb/dbx/satellitedb.dbx.go +++ b/satellite/satellitedb/dbx/satellitedb.dbx.go @@ -827,6 +827,14 @@ CREATE TABLE value_attributions ( last_updated timestamp with time zone NOT NULL, PRIMARY KEY ( project_id, bucket_name ) ); +CREATE TABLE verification_audits ( + inserted_at timestamp with time zone NOT NULL DEFAULT current_timestamp, + stream_id bytea NOT NULL, + position bigint NOT NULL, + expires_at timestamp with time zone, + encrypted_size integer NOT NULL, + PRIMARY KEY ( inserted_at, stream_id, position ) +); CREATE TABLE webapp_sessions ( id bytea NOT NULL, user_id bytea NOT NULL, @@ -1532,6 +1540,14 @@ CREATE TABLE value_attributions ( last_updated timestamp with time zone NOT NULL, PRIMARY KEY ( project_id, bucket_name ) ); +CREATE TABLE verification_audits ( + inserted_at timestamp with time zone NOT NULL DEFAULT current_timestamp, + stream_id bytea NOT NULL, + position bigint NOT NULL, + expires_at timestamp with time zone, + encrypted_size integer NOT NULL, + PRIMARY KEY ( inserted_at, stream_id, position ) +); CREATE TABLE webapp_sessions ( id bytea NOT NULL, user_id bytea NOT NULL, @@ -10752,6 +10768,134 @@ func (f ValueAttribution_LastUpdated_Field) value() interface{} { func (ValueAttribution_LastUpdated_Field) _Column() string { return "last_updated" } +type VerificationAudits struct { + InsertedAt time.Time + StreamId []byte + Position uint64 + ExpiresAt *time.Time + EncryptedSize int +} + +func (VerificationAudits) _Table() string { return "verification_audits" } + +type VerificationAudits_Create_Fields struct { + InsertedAt VerificationAudits_InsertedAt_Field + ExpiresAt VerificationAudits_ExpiresAt_Field +} + +type VerificationAudits_Update_Fields struct { +} + +type VerificationAudits_InsertedAt_Field struct { + _set bool + _null bool + _value time.Time +} + +func VerificationAudits_InsertedAt(v time.Time) VerificationAudits_InsertedAt_Field { + return VerificationAudits_InsertedAt_Field{_set: true, _value: v} +} + +func (f VerificationAudits_InsertedAt_Field) value() interface{} { + if !f._set || f._null { + return nil + } + return f._value +} + +func (VerificationAudits_InsertedAt_Field) _Column() string { return "inserted_at" } + +type VerificationAudits_StreamId_Field struct { + _set bool + _null bool + _value []byte +} + +func VerificationAudits_StreamId(v []byte) VerificationAudits_StreamId_Field { + return VerificationAudits_StreamId_Field{_set: true, _value: v} +} + +func (f VerificationAudits_StreamId_Field) value() interface{} { + if !f._set || f._null { + return nil + } + return f._value +} + +func (VerificationAudits_StreamId_Field) _Column() string { return "stream_id" } + +type VerificationAudits_Position_Field struct { + _set bool + _null bool + _value uint64 +} + +func VerificationAudits_Position(v uint64) VerificationAudits_Position_Field { + return VerificationAudits_Position_Field{_set: true, _value: v} +} + +func (f VerificationAudits_Position_Field) value() interface{} { + if !f._set || f._null { + return nil + } + return f._value +} + +func (VerificationAudits_Position_Field) _Column() string { return "position" } + +type VerificationAudits_ExpiresAt_Field struct { + _set bool + _null bool + _value *time.Time +} + +func VerificationAudits_ExpiresAt(v time.Time) VerificationAudits_ExpiresAt_Field { + return VerificationAudits_ExpiresAt_Field{_set: true, _value: &v} +} + +func VerificationAudits_ExpiresAt_Raw(v *time.Time) VerificationAudits_ExpiresAt_Field { + if v == nil { + return VerificationAudits_ExpiresAt_Null() + } + return VerificationAudits_ExpiresAt(*v) +} + +func VerificationAudits_ExpiresAt_Null() VerificationAudits_ExpiresAt_Field { + return VerificationAudits_ExpiresAt_Field{_set: true, _null: true} +} + +func (f VerificationAudits_ExpiresAt_Field) isnull() bool { + return !f._set || f._null || f._value == nil +} + +func (f VerificationAudits_ExpiresAt_Field) value() interface{} { + if !f._set || f._null { + return nil + } + return f._value +} + +func (VerificationAudits_ExpiresAt_Field) _Column() string { return "expires_at" } + +type VerificationAudits_EncryptedSize_Field struct { + _set bool + _null bool + _value int +} + +func VerificationAudits_EncryptedSize(v int) VerificationAudits_EncryptedSize_Field { + return VerificationAudits_EncryptedSize_Field{_set: true, _value: v} +} + +func (f VerificationAudits_EncryptedSize_Field) value() interface{} { + if !f._set || f._null { + return nil + } + return f._value +} + +func (VerificationAudits_EncryptedSize_Field) _Column() string { return "encrypted_size" } + type WebappSession struct { Id []byte UserId []byte @@ -19701,6 +19845,16 @@ func (obj *pgxImpl) deleteAll(ctx context.Context) (count int64, err error) { return 0, obj.makeErr(err) } + __count, err = __res.RowsAffected() + if err != nil { + return 0, obj.makeErr(err) + } + count += __count + __res, err = obj.driver.ExecContext(ctx, "DELETE FROM verification_audits;") + if err != nil { + return 0, obj.makeErr(err) + } + __count, err = __res.RowsAffected() if err != nil { return 0, obj.makeErr(err) @@ -27437,6 +27591,16 @@ func (obj *pgxcockroachImpl) deleteAll(ctx context.Context) (count int64, err er return 0, obj.makeErr(err) } + __count, err = __res.RowsAffected() + if err != nil { + return 0, obj.makeErr(err) + } + count += __count + __res, err = obj.driver.ExecContext(ctx, "DELETE FROM verification_audits;") + if err != nil { + return 0, obj.makeErr(err) + } + __count, err = __res.RowsAffected() if err != nil { return 0, obj.makeErr(err) diff --git a/satellite/satellitedb/dbx/satellitedb.dbx.pgx.sql b/satellite/satellitedb/dbx/satellitedb.dbx.pgx.sql index cc9348756..c63ac143a 100644 --- a/satellite/satellitedb/dbx/satellitedb.dbx.pgx.sql +++ b/satellite/satellitedb/dbx/satellitedb.dbx.pgx.sql @@ -507,6 +507,14 @@ CREATE TABLE value_attributions ( last_updated timestamp with time zone NOT NULL, PRIMARY KEY ( project_id, bucket_name ) ); +CREATE TABLE verification_audits ( + inserted_at timestamp with time zone NOT NULL DEFAULT current_timestamp, + stream_id bytea NOT NULL, + position bigint NOT NULL, + expires_at timestamp with time zone, + encrypted_size integer NOT NULL, + PRIMARY KEY ( inserted_at, stream_id, position ) +); CREATE TABLE webapp_sessions ( id bytea NOT NULL, user_id bytea NOT NULL, diff --git a/satellite/satellitedb/dbx/satellitedb.dbx.pgxcockroach.sql b/satellite/satellitedb/dbx/satellitedb.dbx.pgxcockroach.sql index cc9348756..c63ac143a 100644 --- a/satellite/satellitedb/dbx/satellitedb.dbx.pgxcockroach.sql +++ b/satellite/satellitedb/dbx/satellitedb.dbx.pgxcockroach.sql @@ -507,6 +507,14 @@ CREATE TABLE value_attributions ( last_updated timestamp with time zone NOT NULL, PRIMARY KEY ( project_id, bucket_name ) ); +CREATE TABLE verification_audits ( + inserted_at timestamp with time zone NOT NULL DEFAULT current_timestamp, + stream_id bytea NOT NULL, + position bigint NOT NULL, + expires_at timestamp with time zone, + encrypted_size integer NOT NULL, + PRIMARY KEY ( inserted_at, stream_id, position ) +); CREATE TABLE webapp_sessions ( id bytea NOT NULL, user_id bytea NOT NULL, diff --git a/satellite/satellitedb/migrate.go b/satellite/satellitedb/migrate.go index d4161f5af..ba3c8788e 100644 --- a/satellite/satellitedb/migrate.go +++ b/satellite/satellitedb/migrate.go @@ -2133,6 +2133,21 @@ func (db *satelliteDB) PostgresMigration() *migrate.Migration { WHERE email_sent IS NULL;`, }, }, + { + DB: &db.migrationDB, + Description: "Create table for verification queue", + Version: 215, + Action: migrate.SQL{ + `CREATE TABLE verification_audits ( + inserted_at timestamp with time zone NOT NULL DEFAULT current_timestamp, + stream_id bytea NOT NULL, + position bigint NOT NULL, + expires_at timestamp with time zone, + encrypted_size integer NOT NULL, + PRIMARY KEY ( inserted_at, stream_id, position ) + );`, + }, + }, // NB: after updating testdata in `testdata`, run // `go generate` to update `migratez.go`. }, diff --git a/satellite/satellitedb/migratez.go b/satellite/satellitedb/migratez.go index 22b77a000..74bb28229 100755 --- a/satellite/satellitedb/migratez.go +++ b/satellite/satellitedb/migratez.go @@ -13,7 +13,7 @@ func (db *satelliteDB) testMigration() *migrate.Migration { { DB: &db.migrationDB, Description: "Testing setup", - Version: 214, + Version: 215, Action: migrate.SQL{`-- AUTOGENERATED BY storj.io/dbx -- DO NOT EDIT CREATE TABLE accounting_rollups ( @@ -340,6 +340,14 @@ CREATE TABLE reset_password_tokens ( PRIMARY KEY ( secret ), UNIQUE ( owner_id ) ); +CREATE TABLE verification_audits ( + inserted_at timestamp with time zone NOT NULL DEFAULT current_timestamp, + stream_id bytea NOT NULL, + position bigint NOT NULL, + expires_at timestamp with time zone, + encrypted_size integer NOT NULL, + PRIMARY KEY ( inserted_at, stream_id, position ) +); CREATE TABLE reverification_audits ( node_id bytea NOT NULL, stream_id bytea NOT NULL, diff --git a/satellite/satellitedb/testdata/postgres.v215.sql b/satellite/satellitedb/testdata/postgres.v215.sql new file mode 100644 index 000000000..ae7a73077 --- /dev/null +++ b/satellite/satellitedb/testdata/postgres.v215.sql @@ -0,0 +1,787 @@ +-- AUTOGENERATED BY storj.io/dbx +-- DO NOT EDIT +CREATE TABLE accounting_rollups ( + node_id bytea NOT NULL, + start_time timestamp with time zone NOT NULL, + put_total bigint NOT NULL, + get_total bigint NOT NULL, + get_audit_total bigint NOT NULL, + get_repair_total bigint NOT NULL, + put_repair_total bigint NOT NULL, + at_rest_total double precision NOT NULL, + interval_end_time timestamp with time zone, + PRIMARY KEY ( node_id, start_time ) +); +CREATE TABLE accounting_timestamps ( + name text NOT NULL, + value timestamp with time zone NOT NULL, + PRIMARY KEY ( name ) +); +CREATE TABLE billing_balances ( + user_id bytea NOT NULL, + balance bigint NOT NULL, + last_updated timestamp with time zone NOT NULL, + PRIMARY KEY ( user_id ) +); +CREATE TABLE billing_transactions ( + id bigserial NOT NULL, + user_id bytea NOT NULL, + amount bigint NOT NULL, + currency text NOT NULL, + description text NOT NULL, + source text NOT NULL, + status text NOT NULL, + type text NOT NULL, + metadata jsonb NOT NULL, + timestamp timestamp with time zone NOT NULL, + created_at timestamp with time zone NOT NULL, + PRIMARY KEY ( id ) +); +CREATE TABLE bucket_bandwidth_rollups ( + bucket_name bytea NOT NULL, + project_id bytea NOT NULL, + interval_start timestamp with time zone NOT NULL, + interval_seconds integer NOT NULL, + action integer NOT NULL, + inline bigint NOT NULL, + allocated bigint NOT NULL, + settled bigint NOT NULL, + PRIMARY KEY ( bucket_name, project_id, interval_start, action ) +); +CREATE TABLE bucket_bandwidth_rollup_archives ( + bucket_name bytea NOT NULL, + project_id bytea NOT NULL, + interval_start timestamp with time zone NOT NULL, + interval_seconds integer NOT NULL, + action integer NOT NULL, + inline bigint NOT NULL, + allocated bigint NOT NULL, + settled bigint NOT NULL, + PRIMARY KEY ( bucket_name, project_id, interval_start, action ) +); +CREATE TABLE bucket_storage_tallies ( + bucket_name bytea NOT NULL, + project_id bytea NOT NULL, + interval_start timestamp with time zone NOT NULL, + total_bytes bigint NOT NULL DEFAULT 0, + inline bigint NOT NULL, + remote bigint NOT NULL, + total_segments_count integer NOT NULL DEFAULT 0, + remote_segments_count integer NOT NULL, + inline_segments_count integer NOT NULL, + object_count integer NOT NULL, + metadata_size bigint NOT NULL, + PRIMARY KEY ( bucket_name, project_id, interval_start ) +); +CREATE TABLE coinpayments_transactions ( + id text NOT NULL, + user_id bytea NOT NULL, + address text NOT NULL, + amount_numeric bigint NOT NULL, + received_numeric bigint NOT NULL, + status integer NOT NULL, + key text NOT NULL, + timeout integer NOT NULL, + created_at timestamp with time zone NOT NULL, + PRIMARY KEY ( id ) +); +CREATE TABLE coupons ( + id bytea NOT NULL, + user_id bytea NOT NULL, + amount bigint NOT NULL, + description text NOT NULL, + type integer NOT NULL, + status integer NOT NULL, + duration bigint NOT NULL, + billing_periods bigint, + coupon_code_name text, + created_at timestamp with time zone NOT NULL, + PRIMARY KEY ( id ) +); +CREATE TABLE coupon_codes ( + id bytea NOT NULL, + name text NOT NULL, + amount bigint NOT NULL, + description text NOT NULL, + type integer NOT NULL, + billing_periods bigint, + created_at timestamp with time zone NOT NULL, + PRIMARY KEY ( id ), + UNIQUE ( name ) +); +CREATE TABLE coupon_usages ( + coupon_id bytea NOT NULL, + amount bigint NOT NULL, + status integer NOT NULL, + period timestamp with time zone NOT NULL, + PRIMARY KEY ( coupon_id, period ) +); +CREATE TABLE graceful_exit_progress ( + node_id bytea NOT NULL, + bytes_transferred bigint NOT NULL, + pieces_transferred bigint NOT NULL DEFAULT 0, + pieces_failed bigint NOT NULL DEFAULT 0, + updated_at timestamp with time zone NOT NULL, + PRIMARY KEY ( node_id ) +); +CREATE TABLE graceful_exit_segment_transfer_queue ( + node_id bytea NOT NULL, + stream_id bytea NOT NULL, + position bigint NOT NULL, + piece_num integer NOT NULL, + root_piece_id bytea, + durability_ratio double precision NOT NULL, + queued_at timestamp with time zone NOT NULL, + requested_at timestamp with time zone, + last_failed_at timestamp with time zone, + last_failed_code integer, + failed_count integer, + finished_at timestamp with time zone, + order_limit_send_count integer NOT NULL DEFAULT 0, + PRIMARY KEY ( node_id, stream_id, position, piece_num ) +); +CREATE TABLE nodes ( + id bytea NOT NULL, + address text NOT NULL DEFAULT '', + last_net text NOT NULL, + last_ip_port text, + country_code text, + protocol integer NOT NULL DEFAULT 0, + type integer NOT NULL DEFAULT 0, + email text NOT NULL, + wallet text NOT NULL, + wallet_features text NOT NULL DEFAULT '', + free_disk bigint NOT NULL DEFAULT -1, + piece_count bigint NOT NULL DEFAULT 0, + major bigint NOT NULL DEFAULT 0, + minor bigint NOT NULL DEFAULT 0, + patch bigint NOT NULL DEFAULT 0, + hash text NOT NULL DEFAULT '', + timestamp timestamp with time zone NOT NULL DEFAULT '0001-01-01 00:00:00+00', + release boolean NOT NULL DEFAULT false, + latency_90 bigint NOT NULL DEFAULT 0, + vetted_at timestamp with time zone, + created_at timestamp with time zone NOT NULL DEFAULT current_timestamp, + updated_at timestamp with time zone NOT NULL DEFAULT current_timestamp, + last_contact_success timestamp with time zone NOT NULL DEFAULT 'epoch', + last_contact_failure timestamp with time zone NOT NULL DEFAULT 'epoch', + disqualified timestamp with time zone, + disqualification_reason integer, + unknown_audit_suspended timestamp with time zone, + offline_suspended timestamp with time zone, + under_review timestamp with time zone, + exit_initiated_at timestamp with time zone, + exit_loop_completed_at timestamp with time zone, + exit_finished_at timestamp with time zone, + exit_success boolean NOT NULL DEFAULT false, + PRIMARY KEY ( id ) +); +CREATE TABLE node_events ( + id bytea NOT NULL, + email text NOT NULL, + node_id bytea NOT NULL, + event integer NOT NULL, + created_at timestamp with time zone NOT NULL DEFAULT current_timestamp, + email_sent timestamp with time zone, + PRIMARY KEY ( id ) +); +CREATE TABLE node_api_versions ( + id bytea NOT NULL, + api_version integer NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + PRIMARY KEY ( id ) +); +CREATE TABLE oauth_clients ( + id bytea NOT NULL, + encrypted_secret bytea NOT NULL, + redirect_url text NOT NULL, + user_id bytea NOT NULL, + app_name text NOT NULL, + app_logo_url text NOT NULL, + PRIMARY KEY ( id ) +); +CREATE TABLE oauth_codes ( + client_id bytea NOT NULL, + user_id bytea NOT NULL, + scope text NOT NULL, + redirect_url text NOT NULL, + challenge text NOT NULL, + challenge_method text NOT NULL, + code text NOT NULL, + created_at timestamp with time zone NOT NULL, + expires_at timestamp with time zone NOT NULL, + claimed_at timestamp with time zone, + PRIMARY KEY ( code ) +); +CREATE TABLE oauth_tokens ( + client_id bytea NOT NULL, + user_id bytea NOT NULL, + scope text NOT NULL, + kind integer NOT NULL, + token bytea NOT NULL, + created_at timestamp with time zone NOT NULL, + expires_at timestamp with time zone NOT NULL, + PRIMARY KEY ( token ) +); +CREATE TABLE offers ( + id serial NOT NULL, + name text NOT NULL, + description text NOT NULL, + award_credit_in_cents integer NOT NULL DEFAULT 0, + invitee_credit_in_cents integer NOT NULL DEFAULT 0, + award_credit_duration_days integer, + invitee_credit_duration_days integer, + redeemable_cap integer, + expires_at timestamp with time zone NOT NULL, + created_at timestamp with time zone NOT NULL, + status integer NOT NULL, + type integer NOT NULL, + PRIMARY KEY ( id ) +); +CREATE TABLE peer_identities ( + node_id bytea NOT NULL, + leaf_serial_number bytea NOT NULL, + chain bytea NOT NULL, + updated_at timestamp with time zone NOT NULL, + PRIMARY KEY ( node_id ) +); +CREATE TABLE projects ( + id bytea NOT NULL, + public_id bytea, + name text NOT NULL, + description text NOT NULL, + usage_limit bigint, + bandwidth_limit bigint, + user_specified_usage_limit bigint, + user_specified_bandwidth_limit bigint, + segment_limit bigint DEFAULT 1000000, + rate_limit integer, + burst_limit integer, + max_buckets integer, + partner_id bytea, + user_agent bytea, + owner_id bytea NOT NULL, + salt bytea, + created_at timestamp with time zone NOT NULL, + PRIMARY KEY ( id ) +); +CREATE TABLE project_bandwidth_daily_rollups ( + project_id bytea NOT NULL, + interval_day date NOT NULL, + egress_allocated bigint NOT NULL, + egress_settled bigint NOT NULL, + egress_dead bigint NOT NULL DEFAULT 0, + PRIMARY KEY ( project_id, interval_day ) +); +CREATE TABLE project_bandwidth_rollups ( + project_id bytea NOT NULL, + interval_month date NOT NULL, + egress_allocated bigint NOT NULL, + PRIMARY KEY ( project_id, interval_month ) +); +CREATE TABLE registration_tokens ( + secret bytea NOT NULL, + owner_id bytea, + project_limit integer NOT NULL, + created_at timestamp with time zone NOT NULL, + PRIMARY KEY ( secret ), + UNIQUE ( owner_id ) +); +CREATE TABLE repair_queue ( + stream_id bytea NOT NULL, + position bigint NOT NULL, + attempted_at timestamp with time zone, + updated_at timestamp with time zone NOT NULL DEFAULT current_timestamp, + inserted_at timestamp with time zone NOT NULL DEFAULT current_timestamp, + segment_health double precision NOT NULL DEFAULT 1, + PRIMARY KEY ( stream_id, position ) +); +CREATE TABLE reputations ( + id bytea NOT NULL, + audit_success_count bigint NOT NULL DEFAULT 0, + total_audit_count bigint NOT NULL DEFAULT 0, + vetted_at timestamp with time zone, + created_at timestamp with time zone NOT NULL DEFAULT current_timestamp, + updated_at timestamp with time zone NOT NULL DEFAULT current_timestamp, + disqualified timestamp with time zone, + disqualification_reason integer, + unknown_audit_suspended timestamp with time zone, + offline_suspended timestamp with time zone, + under_review timestamp with time zone, + online_score double precision NOT NULL DEFAULT 1, + audit_history bytea NOT NULL, + audit_reputation_alpha double precision NOT NULL DEFAULT 1, + audit_reputation_beta double precision NOT NULL DEFAULT 0, + unknown_audit_reputation_alpha double precision NOT NULL DEFAULT 1, + unknown_audit_reputation_beta double precision NOT NULL DEFAULT 0, + PRIMARY KEY ( id ) +); +CREATE TABLE reset_password_tokens ( + secret bytea NOT NULL, + owner_id bytea NOT NULL, + created_at timestamp with time zone NOT NULL, + PRIMARY KEY ( secret ), + UNIQUE ( owner_id ) +); +CREATE TABLE verification_audits ( + inserted_at timestamp with time zone NOT NULL DEFAULT current_timestamp, + stream_id bytea NOT NULL, + position bigint NOT NULL, + expires_at timestamp with time zone, + encrypted_size integer NOT NULL, + PRIMARY KEY ( inserted_at, stream_id, position ) +); +CREATE TABLE reverification_audits ( + node_id bytea NOT NULL, + stream_id bytea NOT NULL, + position bigint NOT NULL, + piece_num integer NOT NULL, + inserted_at timestamp with time zone NOT NULL DEFAULT current_timestamp, + last_attempt timestamp with time zone, + reverify_count bigint NOT NULL DEFAULT 0, + PRIMARY KEY ( node_id, stream_id, position ) +); +CREATE TABLE revocations ( + revoked bytea NOT NULL, + api_key_id bytea NOT NULL, + PRIMARY KEY ( revoked ) +); +CREATE TABLE segment_pending_audits ( + node_id bytea NOT NULL, + stream_id bytea NOT NULL, + position bigint NOT NULL, + piece_id bytea NOT NULL, + stripe_index bigint NOT NULL, + share_size bigint NOT NULL, + expected_share_hash bytea NOT NULL, + reverify_count bigint NOT NULL, + PRIMARY KEY ( node_id ) +); +CREATE TABLE storagenode_bandwidth_rollups ( + storagenode_id bytea NOT NULL, + interval_start timestamp with time zone NOT NULL, + interval_seconds integer NOT NULL, + action integer NOT NULL, + allocated bigint DEFAULT 0, + settled bigint NOT NULL, + PRIMARY KEY ( storagenode_id, interval_start, action ) +); +CREATE TABLE storagenode_bandwidth_rollup_archives ( + storagenode_id bytea NOT NULL, + interval_start timestamp with time zone NOT NULL, + interval_seconds integer NOT NULL, + action integer NOT NULL, + allocated bigint DEFAULT 0, + settled bigint NOT NULL, + PRIMARY KEY ( storagenode_id, interval_start, action ) +); +CREATE TABLE storagenode_bandwidth_rollups_phase2 ( + storagenode_id bytea NOT NULL, + interval_start timestamp with time zone NOT NULL, + interval_seconds integer NOT NULL, + action integer NOT NULL, + allocated bigint DEFAULT 0, + settled bigint NOT NULL, + PRIMARY KEY ( storagenode_id, interval_start, action ) +); +CREATE TABLE storagenode_payments ( + id bigserial NOT NULL, + created_at timestamp with time zone NOT NULL, + node_id bytea NOT NULL, + period text NOT NULL, + amount bigint NOT NULL, + receipt text, + notes text, + PRIMARY KEY ( id ) +); +CREATE TABLE storagenode_paystubs ( + period text NOT NULL, + node_id bytea NOT NULL, + created_at timestamp with time zone NOT NULL, + codes text NOT NULL, + usage_at_rest double precision NOT NULL, + usage_get bigint NOT NULL, + usage_put bigint NOT NULL, + usage_get_repair bigint NOT NULL, + usage_put_repair bigint NOT NULL, + usage_get_audit bigint NOT NULL, + comp_at_rest bigint NOT NULL, + comp_get bigint NOT NULL, + comp_put bigint NOT NULL, + comp_get_repair bigint NOT NULL, + comp_put_repair bigint NOT NULL, + comp_get_audit bigint NOT NULL, + surge_percent bigint NOT NULL, + held bigint NOT NULL, + owed bigint NOT NULL, + disposed bigint NOT NULL, + paid bigint NOT NULL, + distributed bigint NOT NULL, + PRIMARY KEY ( period, node_id ) +); +CREATE TABLE storagenode_storage_tallies ( + node_id bytea NOT NULL, + interval_end_time timestamp with time zone NOT NULL, + data_total double precision NOT NULL, + PRIMARY KEY ( interval_end_time, node_id ) +); +CREATE TABLE storjscan_payments ( + block_hash bytea NOT NULL, + block_number bigint NOT NULL, + transaction bytea NOT NULL, + log_index integer NOT NULL, + from_address bytea NOT NULL, + to_address bytea NOT NULL, + token_value bigint NOT NULL, + usd_value bigint NOT NULL, + status text NOT NULL, + timestamp timestamp with time zone NOT NULL, + created_at timestamp with time zone NOT NULL, + PRIMARY KEY ( block_hash, log_index ) +); +CREATE TABLE storjscan_wallets ( + user_id bytea NOT NULL, + wallet_address bytea NOT NULL, + created_at timestamp with time zone NOT NULL, + PRIMARY KEY ( user_id, wallet_address ) +); +CREATE TABLE stripe_customers ( + user_id bytea NOT NULL, + customer_id text NOT NULL, + created_at timestamp with time zone NOT NULL, + PRIMARY KEY ( user_id ), + UNIQUE ( customer_id ) +); +CREATE TABLE stripecoinpayments_invoice_project_records ( + id bytea NOT NULL, + project_id bytea NOT NULL, + storage double precision NOT NULL, + egress bigint NOT NULL, + objects bigint, + segments bigint, + period_start timestamp with time zone NOT NULL, + period_end timestamp with time zone NOT NULL, + state integer NOT NULL, + created_at timestamp with time zone NOT NULL, + PRIMARY KEY ( id ), + UNIQUE ( project_id, period_start, period_end ) +); +CREATE TABLE stripecoinpayments_tx_conversion_rates ( + tx_id text NOT NULL, + rate_numeric double precision NOT NULL, + created_at timestamp with time zone NOT NULL, + PRIMARY KEY ( tx_id ) +); +CREATE TABLE users ( + id bytea NOT NULL, + email text NOT NULL, + normalized_email text NOT NULL, + full_name text NOT NULL, + short_name text, + password_hash bytea NOT NULL, + status integer NOT NULL, + partner_id bytea, + user_agent bytea, + created_at timestamp with time zone NOT NULL, + project_limit integer NOT NULL DEFAULT 0, + project_bandwidth_limit bigint NOT NULL DEFAULT 0, + project_storage_limit bigint NOT NULL DEFAULT 0, + project_segment_limit bigint NOT NULL DEFAULT 0, + paid_tier boolean NOT NULL DEFAULT false, + position text, + company_name text, + company_size integer, + working_on text, + is_professional boolean NOT NULL DEFAULT false, + employee_count text, + have_sales_contact boolean NOT NULL DEFAULT false, + mfa_enabled boolean NOT NULL DEFAULT false, + mfa_secret_key text, + mfa_recovery_codes text, + signup_promo_code text, + last_verification_reminder timestamp with time zone, + verification_reminders integer NOT NULL DEFAULT 0, + failed_login_count integer, + login_lockout_expiration timestamp with time zone, + signup_captcha double precision, + PRIMARY KEY ( id ) +); +CREATE TABLE value_attributions ( + project_id bytea NOT NULL, + bucket_name bytea NOT NULL, + partner_id bytea NOT NULL, + user_agent bytea, + last_updated timestamp with time zone NOT NULL, + PRIMARY KEY ( project_id, bucket_name ) +); +CREATE TABLE webapp_sessions ( + id bytea NOT NULL, + user_id bytea NOT NULL, + ip_address text NOT NULL, + user_agent text NOT NULL, + status integer NOT NULL, + expires_at timestamp with time zone NOT NULL, + PRIMARY KEY ( id ) +); +CREATE TABLE api_keys ( + id bytea NOT NULL, + project_id bytea NOT NULL REFERENCES projects( id ) ON DELETE CASCADE, + head bytea NOT NULL, + name text NOT NULL, + secret bytea NOT NULL, + partner_id bytea, + user_agent bytea, + created_at timestamp with time zone NOT NULL, + PRIMARY KEY ( id ), + UNIQUE ( head ), + UNIQUE ( name, project_id ) +); +CREATE TABLE bucket_metainfos ( + id bytea NOT NULL, + project_id bytea NOT NULL REFERENCES projects( id ), + name bytea NOT NULL, + partner_id bytea, + user_agent bytea, + path_cipher integer NOT NULL, + created_at timestamp with time zone NOT NULL, + default_segment_size integer NOT NULL, + default_encryption_cipher_suite integer NOT NULL, + default_encryption_block_size integer NOT NULL, + default_redundancy_algorithm integer NOT NULL, + default_redundancy_share_size integer NOT NULL, + default_redundancy_required_shares integer NOT NULL, + default_redundancy_repair_shares integer NOT NULL, + default_redundancy_optimal_shares integer NOT NULL, + default_redundancy_total_shares integer NOT NULL, + placement integer, + PRIMARY KEY ( id ), + UNIQUE ( project_id, name ) +); +CREATE TABLE project_members ( + member_id bytea NOT NULL REFERENCES users( id ) ON DELETE CASCADE, + project_id bytea NOT NULL REFERENCES projects( id ) ON DELETE CASCADE, + created_at timestamp with time zone NOT NULL, + PRIMARY KEY ( member_id, project_id ) +); +CREATE TABLE stripecoinpayments_apply_balance_intents ( + tx_id text NOT NULL REFERENCES coinpayments_transactions( id ) ON DELETE CASCADE, + state integer NOT NULL, + created_at timestamp with time zone NOT NULL, + PRIMARY KEY ( tx_id ) +); +CREATE TABLE user_credits ( + id serial NOT NULL, + user_id bytea NOT NULL REFERENCES users( id ) ON DELETE CASCADE, + offer_id integer NOT NULL REFERENCES offers( id ), + referred_by bytea REFERENCES users( id ) ON DELETE SET NULL, + type text NOT NULL, + credits_earned_in_cents integer NOT NULL, + credits_used_in_cents integer NOT NULL, + expires_at timestamp with time zone NOT NULL, + created_at timestamp with time zone NOT NULL, + PRIMARY KEY ( id ), + UNIQUE ( id, offer_id ) +); +CREATE INDEX accounting_rollups_start_time_index ON accounting_rollups ( start_time ) ; +CREATE INDEX billing_transactions_timestamp_index ON billing_transactions ( timestamp ) ; +CREATE INDEX bucket_bandwidth_rollups_project_id_action_interval_index ON bucket_bandwidth_rollups ( project_id, action, interval_start ) ; +CREATE INDEX bucket_bandwidth_rollups_action_interval_project_id_index ON bucket_bandwidth_rollups ( action, interval_start, project_id ) ; +CREATE INDEX bucket_bandwidth_rollups_archive_project_id_action_interval_index ON bucket_bandwidth_rollup_archives ( project_id, action, interval_start ) ; +CREATE INDEX bucket_bandwidth_rollups_archive_action_interval_project_id_index ON bucket_bandwidth_rollup_archives ( action, interval_start, project_id ) ; +CREATE INDEX bucket_storage_tallies_project_id_interval_start_index ON bucket_storage_tallies ( project_id, interval_start ) ; +CREATE INDEX graceful_exit_segment_transfer_nid_dr_qa_fa_lfa_index ON graceful_exit_segment_transfer_queue ( node_id, durability_ratio, queued_at, finished_at, last_failed_at ) ; +CREATE INDEX node_last_ip ON nodes ( last_net ) ; +CREATE INDEX nodes_dis_unk_off_exit_fin_last_success_index ON nodes ( disqualified, unknown_audit_suspended, offline_suspended, exit_finished_at, last_contact_success ) ; +CREATE INDEX nodes_type_last_cont_success_free_disk_ma_mi_patch_vetted_partial_index ON nodes ( type, last_contact_success, free_disk, major, minor, patch, vetted_at ) WHERE nodes.disqualified is NULL AND nodes.unknown_audit_suspended is NULL AND nodes.exit_initiated_at is NULL AND nodes.release = true AND nodes.last_net != '' ; +CREATE INDEX nodes_dis_unk_aud_exit_init_rel_type_last_cont_success_stored_index ON nodes ( disqualified, unknown_audit_suspended, exit_initiated_at, release, type, last_contact_success ) WHERE nodes.disqualified is NULL AND nodes.unknown_audit_suspended is NULL AND nodes.exit_initiated_at is NULL AND nodes.release = true ; +CREATE INDEX node_events_email_event_created_at_index ON node_events ( email, event, created_at ) WHERE node_events.email_sent IS NULL ; +CREATE INDEX oauth_clients_user_id_index ON oauth_clients ( user_id ) ; +CREATE INDEX oauth_codes_user_id_index ON oauth_codes ( user_id ) ; +CREATE INDEX oauth_codes_client_id_index ON oauth_codes ( client_id ) ; +CREATE INDEX oauth_tokens_user_id_index ON oauth_tokens ( user_id ) ; +CREATE INDEX oauth_tokens_client_id_index ON oauth_tokens ( client_id ) ; +CREATE INDEX projects_public_id_index ON projects ( public_id ) ; +CREATE INDEX repair_queue_updated_at_index ON repair_queue ( updated_at ) ; +CREATE INDEX repair_queue_num_healthy_pieces_attempted_at_index ON repair_queue ( segment_health, attempted_at ) ; +CREATE INDEX reverification_audits_inserted_at_index ON reverification_audits ( inserted_at ) ; +CREATE INDEX storagenode_bandwidth_rollups_interval_start_index ON storagenode_bandwidth_rollups ( interval_start ) ; +CREATE INDEX storagenode_bandwidth_rollup_archives_interval_start_index ON storagenode_bandwidth_rollup_archives ( interval_start ) ; +CREATE INDEX storagenode_payments_node_id_period_index ON storagenode_payments ( node_id, period ) ; +CREATE INDEX storagenode_paystubs_node_id_index ON storagenode_paystubs ( node_id ) ; +CREATE INDEX storagenode_storage_tallies_node_id_index ON storagenode_storage_tallies ( node_id ) ; +CREATE INDEX storjscan_payments_block_number_log_index_index ON storjscan_payments ( block_number, log_index ) ; +CREATE INDEX storjscan_wallets_wallet_address_index ON storjscan_wallets ( wallet_address ) ; +CREATE INDEX webapp_sessions_user_id_index ON webapp_sessions ( user_id ) ; +CREATE UNIQUE INDEX credits_earned_user_id_offer_id ON user_credits ( id, offer_id ) ; + +INSERT INTO "offers" ("id", "name", "description", "award_credit_in_cents", "invitee_credit_in_cents", "expires_at", "created_at", "status", "type", "award_credit_duration_days", "invitee_credit_duration_days") VALUES (1, 'Default referral offer', 'Is active when no other active referral offer', 300, 600, '2119-03-14 08:28:24.636949+00', '2019-07-14 08:28:24.636949+00', 1, 2, 365, 14); +INSERT INTO "offers" ("id", "name", "description", "award_credit_in_cents", "invitee_credit_in_cents", "expires_at", "created_at", "status", "type", "award_credit_duration_days", "invitee_credit_duration_days") VALUES (2, 'Default free credit offer', 'Is active when no active free credit offer', 0, 300, '2119-03-14 08:28:24.636949+00', '2019-07-14 08:28:24.636949+00', 1, 1, NULL, 14); + +-- MAIN DATA -- + +INSERT INTO "accounting_rollups"("node_id", "start_time", "put_total", "get_total", "get_audit_total", "get_repair_total", "put_repair_total", "at_rest_total") VALUES (E'\\367M\\177\\251]t/\\022\\256\\214\\265\\025\\224\\204:\\217\\212\\0102<\\321\\374\\020&\\271Qc\\325\\261\\354\\246\\233'::bytea, '2019-02-09 00:00:00+00', 3000, 6000, 9000, 12000, 0, 15000); + +INSERT INTO "accounting_timestamps" VALUES ('LastAtRestTally', '0001-01-01 00:00:00+00'); +INSERT INTO "accounting_timestamps" VALUES ('LastRollup', '0001-01-01 00:00:00+00'); +INSERT INTO "accounting_timestamps" VALUES ('LastBandwidthTally', '0001-01-01 00:00:00+00'); + +INSERT INTO "nodes"("id", "address", "last_net", "protocol", "type", "email", "wallet", "free_disk", "piece_count", "major", "minor", "patch", "hash", "timestamp", "release","latency_90", "created_at", "updated_at", "last_contact_success", "last_contact_failure", "disqualified", "disqualification_reason", "exit_success") VALUES (E'\\153\\313\\233\\074\\327\\177\\136\\070\\346\\001', '127.0.0.1:55516', '', 0, 4, '', '', -1, 0, 0, 1, 0, '', 'epoch', false, 0, '2019-02-14 08:07:31.028103+00', '2019-02-14 08:07:31.108963+00', 'epoch', 'epoch', NULL, NULL, false); +INSERT INTO "nodes"("id", "address", "last_net", "protocol", "type", "email", "wallet", "free_disk", "piece_count", "major", "minor", "patch", "hash", "timestamp", "release","latency_90","created_at", "updated_at", "last_contact_success", "last_contact_failure", "disqualified", "disqualification_reason", "exit_success") VALUES (E'\\006\\223\\250R\\221\\005\\365\\377v>0\\266\\365\\216\\255?\\347\\244\\371?2\\264\\262\\230\\007<\\001\\262\\263\\237\\247n', '127.0.0.1:55518', '', 0, 4, '', '', -1, 0, 0, 1, 0, '', 'epoch', false, 0, '2019-02-14 08:07:31.028103+00', '2019-02-14 08:07:31.108963+00', 'epoch', 'epoch', NULL, NULL, false); +INSERT INTO "nodes"("id", "address", "last_net", "protocol", "type", "email", "wallet", "free_disk", "piece_count", "major", "minor", "patch", "hash", "timestamp", "release","latency_90","created_at", "updated_at", "last_contact_success", "last_contact_failure", "disqualified", "disqualification_reason", "exit_success") VALUES (E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\014', '127.0.0.1:55517', '', 0, 4, '', '', -1, 0, 0, 1, 0, '', 'epoch', false, 0, '2019-02-14 08:07:31.028103+00', '2019-02-14 08:07:31.108963+00', 'epoch', 'epoch', NULL, NULL, false); +INSERT INTO "nodes"("id", "address", "last_net", "protocol", "type", "email", "wallet", "free_disk", "piece_count", "major", "minor", "patch", "hash", "timestamp", "release","latency_90","created_at", "updated_at", "last_contact_success", "last_contact_failure", "disqualified", "disqualification_reason", "exit_success") VALUES (E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\015', '127.0.0.1:55519', '', 0, 4, '', '', -1, 0, 0, 1, 0, '', 'epoch', false, 0, '2019-02-14 08:07:31.028103+00', '2019-02-14 08:07:31.108963+00', 'epoch', 'epoch', NULL, NULL, false); +INSERT INTO "nodes"("id", "address", "last_net", "protocol", "type", "email", "wallet", "free_disk", "piece_count", "major", "minor", "patch", "hash", "timestamp", "release","latency_90","created_at", "updated_at", "last_contact_success", "last_contact_failure", "disqualified", "disqualification_reason", "exit_success", "vetted_at") VALUES (E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\016', '127.0.0.1:55520', '', 0, 4, '', '', -1, 0, 0, 1, 0, '', 'epoch', false, 0, '2019-02-14 08:07:31.028103+00', '2019-02-14 08:07:31.108963+00', 'epoch', 'epoch', NULL, NULL, false, '2020-03-18 12:00:00.000000+00'); +INSERT INTO "nodes"("id", "address", "last_net", "protocol", "type", "email", "wallet", "free_disk", "piece_count", "major", "minor", "patch", "hash", "timestamp", "release","latency_90","created_at", "updated_at", "last_contact_success", "last_contact_failure", "disqualified", "disqualification_reason", "exit_success") VALUES (E'\\154\\313\\233\\074\\327\\177\\136\\070\\346\\001', '127.0.0.1:55516', '', 0, 4, '', '', -1, 0, 0, 1, 0, '', 'epoch', false, 0, '2019-02-14 08:07:31.028103+00', '2019-02-14 08:07:31.108963+00', 'epoch', 'epoch', NULL, NULL, false); +INSERT INTO "nodes"("id", "address", "last_net", "last_ip_port", "protocol", "type", "email", "wallet", "free_disk", "piece_count", "major", "minor", "patch", "hash", "timestamp", "release","latency_90", "created_at", "updated_at", "last_contact_success", "last_contact_failure", "disqualified", "disqualification_reason", "exit_success") VALUES (E'\\154\\313\\233\\074\\327\\177\\136\\070\\346\\002', '127.0.0.1:55516', '127.0.0.0', '127.0.0.1:55516', 0, 4, '', '', -1, 0, 0, 1, 0, '', 'epoch', false, 0, '2019-02-14 08:07:31.028103+00', '2019-02-14 08:07:31.108963+00', 'epoch', 'epoch', NULL, NUll, false); +INSERT INTO "nodes"("id", "address", "last_net", "protocol", "type", "email", "wallet", "free_disk", "piece_count", "major", "minor", "patch", "hash", "timestamp", "release","latency_90","created_at", "updated_at", "last_contact_success", "last_contact_failure", "disqualified", "disqualification_reason", "exit_success") VALUES (E'\\363\\341\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\016', '127.0.0.1:55516', '', 0, 4, '', '', -1, 0, 0, 1, 0, '', 'epoch', false, 0, '2019-02-14 08:07:31.028103+00', '2019-02-14 08:07:31.108963+00', 'epoch', 'epoch', NULL, NULL, false); +INSERT INTO "nodes"("id", "address", "last_net", "protocol", "type", "email", "wallet", "wallet_features", "free_disk", "piece_count", "major", "minor", "patch", "hash", "timestamp", "release","latency_90","created_at", "updated_at", "last_contact_success", "last_contact_failure", "disqualified", "disqualification_reason", "exit_success") VALUES (E'\\362\\341\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\016', '127.0.0.1:55516', '', 0, 4, '', '', '', -1, 0, 0, 1, 0, '', 'epoch', false, 0, '2019-02-14 08:07:31.028103+00', '2019-02-14 08:07:31.108963+00', 'epoch', 'epoch', NULL, NULL, false); + +INSERT INTO "users"("id", "full_name", "short_name", "email", "normalized_email", "password_hash", "status", "partner_id", "created_at", "is_professional", "project_limit", "project_bandwidth_limit", "project_storage_limit", "paid_tier", "project_segment_limit") VALUES (E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, 'Noahson', 'William', '1email1@mail.test', '1EMAIL1@MAIL.TEST', E'some_readable_hash'::bytea, 1, NULL, '2019-02-14 08:28:24.614594+00', false, 10, 50000000000, 50000000000, false, 150000); +INSERT INTO "users"("id", "full_name", "short_name", "email", "normalized_email", "password_hash", "status", "partner_id", "created_at", "position", "company_name", "working_on", "company_size", "is_professional", "employee_count", "project_limit", "project_bandwidth_limit", "project_storage_limit", "have_sales_contact", "project_segment_limit") VALUES (E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\304\\313\\206\\311",'::bytea, 'Ian', 'Pires', '3email3@mail.test', '3EMAIL3@MAIL.TEST', E'some_readable_hash'::bytea, 2, NULL, '2020-03-18 10:28:24.614594+00', 'engineer', 'storj', 'data storage', 51, true, '1-50', 10, 50000000000, 50000000000, true, 150000); +INSERT INTO "users"("id", "full_name", "short_name", "email", "normalized_email", "password_hash", "status", "partner_id", "created_at", "position", "company_name", "working_on", "company_size", "is_professional", "employee_count", "project_limit", "project_bandwidth_limit", "project_storage_limit", "project_segment_limit") VALUES (E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\205\\312",'::bytea, 'Campbell', 'Wright', '4email4@mail.test', '4EMAIL4@MAIL.TEST', E'some_readable_hash'::bytea, 2, NULL, '2020-07-17 10:28:24.614594+00', 'engineer', 'storj', 'data storage', 82, true, '1-50', 10, 50000000000, 50000000000, 150000); +INSERT INTO "users"("id", "full_name", "short_name", "email", "normalized_email", "password_hash", "status", "partner_id", "created_at", "position", "company_name", "working_on", "company_size", "is_professional", "project_limit", "project_bandwidth_limit", "project_storage_limit", "paid_tier", "mfa_enabled", "mfa_secret_key", "mfa_recovery_codes", "project_segment_limit") VALUES (E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\205\\311",'::bytea, 'Thierry', 'Berg', '2email2@mail.test', '2EMAIL2@MAIL.TEST', E'some_readable_hash'::bytea, 2, NULL, '2020-05-16 10:28:24.614594+00', 'engineer', 'storj', 'data storage', 55, true, 10, 50000000000, 50000000000, false, false, NULL, NULL, 150000); + +INSERT INTO "projects"("id", "name", "description", "usage_limit", "bandwidth_limit", "max_buckets", "partner_id", "owner_id", "created_at", "segment_limit") VALUES (E'\\022\\217/\\014\\376!K\\023\\276\\031\\311}m\\236\\205\\300'::bytea, 'ProjectName', 'projects description', 5e11, 5e11, NULL, NULL, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, '2019-02-14 08:28:24.254934+00', 150000); +INSERT INTO "projects"("id", "name", "description", "usage_limit", "bandwidth_limit", "max_buckets", "partner_id", "owner_id", "created_at", "segment_limit") VALUES (E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\014'::bytea, 'projName1', 'Test project 1', 5e11, 5e11, NULL, NULL, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, '2019-02-14 08:28:24.636949+00', 150000); +INSERT INTO "project_members"("member_id", "project_id", "created_at") VALUES (E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\014'::bytea, '2019-02-14 08:28:24.677953+00'); +INSERT INTO "project_members"("member_id", "project_id", "created_at") VALUES (E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, E'\\022\\217/\\014\\376!K\\023\\276\\031\\311}m\\236\\205\\300'::bytea, '2019-02-13 08:28:24.677953+00'); + +INSERT INTO "registration_tokens" ("secret", "owner_id", "project_limit", "created_at") VALUES (E'\\070\\127\\144\\013\\332\\344\\102\\376\\306\\056\\303\\130\\106\\132\\321\\276\\321\\274\\170\\264\\054\\333\\221\\116\\154\\221\\335\\070\\220\\146\\344\\216'::bytea, null, 1, '2019-02-14 08:28:24.677953+00'); + +INSERT INTO "storagenode_bandwidth_rollups" ("storagenode_id", "interval_start", "interval_seconds", "action", "allocated", "settled") VALUES (E'\\006\\223\\250R\\221\\005\\365\\377v>0\\266\\365\\216\\255?\\347\\244\\371?2\\264\\262\\230\\007<\\001\\262\\263\\237\\247n', '2019-03-06 08:00:00.000000' AT TIME ZONE current_setting('TIMEZONE'), 3600, 1, 1024, 2024); +INSERT INTO "storagenode_storage_tallies" VALUES (E'\\3510\\323\\225"~\\036<\\342\\330m\\0253Jhr\\246\\233K\\246#\\2303\\351\\256\\275j\\212UM\\362\\207', '2019-02-14 08:16:57.812849+00', 1000); + +INSERT INTO "bucket_bandwidth_rollups" ("bucket_name", "project_id", "interval_start", "interval_seconds", "action", "inline", "allocated", "settled") VALUES (E'testbucket'::bytea, E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\014'::bytea,'2019-03-06 08:00:00.000000' AT TIME ZONE current_setting('TIMEZONE'), 3600, 1, 1024, 2024, 3024); +INSERT INTO "bucket_storage_tallies" ("bucket_name", "project_id", "interval_start", "inline", "remote", "remote_segments_count", "inline_segments_count", "object_count", "metadata_size") VALUES (E'testbucket'::bytea, E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\014'::bytea,'2019-03-06 08:00:00.000000' AT TIME ZONE current_setting('TIMEZONE'), 4024, 5024, 0, 0, 0, 0); +INSERT INTO "bucket_bandwidth_rollups" ("bucket_name", "project_id", "interval_start", "interval_seconds", "action", "inline", "allocated", "settled") VALUES (E'testbucket'::bytea, E'\\170\\160\\157\\370\\274\\366\\113\\364\\272\\235\\301\\243\\321\\102\\321\\136'::bytea,'2019-03-06 08:00:00.000000' AT TIME ZONE current_setting('TIMEZONE'), 3600, 1, 1024, 2024, 3024); +INSERT INTO "bucket_storage_tallies" ("bucket_name", "project_id", "interval_start", "inline", "remote", "remote_segments_count", "inline_segments_count", "object_count", "metadata_size") VALUES (E'testbucket'::bytea, E'\\170\\160\\157\\370\\274\\366\\113\\364\\272\\235\\301\\243\\321\\102\\321\\136'::bytea,'2019-03-06 08:00:00.000000' AT TIME ZONE current_setting('TIMEZONE'), 4024, 5024, 0, 0, 0, 0); + +INSERT INTO "reset_password_tokens" ("secret", "owner_id", "created_at") VALUES (E'\\070\\127\\144\\013\\332\\344\\102\\376\\306\\056\\303\\130\\106\\132\\321\\276\\321\\274\\170\\264\\054\\333\\221\\116\\154\\221\\335\\070\\220\\146\\344\\216'::bytea, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, '2019-05-08 08:28:24.677953+00'); + +INSERT INTO "api_keys" ("id", "project_id", "head", "name", "secret", "partner_id", "created_at") VALUES (E'\\334/\\302;\\225\\355O\\323\\276f\\247\\354/6\\241\\033'::bytea, E'\\022\\217/\\014\\376!K\\023\\276\\031\\311}m\\236\\205\\300'::bytea, E'\\111\\142\\147\\304\\132\\375\\070\\163\\270\\160\\251\\370\\126\\063\\351\\037\\257\\071\\143\\375\\351\\320\\253\\232\\220\\260\\075\\173\\306\\307\\115\\136'::bytea, 'key 2', E'\\254\\011\\315\\333\\273\\365\\001\\071\\024\\154\\253\\332\\301\\216\\361\\074\\221\\367\\251\\231\\274\\333\\300\\367\\001\\272\\327\\111\\315\\123\\042\\016'::bytea, NULL, '2019-02-14 08:28:24.267934+00'); + +INSERT INTO "value_attributions" ("project_id", "bucket_name", "partner_id", "user_agent", "last_updated") VALUES (E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, E''::bytea, E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\014'::bytea, NULL, '2019-02-14 08:07:31.028103+00'); + +INSERT INTO "user_credits" ("id", "user_id", "offer_id", "referred_by", "credits_earned_in_cents", "credits_used_in_cents", "type", "expires_at", "created_at") VALUES (1, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, 1, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, 200, 0, 'invalid', '2019-10-01 08:28:24.267934+00', '2019-06-01 08:28:24.267934+00'); + +INSERT INTO "bucket_metainfos" ("id", "project_id", "name", "partner_id", "created_at", "path_cipher", "default_segment_size", "default_encryption_cipher_suite", "default_encryption_block_size", "default_redundancy_algorithm", "default_redundancy_share_size", "default_redundancy_required_shares", "default_redundancy_repair_shares", "default_redundancy_optimal_shares", "default_redundancy_total_shares") VALUES (E'\\334/\\302;\\225\\355O\\323\\276f\\247\\354/6\\241\\033'::bytea, E'\\022\\217/\\014\\376!K\\023\\276\\031\\311}m\\236\\205\\300'::bytea, E'testbucketuniquename'::bytea, NULL, '2019-06-14 08:28:24.677953+00', 1, 65536, 1, 8192, 1, 4096, 4, 6, 8, 10); + +INSERT INTO "peer_identities" VALUES (E'\\334/\\302;\\225\\355O\\323\\276f\\247\\354/6\\241\\033'::bytea, E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\014'::bytea, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, '2019-02-14 08:07:31.335028+00'); + +INSERT INTO "graceful_exit_progress" ("node_id", "bytes_transferred", "pieces_transferred", "pieces_failed", "updated_at") VALUES (E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\016', 1000000000000000, 0, 0, '2019-09-12 10:07:31.028103+00'); + +INSERT INTO "stripe_customers" ("user_id", "customer_id", "created_at") VALUES (E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, 'stripe_id', '2019-06-01 08:28:24.267934+00'); + +INSERT INTO "stripecoinpayments_invoice_project_records"("id", "project_id", "storage", "egress", "objects", "period_start", "period_end", "state", "created_at") VALUES (E'\\022\\217/\\014\\376!K\\023\\276\\031\\311}m\\236\\205\\300'::bytea, E'\\021\\217/\\014\\376!K\\023\\276\\031\\311}m\\236\\205\\300'::bytea, 0, 0, 0, '2019-06-01 08:28:24.267934+00', '2019-06-01 08:28:24.267934+00', 0, '2019-06-01 08:28:24.267934+00'); + +INSERT INTO "stripecoinpayments_tx_conversion_rates" ("tx_id", "rate_numeric", "created_at") VALUES ('tx_id', '1.929883831', '2019-06-01 08:28:24.267934+00'); + +INSERT INTO "coinpayments_transactions" ("id", "user_id", "address", "amount_numeric", "received_numeric", "status", "key", "timeout", "created_at") VALUES ('tx_id', E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, 'address', 1411112222, 1311112222, 1, 'key', 60, '2019-06-01 08:28:24.267934+00'); + +INSERT INTO "storagenode_bandwidth_rollups" ("storagenode_id", "interval_start", "interval_seconds", "action", "settled") VALUES (E'\\006\\223\\250R\\221\\005\\365\\377v>0\\266\\365\\216\\255?\\347\\244\\371?2\\264\\262\\230\\007<\\001\\262\\263\\237\\247n', '2020-01-11 08:00:00.000000' AT TIME ZONE current_setting('TIMEZONE'), 3600, 1, 2024); + +INSERT INTO "coupons" ("id", "user_id", "amount", "description", "type", "status", "duration", "billing_periods", "created_at") VALUES (E'\\362\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\014'::bytea, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, 50, 'description', 0, 0, 2, 2, '2019-06-01 08:28:24.267934+00'); +INSERT INTO "coupons" ("id", "user_id", "amount", "description", "type", "status", "duration", "billing_periods", "created_at") VALUES (E'\\362\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\012'::bytea, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, 50, 'description', 0, 0, 2, 2, '2019-06-01 08:28:24.267934+00'); +INSERT INTO "coupons" ("id", "user_id", "amount", "description", "type", "status", "duration", "billing_periods", "created_at") VALUES (E'\\362\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\015'::bytea, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, 50, 'description', 0, 0, 2, 2, '2019-06-01 08:28:24.267934+00'); +INSERT INTO "coupon_usages" ("coupon_id", "amount", "status", "period") VALUES (E'\\362\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\014'::bytea, 22, 0, '2019-06-01 09:28:24.267934+00'); +INSERT INTO "coupon_codes" ("id", "name", "amount", "description", "type", "billing_periods", "created_at") VALUES (E'\\362\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\014'::bytea, 'STORJ50', 50, '$50 for your first 5 months', 0, NULL, '2019-06-01 08:28:24.267934+00'); +INSERT INTO "coupon_codes" ("id", "name", "amount", "description", "type", "billing_periods", "created_at") VALUES (E'\\362\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\015'::bytea, 'STORJ75', 75, '$75 for your first 5 months', 0, 2, '2019-06-01 08:28:24.267934+00'); + +INSERT INTO "stripecoinpayments_apply_balance_intents" ("tx_id", "state", "created_at") VALUES ('tx_id', 0, '2019-06-01 08:28:24.267934+00'); + +INSERT INTO "projects"("id", "name", "description", "usage_limit", "bandwidth_limit", "max_buckets", "rate_limit", "partner_id", "owner_id", "created_at", "segment_limit") VALUES (E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\347'::bytea, 'projName1', 'Test project 1', 5e11, 5e11, NULL, 2000000, NULL, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, '2020-01-15 08:28:24.636949+00', 150000); + +INSERT INTO "project_bandwidth_rollups"("project_id", "interval_month", egress_allocated) VALUES (E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\347'::bytea, '2020-04-01', 10000); +INSERT INTO "project_bandwidth_daily_rollups"("project_id", "interval_day", egress_allocated, egress_settled, egress_dead) VALUES (E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\347'::bytea, '2021-04-22', 10000, 5000, 0); + +INSERT INTO "projects"("id", "name", "description", "usage_limit", "bandwidth_limit", "max_buckets","rate_limit", "partner_id", "owner_id", "created_at", "segment_limit") VALUES (E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\345'::bytea, 'egress101', 'High Bandwidth Project', 5e11, 5e11, NULL, 2000000, NULL, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, '2020-05-15 08:46:24.000000+00', 150000); + +INSERT INTO "storagenode_paystubs"("period", "node_id", "created_at", "codes", "usage_at_rest", "usage_get", "usage_put", "usage_get_repair", "usage_put_repair", "usage_get_audit", "comp_at_rest", "comp_get", "comp_put", "comp_get_repair", "comp_put_repair", "comp_get_audit", "surge_percent", "held", "owed", "disposed", "paid", "distributed") VALUES ('2020-01', '\xf2a3b4c4dfdf7221310382fd5db5aa73e1d227d6df09734ec4e5305000000000', '2020-04-07T20:14:21.479141Z', '', 1327959864508416, 294054066688, 159031363328, 226751, 0, 836608, 2861984, 5881081, 0, 226751, 0, 8, 300, 0, 26909472, 0, 26909472, 0); +INSERT INTO "nodes"("id", "address", "last_net", "protocol", "type", "email", "wallet", "free_disk", "piece_count", "major", "minor", "patch", "hash", "timestamp", "release","latency_90","created_at", "updated_at", "last_contact_success", "last_contact_failure", "disqualified", "disqualification_reason", "exit_success", "unknown_audit_suspended", "offline_suspended", "under_review") VALUES (E'\\153\\313\\233\\074\\327\\255\\136\\070\\346\\001', '127.0.0.1:55516', '', 0, 4, '', '', -1, 0, 0, 1, 0, '', 'epoch', false, 0, '2019-02-14 08:07:31.028103+00', '2019-02-14 08:07:31.108963+00', 'epoch', 'epoch', NULL, NULL, false, '2019-02-14 08:07:31.108963+00', '2019-02-14 08:07:31.108963+00', '2019-02-14 08:07:31.108963+00'); + +INSERT INTO "node_api_versions"("id", "api_version", "created_at", "updated_at") VALUES (E'\\153\\313\\233\\074\\327\\177\\136\\070\\346\\001', 1, '2019-02-14 08:07:31.028103+00', '2019-02-14 08:07:31.108963+00'); +INSERT INTO "node_api_versions"("id", "api_version", "created_at", "updated_at") VALUES (E'\\006\\223\\250R\\221\\005\\365\\377v>0\\266\\365\\216\\255?\\347\\244\\371?2\\264\\262\\230\\007<\\001\\262\\263\\237\\247n', 2, '2019-02-14 08:07:31.028103+00', '2019-02-14 08:07:31.108963+00'); +INSERT INTO "node_api_versions"("id", "api_version", "created_at", "updated_at") VALUES (E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\014', 3, '2019-02-14 08:07:31.028103+00', '2019-02-14 08:07:31.108963+00'); + +INSERT INTO "projects"("id", "name", "description", "usage_limit", "bandwidth_limit", "rate_limit", "partner_id", "owner_id", "created_at", "max_buckets", "segment_limit") VALUES (E'300\\273|\\342N\\347\\347\\363\\342\\363\\371>+F\\256\\263'::bytea, 'egress102', 'High Bandwidth Project 2', 5e11, 5e11, 2000000, NULL, E'265\\343U\\303\\312\\312\\363\\311\\033w\\222\\303Ci",'::bytea, '2020-05-15 08:46:24.000000+00', 1000, 150000); +INSERT INTO "projects"("id", "name", "description", "usage_limit", "bandwidth_limit", "rate_limit", "partner_id", "owner_id", "created_at", "max_buckets", "segment_limit") VALUES (E'300\\273|\\342N\\347\\347\\363\\342\\363\\371>+F\\255\\244'::bytea, 'egress103', 'High Bandwidth Project 3', 5e11, 5e11, 2000000, NULL, E'265\\343U\\303\\312\\312\\363\\311\\033w\\222\\303Ci",'::bytea, '2020-05-15 08:46:24.000000+00', 1000, 150000); + +INSERT INTO "projects"("id", "name", "description", "usage_limit", "bandwidth_limit", "rate_limit", "partner_id", "owner_id", "created_at", "max_buckets", "segment_limit") VALUES (E'300\\273|\\342N\\347\\347\\363\\342\\363\\371>+F\\253\\231'::bytea, 'Limit Test 1', 'This project is above the default', 50000000001, 50000000001, 2000000, NULL, E'265\\343U\\303\\312\\312\\363\\311\\033w\\222\\303Ci",'::bytea, '2020-10-14 10:10:10.000000+00', 101, 150000); +INSERT INTO "projects"("id", "name", "description", "usage_limit", "bandwidth_limit", "rate_limit", "partner_id", "owner_id", "created_at", "max_buckets", "segment_limit") VALUES (E'300\\273|\\342N\\347\\347\\363\\342\\363\\371>+F\\252\\230'::bytea, 'Limit Test 2', 'This project is below the default', 5e11, 5e11, 2000000, NULL, E'265\\343U\\303\\312\\312\\363\\311\\033w\\222\\303Ci",'::bytea, '2020-10-14 10:10:11.000000+00', NULL, 150000); + +INSERT INTO "storagenode_bandwidth_rollups_phase2" ("storagenode_id", "interval_start", "interval_seconds", "action", "allocated", "settled") VALUES (E'\\006\\223\\250R\\221\\005\\365\\377v>0\\266\\365\\216\\255?\\347\\244\\371?2\\264\\262\\230\\007<\\001\\262\\263\\237\\247n', '2019-03-06 08:00:00.000000' AT TIME ZONE current_setting('TIMEZONE'), 3600, 1, 1024, 2024); + +INSERT INTO "storagenode_bandwidth_rollup_archives" ("storagenode_id", "interval_start", "interval_seconds", "action", "allocated", "settled") VALUES (E'\\006\\223\\250R\\221\\005\\365\\377v>0\\266\\365\\216\\255?\\347\\244\\371?2\\264\\262\\230\\007<\\001\\262\\263\\237\\247n', '2019-03-06 08:00:00.000000' AT TIME ZONE current_setting('TIMEZONE'), 3600, 1, 1024, 2024); +INSERT INTO "bucket_bandwidth_rollup_archives" ("bucket_name", "project_id", "interval_start", "interval_seconds", "action", "inline", "allocated", "settled") VALUES (E'testbucket'::bytea, E'\\170\\160\\157\\370\\274\\366\\113\\364\\272\\235\\301\\243\\321\\102\\321\\136'::bytea,'2019-03-06 08:00:00.000000' AT TIME ZONE current_setting('TIMEZONE'), 3600, 1, 1024, 2024, 3024); + +INSERT INTO "storagenode_paystubs"("period", "node_id", "created_at", "codes", "usage_at_rest", "usage_get", "usage_put", "usage_get_repair", "usage_put_repair", "usage_get_audit", "comp_at_rest", "comp_get", "comp_put", "comp_get_repair", "comp_put_repair", "comp_get_audit", "surge_percent", "held", "owed", "disposed", "paid", "distributed") VALUES ('2020-12', '\x1111111111111111111111111111111111111111111111111111111111111111', '2020-04-07T20:14:21.479141Z', '', 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 117); +INSERT INTO "storagenode_payments"("id", "created_at", "period", "node_id", "amount") VALUES (1, '2020-04-07T20:14:21.479141Z', '2020-12', '\x1111111111111111111111111111111111111111111111111111111111111111', 117); + +INSERT INTO "reputations"("id", "audit_success_count", "total_audit_count", "created_at", "updated_at", "disqualified", "audit_reputation_alpha", "audit_reputation_beta", "unknown_audit_reputation_alpha", "unknown_audit_reputation_beta", "online_score", "audit_history") VALUES (E'\\153\\313\\233\\074\\327\\177\\136\\070\\346\\001', 0, 5, '2019-02-14 08:07:31.028103+00', '2019-02-14 08:07:31.108963+00', NULL, 1000, 0, 1, 0, 1, '\x0a23736f2f6d616e792f69636f6e69632f70617468732f746f2f63686f6f73652f66726f6d120a0102030405060708090a'); + +INSERT INTO "graceful_exit_segment_transfer_queue" ("node_id", "stream_id", "position", "piece_num", "durability_ratio", "queued_at", "requested_at", "last_failed_at", "last_failed_code", "failed_count", "finished_at", "order_limit_send_count") VALUES (E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\016', E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, 10 , 8, 1.0, '2019-09-12 10:07:31.028103+00', '2019-09-12 10:07:32.028103+00', null, null, 0, '2019-09-12 10:07:33.028103+00', 0); + +INSERT INTO "segment_pending_audits" ("node_id", "piece_id", "stripe_index", "share_size", "expected_share_hash", "reverify_count", "stream_id", position) VALUES (E'\\153\\313\\233\\074\\327\\177\\136\\070\\346\\001'::bytea, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, 5, 1024, E'\\070\\127\\144\\013\\332\\344\\102\\376\\306\\056\\303\\130\\106\\132\\321\\276\\321\\274\\170\\264\\054\\333\\221\\116\\154\\221\\335\\070\\220\\146\\344\\216'::bytea, 1, '\x010101', 1); + +INSERT INTO "users"("id", "full_name", "short_name", "email", "normalized_email", "password_hash", "status", "partner_id", "created_at", "is_professional", "project_limit", "project_bandwidth_limit", "project_storage_limit", "paid_tier", "project_segment_limit") VALUES (E'\\363\\311\\033w\\222\\303Ci\\266\\342U\\303\\312\\204",'::bytea, 'Noahson', 'William', '100email1@mail.test', '100EMAIL1@MAIL.TEST', E'some_readable_hash'::bytea, 1, NULL, '2019-02-14 08:28:24.614594+00', false, 10, 100000000000000, 25000000000000, true, 100000000); + +INSERT INTO "repair_queue" ("stream_id", "position", "attempted_at", "segment_health", "updated_at", "inserted_at") VALUES ('\x01', 1, null, 1, '2020-09-01 00:00:00.000000+00', '2021-09-01 00:00:00.000000+00'); + +INSERT INTO "users"("id", "full_name", "email", "normalized_email", "password_hash", "status", "created_at", "mfa_enabled", "mfa_secret_key", "mfa_recovery_codes", "project_limit", "project_bandwidth_limit", "project_storage_limit", "project_segment_limit") VALUES (E'\\363\\311\\033w\\222\\303Ci\\266\\344U\\303\\312\\204",'::bytea, 'Noahson William', '101email1@mail.test', '101EMAIL1@MAIL.TEST', E'some_readable_hash'::bytea, 1, '2019-02-14 08:28:24.614594+00', true, 'mfa secret key', '["1a2b3c4d","e5f6g7h8"]', 3, 50000000000, 50000000000, 150000); + +INSERT INTO "projects"("id", "name", "description", "usage_limit", "bandwidth_limit", "rate_limit", "burst_limit", "partner_id", "owner_id", "created_at", "max_buckets", "segment_limit") VALUES (E'300\\273|\\342N\\347\\347\\363\\342\\363\\371>+F\\251\\247'::bytea, 'Limit Test 2', 'This project is below the default', 5e11, 5e11, 2000000, 4000000, NULL, E'265\\343U\\303\\312\\312\\363\\311\\033w\\222\\303Ci",'::bytea, '2020-10-14 10:10:11.000000+00', NULL, 150000); + +INSERT INTO "users"("id", "full_name", "email", "normalized_email", "password_hash", "status", "created_at", "mfa_enabled", "mfa_secret_key", "mfa_recovery_codes", "signup_promo_code", "project_limit", "project_bandwidth_limit", "project_storage_limit", "project_segment_limit") VALUES (E'\\363\\311\\033w\\222\\303Ci\\266\\344U\\303\\312\\205",'::bytea, 'Felicia Smith', '99email1@mail.test', '99EMAIL1@MAIL.TEST', E'some_readable_hash'::bytea, 1, '2021-08-14 09:13:44.614594+00', true, 'mfa secret key', '["1a2b3c4d","e5f6d7h8"]', 'promo123', 3, 50000000000, 50000000000, 150000); + +INSERT INTO "stripecoinpayments_invoice_project_records"("id", "project_id", "storage", "egress", "objects", "segments", "period_start", "period_end", "state", "created_at") VALUES (E'\\300\\217/\\014\\376!K\\023\\276\\031\\311}m\\236\\205\\300'::bytea, E'\\300\\217/\\014\\376!K\\023\\276\\031\\311}m\\236\\205\\300'::bytea, 0, 0, 0, 0, '2019-06-01 08:28:24.267934+00', '2019-06-01 08:28:24.267934+00', 0, '2019-06-01 08:28:24.267934+00'); + +INSERT INTO "nodes"("id", "address", "last_net", "protocol", "type", "email", "wallet", "free_disk", "piece_count", "major", "minor", "patch", "hash", "timestamp", "release","latency_90", "created_at", "updated_at", "last_contact_success", "last_contact_failure", "disqualified", "disqualification_reason", "exit_success", "country_code") VALUES (E'\\153\\313\\233\\074\\327\\177\\136\\070\\346\\002', '127.0.0.1:55517', '', 0, 4, '', '', -1, 0, 0, 1, 0, '', 'epoch', false, 0, '2021-02-14 08:07:31.028103+00', '2021-02-14 08:07:31.108963+00', 'epoch', 'epoch', NULL, NULL, false, 'DE'); +INSERT INTO "bucket_metainfos" ("id", "project_id", "name", "partner_id", "created_at", "path_cipher", "default_segment_size", "default_encryption_cipher_suite", "default_encryption_block_size", "default_redundancy_algorithm", "default_redundancy_share_size", "default_redundancy_required_shares", "default_redundancy_repair_shares", "default_redundancy_optimal_shares", "default_redundancy_total_shares", "placement") VALUES (E'\\144/\\302;\\225\\355O\\323\\276f\\247\\354/6\\241\\033'::bytea, E'\\022\\217/\\014\\376!K\\023\\276\\031\\311}m\\236\\205\\300'::bytea, E'testbucketotheruniquename'::bytea, NULL, '2019-06-14 08:28:24.677953+00', 1, 65536, 1, 8192, 1, 4096, 4, 6, 8, 10, 1); + +INSERT INTO "nodes"("id", "address", "last_net", "protocol", "type", "email", "wallet", "wallet_features", "free_disk", "piece_count", "major", "minor", "patch", "hash", "timestamp", "release","latency_90","created_at", "updated_at", "last_contact_success", "last_contact_failure", "disqualified", "disqualification_reason", "exit_success", "country_code") VALUES (E'\\362\\341\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\017', '127.0.0.1:55517', '', 0, 4, '', '', '', -1, 0, 0, 1, 0, '', 'epoch', false, 0, '2020-02-14 08:07:31.028103+00', '2021-10-13 08:07:31.108963+00', 'epoch', 'epoch', '2021-10-13 08:07:31.108963+00', 0, false, NULL); + +INSERT INTO "users"("id", "full_name", "email", "normalized_email", "password_hash", "status", "created_at", "mfa_enabled", "mfa_secret_key", "mfa_recovery_codes", "signup_promo_code", "project_limit", "project_bandwidth_limit", "project_storage_limit", "project_segment_limit") VALUES (E'\\363\\311\\033w\\222\\303Ci\\267\\342U\\303\\312\\203",'::bytea, 'Jessica Thompson', '143email1@mail.test', '143EMAIL1@MAIL.TEST', E'some_readable_hash'::bytea, 1, '2021-11-04 08:27:56.614594+00', true, 'mfa secret key', '["2b3c4d5e","f6a7e8e9"]', 'promo123', 3, '150000000000', '150000000000', 150000); + +INSERT INTO "users"("id", "full_name", "email", "normalized_email", "password_hash", "status", "created_at", "mfa_enabled", "mfa_secret_key", "mfa_recovery_codes", "signup_promo_code", "project_limit", "project_bandwidth_limit", "project_storage_limit", "project_segment_limit") VALUES (E'\\363\\311\\033w\\222\\303Ci\\265\\342U\\303\\312\\202",'::bytea, 'Heather Jackson', '762email@mail.test', '762EMAIL1@MAIL.TEST', E'some_readable_hash'::bytea, 1, '2021-11-05 03:22:39.614594+00', true, 'mfa secret key', '["5e4d3c2b","e9e8a7f6"]', 'promo123', 3, '100000000000000', '25000000000000', 150000); + +INSERT INTO "users"("id", "full_name", "email", "normalized_email", "password_hash", "status", "created_at", "mfa_enabled", "mfa_secret_key", "mfa_recovery_codes", "signup_promo_code", "project_limit", "project_bandwidth_limit", "project_storage_limit", "last_verification_reminder", "project_segment_limit") VALUES (E'\\364\\312\\033w\\222\\303Ci\\265\\342U\\303\\312\\202",'::bytea, 'Michael Mint', '333email2@mail.test', '333EMAIL2@MAIL.TEST', E'some_readable_hash'::bytea, 1, '2021-10-05 03:22:39.614594+00', true, 'mfa secret key', '["5e4d3c2c","e9e8a7f7"]', 'promo123', 3, '100000000000000', '25000000000000', '2021-12-05 03:22:39.614594+00', 150000); + +INSERT INTO "oauth_clients"("id", "encrypted_secret", "redirect_url", "user_id", "app_name", "app_logo_url") VALUES (E'FD6209C0-7A17-4FC3-895C-E57A6C7CBBE1'::bytea, E'610B723B-E1FF-4B1D-B372-521250690C6E'::bytea, 'https://example.test/callback/storj', E'\\364\\312\\033w\\222\\303Ci\\265\\342U\\303\\312\\202",'::bytea, 'Example App', 'https://example.test/logo.png'); + +INSERT INTO "oauth_codes"("client_id", "user_id", "scope", "redirect_url", "challenge", "challenge_method", "code", "created_at", "expires_at", "claimed_at") VALUES (E'FD6209C0-7A17-4FC3-895C-E57A6C7CBBE1'::bytea, E'\\364\\312\\033w\\222\\303Ci\\265\\342U\\303\\312\\202",'::bytea, 'scope', 'http://localhost:12345/callback', 'challenge', 'challenge method', 'plaintext code', '2021-12-05 03:22:39.614594+00', '2021-12-05 03:22:39.614594+00', '2021-12-05 03:22:39.614594+00'); + +INSERT INTO "oauth_tokens"("client_id", "user_id", "scope", "kind", "token", "created_at", "expires_at") VALUES (E'FD6209C0-7A17-4FC3-895C-E57A6C7CBBE1'::bytea, E'\\364\\312\\033w\\222\\303Ci\\265\\342U\\303\\312\\202",'::bytea, 'scope', 1, E'B9C93D5F-CBD7-4615-9184-E714CFE14365'::bytea, '2021-12-05 03:22:39.614594+00', '2021-12-05 03:22:39.614594+00'); + +INSERT INTO "coinpayments_transactions" ("id", "user_id", "address", "amount_numeric", "received_numeric", "status", "key", "timeout", "created_at") VALUES ('different_tx_id_from_before', E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, 'address', 125419938429, 1, 1, 'key', 60, '2021-07-28 20:24:11.932313-05'); +INSERT INTO "stripecoinpayments_tx_conversion_rates" ("tx_id", "rate_numeric", "created_at") VALUES ('different_tx_id_from_before', 3.14159265359, '2021-07-28 20:24:11.932313-05'); + +INSERT INTO "webapp_sessions"("id", "user_id", "ip_address", "user_agent", "status", "expires_at") VALUES (E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, '127.0.0.1', 'Firefox', 0, '2019-02-14 08:28:24.614594+00'); + +INSERT INTO "users"("id", "full_name", "email", "normalized_email", "password_hash", "status", "created_at", "mfa_enabled", "mfa_secret_key", "mfa_recovery_codes", "signup_promo_code", "project_limit", "project_bandwidth_limit", "project_storage_limit", "project_segment_limit", "verification_reminders") VALUES (E'\\363\\311\\033w\\222\\303Ci\\266\\344U\\304\\312\\205",'::bytea, 'Felicia Smith', '1testemail1@mail.test', '1TESTEMAIL1@MAIL.TEST', E'some_readable_hash'::bytea, 1, '2021-08-14 09:13:44.614594+00', true, 'mfa secret key', '["1a2b3c4d","e5f6d7h8"]', 'promo123', 3, 50000000000, 50000000000, 150000, 1); + +INSERT INTO "reputations"("id", "audit_success_count", "total_audit_count", "created_at", "updated_at", "disqualified", "disqualification_reason", "audit_reputation_alpha", "audit_reputation_beta", "unknown_audit_reputation_alpha", "unknown_audit_reputation_beta", "online_score", "audit_history") VALUES (E'\\153\\313\\233\\074\\327\\177\\136\\070\\346\\002', 2, 5, '2022-04-20 04:20:59.028103+00', '2022-04-20 04:21:09.028103+00', '2022-04-20 04:22:09.028103+00', 3, 50, 0, 1, 0, 1, '\x0a23736f2f6d616e792f69636f6e69632f70617468732f746f2f63686f6f73652f66726f6d120a0102030405060708090a'); + +INSERT INTO "storjscan_wallets" ("user_id", "wallet_address", "created_at") VALUES (E'\\363\\301\\032w\\222\\203Ci\\245\\342U\\304\\332\\202",'::bytea, E'\\343\\301\\042w\\222\\263Ci\\245\\312U\\304\\312\\202",'::bytea, '2021-07-28 20:04:11.932313+00'); + +INSERT INTO "storjscan_payments" ("block_hash", "block_number", "transaction", "log_index", "from_address", "to_address", "token_value", "usd_value", "status", "timestamp", "created_at") VALUES (E'\\363\\301\\032w\\222\\203Ci\\245\\342U\\304\\332\\202",'::bytea, 0, E'\\363\\301\\032w\\222\\203Ci\\245\\342U\\304\\332\\202",'::bytea, 0, E'\\363\\301\\032w\\222\\203Ci\\245\\342U\\304\\332\\202",'::bytea, E'\\363\\301\\032w\\222\\203Ci\\245\\342U\\304\\332\\202",'::bytea, 1, 1, 'example', '2022-04-20 04:22:09.028103+00', '2022-04-20 04:22:09.028103+00'); + +INSERT INTO "projects"("id", "public_id", "name", "description", "usage_limit", "bandwidth_limit", "rate_limit", "burst_limit", "partner_id", "owner_id", "created_at", "max_buckets", "segment_limit") VALUES (E'300\\273|\\342N\\347\\347\\347\\342\\363\\371>+F\\251\\247'::bytea, E'300\\273|\\342N\\347\\347\\363\\347\\363\\371>+F\\241\\247'::bytea, 'Limit Test 2', 'This project is below the default', 5e11, 5e11, 2000000, 4000000, NULL, E'265\\343U\\303\\312\\312\\363\\311\\033w\\222\\303Ci",'::bytea, '2020-10-14 10:10:11.000000+00', NULL, 150000); + +INSERT INTO "accounting_rollups"("node_id", "start_time", "put_total", "get_total", "get_audit_total", "get_repair_total", "put_repair_total", "at_rest_total", "interval_end_time") VALUES (E'\\367M\\177\\251]t/\\022\\256\\214\\265\\025\\224\\204:\\217\\212\\0102<\\321\\374\\020&\\271Qc\\325\\261\\354\\246\\233'::bytea, '2019-02-10 00:00:00+00', 2875, 5750, 8635, 11500, 0, 14375, '2019-02-10 23:00:00+00'); + +INSERT INTO "billing_transactions" ("id", "user_id", "amount", "currency", "description", "source", "status", "type", "metadata", "timestamp", "created_at") VALUES (1, E'\\363\\331\\032w\\212\\213Ci\\245\\322U\\314\\302\\202",'::bytea, 113219736213, 'usd', 'some_description', 'some_source', 'some_status', 'some_type', '{ "Wallet": "0x1234", "ReferenceID": "0987654321"}'::jsonb, '2021-07-28 19:14:11.932313+00', '2021-07-28 19:34:11.932323+00'); + +INSERT INTO "billing_balances" ("user_id", "balance", "last_updated") VALUES (E'\\363\\331\\032w\\222\\203Ci\\245\\312U\\304\\322\\212",'::bytea, 113219736213, '2021-07-28 19:34:11.932323+00'); + +INSERT INTO "projects"("id", "public_id", "name", "description", "usage_limit", "bandwidth_limit", "user_specified_usage_limit", "user_specified_bandwidth_limit", "rate_limit", "burst_limit", "partner_id", "owner_id", "created_at", "max_buckets", "segment_limit", "salt") VALUES (E'300\\273|\\342N\\347\\347\\347\\342\\363\\371>+F\\252\\247'::bytea, E'300\\273|\\342N\\347\\347\\363\\347\\363\\371>+F\\241\\247'::bytea, 'Limit Test 2', 'This project is below the default', 5e11, 5e11, NULL, NULL, 2000000, 4000000, NULL, E'265\\343U\\303\\312\\312\\363\\311\\033w\\222\\303Ci",'::bytea, '2020-10-14 10:10:11.000000+00', NULL, 150000, E'300\\273|\\342N\\347\\347\\347\\342\\363\\371>+F\\252\\247'::bytea); + +INSERT INTO "users" ("id", "full_name", "email", "normalized_email", "password_hash", "status", "created_at", "mfa_enabled", "mfa_secret_key", "mfa_recovery_codes", "signup_promo_code", "project_limit", "project_bandwidth_limit", "project_storage_limit", "project_segment_limit", "verification_reminders", "signup_captcha") VALUES (E'\\363\\311\\033w\\222\\303Ci\\266\\344U\\304\\312\\206",'::bytea, 'Harold Smith', '1testemail206@mail.test', '1TESTEMAIL206@MAIL.TEST', E'some_readable_hash'::bytea, 1, '2021-08-14 09:13:44.614594+00', true, 'mfa secret key', '["1a2b3c4d","e5f6d7h8"]', 'promo123', 3, 50000000000, 50000000000, 150000, 1, 1); + +INSERT INTO "reverification_audits" ("node_id", "stream_id", "position", "piece_num", "inserted_at", "last_attempt", "reverify_count") VALUES (E'\\xe3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', E'\\x01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b', 1152921504606846976, 4, '2008-06-06 14:13:08.845574-07', '2009-08-23 02:19:52.922832-07', 5); + +INSERT INTO "node_events" ("id", "email", "node_id", "event", "created_at", "email_sent") VALUES (E'\\362\\341\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\017', 'test@storj.test', E'\\153\\313\\233\\074\\327\\177\\136\\070\\346\\001', 1, '2019-02-14 08:28:24.614594+00', '2019-02-14 08:28:24.614594+00'); + +-- NEW DATA -- + +INSERT INTO "verification_audits" ("inserted_at", "stream_id", "position", "expires_at", "encrypted_size") VALUES ('2022-10-31 00:00:00.000000+00', E'\\xb5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c', 42949672970, NULL, 2147483647); +INSERT INTO "verification_audits" ("inserted_at", "stream_id", "position", "expires_at", "encrypted_size") VALUES ('2022-10-31 00:01:00.000000+00', E'\\x6e96e45029870a9b08cff2ed6ac840ccde3edce244327cc1bddefa1e555bc81f', 450971566185, '2023-01-01 23:59:59.999999+13', 12); diff --git a/satellite/satellitedb/verifyqueue.go b/satellite/satellitedb/verifyqueue.go new file mode 100644 index 000000000..f241f2104 --- /dev/null +++ b/satellite/satellitedb/verifyqueue.go @@ -0,0 +1,105 @@ +// Copyright (C) 2022 Storj Labs, Inc. +// See LICENSE for copying information. + +package satellitedb + +import ( + "context" + "database/sql" + "errors" + "time" + + "storj.io/private/dbutil" + "storj.io/private/dbutil/pgutil" + "storj.io/storj/satellite/audit" +) + +const ( + // VerifyRetryInterval defines a limit on how frequently we retry + // verification audits. At least this long should elapse between + // attempts. + VerifyRetryInterval = 4 * time.Hour +) + +// verifyQueue implements storj.io/storj/satellite/audit.VerifyQueue. +type verifyQueue struct { + db *satelliteDB +} + +var _ audit.VerifyQueue = (*verifyQueue)(nil) + +func (vq *verifyQueue) Push(ctx context.Context, segments []audit.Segment) (err error) { + defer mon.Task()(&ctx)(&err) + + streamIDSlice := make([][]byte, len(segments)) + positionSlice := make([]int64, len(segments)) + expirationSlice := make([]*time.Time, len(segments)) + encryptedSizeSlice := make([]int32, len(segments)) + + for i, seg := range segments { + streamIDSlice[i] = seg.StreamID.Bytes() + positionSlice[i] = int64(seg.Position.Encode()) + expirationSlice[i] = seg.ExpiresAt + encryptedSizeSlice[i] = seg.EncryptedSize + } + _, err = vq.db.DB.ExecContext(ctx, ` + INSERT INTO verification_audits (stream_id, position, expires_at, encrypted_size) + SELECT unnest($1::bytea[]), unnest($2::int8[]), unnest($3::timestamptz[]), unnest($4::int4[]) + `, + pgutil.ByteaArray(streamIDSlice), + pgutil.Int8Array(positionSlice), + pgutil.NullTimestampTZArray(expirationSlice), + pgutil.Int4Array(encryptedSizeSlice), + ) + return Error.Wrap(err) +} + +func (vq *verifyQueue) Next(ctx context.Context) (seg audit.Segment, err error) { + defer mon.Task()(&ctx)(&err) + + var getQuery string + switch vq.db.impl { + case dbutil.Postgres: + getQuery = ` + WITH next_row AS ( + SELECT inserted_at, stream_id, position + FROM verification_audits + ORDER BY inserted_at, stream_id, position + FOR UPDATE SKIP LOCKED + LIMIT 1 + ) + DELETE FROM verification_audits v + USING next_row + WHERE v.inserted_at = next_row.inserted_at + AND v.stream_id = next_row.stream_id + AND v.position = next_row.position + RETURNING v.stream_id, v.position, v.expires_at, v.encrypted_size + ` + case dbutil.Cockroach: + // Note: because Cockroach does not support SKIP LOCKED, this implementation + // is likely much less performant under any amount of contention. + getQuery = ` + WITH next_row AS ( + SELECT inserted_at, stream_id, position + FROM verification_audits + ORDER BY inserted_at, stream_id, position + FOR UPDATE + LIMIT 1 + ) + DELETE FROM verification_audits v + WHERE v.inserted_at = (SELECT inserted_at FROM next_row) + AND v.stream_id = (SELECT stream_id FROM next_row) + AND v.position = (SELECT position FROM next_row) + RETURNING v.stream_id, v.position, v.expires_at, v.encrypted_size + ` + } + + err = vq.db.DB.QueryRowContext(ctx, getQuery).Scan(&seg.StreamID, &seg.Position, &seg.ExpiresAt, &seg.EncryptedSize) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return audit.Segment{}, audit.ErrEmptyQueue.Wrap(err) + } + return audit.Segment{}, Error.Wrap(err) + } + return seg, nil +} diff --git a/satellite/satellitedb/verifyqueue_test.go b/satellite/satellitedb/verifyqueue_test.go new file mode 100644 index 000000000..5eaeafab5 --- /dev/null +++ b/satellite/satellitedb/verifyqueue_test.go @@ -0,0 +1,105 @@ +// Copyright (C) 2022 Storj Labs, Inc. +// See LICENSE for copying information. + +package satellitedb_test + +import ( + "math/rand" + "sort" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "storj.io/common/testcontext" + "storj.io/common/testrand" + "storj.io/storj/satellite" + "storj.io/storj/satellite/audit" + "storj.io/storj/satellite/metabase" + "storj.io/storj/satellite/satellitedb/satellitedbtest" +) + +func TestVerifyQueueBasicUsage(t *testing.T) { + satellitedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db satellite.DB) { + verifyQueue := db.VerifyQueue() + + // generate random segment records + segmentsToVerify := make([]audit.Segment, 6) + for i := range segmentsToVerify { + segmentsToVerify[i].StreamID = testrand.UUID() + segmentsToVerify[i].Position = metabase.SegmentPositionFromEncoded(rand.Uint64()) + segmentsToVerify[i].EncryptedSize = rand.Int31() + } + expireTime := time.Now().Add(24 * time.Hour).Truncate(time.Microsecond) + segmentsToVerify[1].ExpiresAt = &expireTime + + // add these segments to the queue, 3 at a time + err := verifyQueue.Push(ctx, segmentsToVerify[0:3]) + require.NoError(t, err) + err = verifyQueue.Push(ctx, segmentsToVerify[3:6]) + require.NoError(t, err) + + // sort both sets of 3. segments inserted in the same call to Push + // can't be differentiated by insertion time, so they are ordered in the + // queue by (stream_id, position). We will sort our list here so that it + // matches the order we expect to receive them from the queue. + sort.Sort(byStreamIDAndPosition(segmentsToVerify[0:3])) + sort.Sort(byStreamIDAndPosition(segmentsToVerify[3:6])) + + // Pop all segments from the queue and check for a match with the input. + for _, expected := range segmentsToVerify { + popped, err := verifyQueue.Next(ctx) + require.NoError(t, err) + require.Equal(t, expected.StreamID, popped.StreamID) + require.Equal(t, expected.Position, popped.Position) + require.Equal(t, expected.ExpiresAt == nil, popped.ExpiresAt == nil) + if expected.ExpiresAt != nil { + require.Truef(t, expected.ExpiresAt.Equal(*popped.ExpiresAt), "expected %s but got %s", expected.ExpiresAt.Format(time.RFC3339), popped.ExpiresAt.Format(time.RFC3339)) + } + require.Equal(t, expected.EncryptedSize, popped.EncryptedSize) + } + + // Check that we got all segments. + popped, err := verifyQueue.Next(ctx) + require.Error(t, err) + require.Truef(t, audit.ErrEmptyQueue.Has(err), "unexpected error %v", err) + require.Equal(t, audit.Segment{}, popped) + }) +} + +type byStreamIDAndPosition []audit.Segment + +func (b byStreamIDAndPosition) Len() int { + return len(b) +} + +func (b byStreamIDAndPosition) Less(i, j int) bool { + comparison := b[i].StreamID.Compare(b[j].StreamID) + if comparison < 0 { + return true + } + if comparison > 0 { + return false + } + return b[i].Position.Less(b[j].Position) +} + +func (b byStreamIDAndPosition) Swap(i, j int) { + b[i], b[j] = b[j], b[i] +} + +func TestVerifyQueueEmpty(t *testing.T) { + satellitedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db satellite.DB) { + verifyQueue := db.VerifyQueue() + + // insert empty list + err := verifyQueue.Push(ctx, []audit.Segment{}) + require.NoError(t, err) + + // read from empty queue + popped, err := verifyQueue.Next(ctx) + require.Error(t, err) + require.Truef(t, audit.ErrEmptyQueue.Has(err), "unexpected error %v", err) + require.Equal(t, audit.Segment{}, popped) + }) +}