storj/satellite/satellitedb/dbx/satellitedb.dbx
Moby von Briesen 4e5a7f13c7 satellite/repair/queue: Prioritize selection of items off repair queue by segment health
Add a column to the repair queue table in the satellite db for healthy
piece count. When an item is selected from the repair queue, the least
durable segment that has not been attempted in the past hour should be
selected first. This prevents our repairer from getting stuck doing work
on segments that are close to the repair threshold while allowing
segments that are more unhealthy to degrade further.

The migration also clears the repair queue so that the migration runs
quickly and we can properly account for segment health in future repair
work.

We do not select items off the repair queue that have been attempted in
the past six hours. This was changed from on hour to allow us time to
try a wider variety of segments when the repair queue is very large.

Change-Id: Iaf183f1e5fd45cd792a52e3563a3e43a2b9f410b
2020-02-26 09:54:16 -05:00

1207 lines
27 KiB
Plaintext

// dbx.v1 golang satellitedb.dbx .
//--- Value Attribution ---//
model value_attribution (
key project_id bucket_name
field project_id blob
field bucket_name blob
field partner_id blob
field last_updated utimestamp ( autoinsert, autoupdate )
)
create value_attribution ()
delete value_attribution (
where value_attribution.project_id = ?
where value_attribution.bucket_name = ?
)
read one (
select value_attribution
where value_attribution.project_id = ?
where value_attribution.bucket_name = ?
)
//--- containment ---//
model pending_audits (
key node_id
field node_id blob
field piece_id blob
field stripe_index int64
field share_size int64
field expected_share_hash blob
field reverify_count int64 ( updatable )
field path blob
)
create pending_audits ( )
update pending_audits ( where pending_audits.node_id = ? )
delete pending_audits ( where pending_audits.node_id = ? )
read one (
select pending_audits
where pending_audits.node_id = ?
)
//--- irreparableDB ---//
model irreparabledb (
key segmentpath
field segmentpath blob
field segmentdetail blob ( updatable )
field pieces_lost_count int64 ( updatable )
field seg_damaged_unix_sec int64 ( updatable )
field repair_attempt_count int64 ( updatable )
)
create irreparabledb ( noreturn )
update irreparabledb (
where irreparabledb.segmentpath = ?
noreturn
)
delete irreparabledb ( where irreparabledb.segmentpath = ? )
read one (
select irreparabledb
where irreparabledb.segmentpath = ?
)
read limitoffset (
select irreparabledb
where irreparabledb.segmentpath > ?
orderby asc irreparabledb.segmentpath
)
//--- accounting ---//
// accounting_timestamps just allows us to save the last time/thing that happened
model accounting_timestamps (
key name
field name text
field value timestamp ( updatable )
)
create accounting_timestamps ( noreturn )
update accounting_timestamps (
where accounting_timestamps.name = ?
noreturn
)
read scalar (
select accounting_timestamps.value
where accounting_timestamps.name = ?
)
model accounting_rollup (
key id
field id serial64
field node_id blob
field start_time timestamp
field put_total int64
field get_total int64
field get_audit_total int64
field get_repair_total int64
field put_repair_total int64
field at_rest_total float64
)
create accounting_rollup ( noreturn )
delete accounting_rollup ( where accounting_rollup.id = ? )
read one (
select accounting_rollup
where accounting_rollup.id = ?
)
read all (
select accounting_rollup
where accounting_rollup.start_time >= ?
)
//--- overlay cache ---//
model node (
key id
index (
name node_last_ip
fields last_net
)
field id blob
field address text ( updatable ) // TODO: use compressed format
field last_net text ( updatable )
field protocol int ( updatable )
field type int ( updatable )
field email text ( updatable )
field wallet text ( updatable ) // TODO: use compressed format
field free_bandwidth int64 ( updatable )
field free_disk int64 ( updatable )
field piece_count int64 ( autoinsert, updatable )
field major int64 ( updatable )
field minor int64 ( updatable )
field patch int64 ( updatable )
field hash text ( updatable )
field timestamp timestamp ( updatable )
field release bool ( updatable )
field latency_90 int64 ( updatable )
field audit_success_count int64 ( updatable )
field total_audit_count int64 ( updatable )
field uptime_success_count int64 ( updatable )
field total_uptime_count int64 ( updatable )
field created_at timestamp ( autoinsert )
field updated_at timestamp ( autoinsert, autoupdate )
field last_contact_success timestamp ( updatable )
field last_contact_failure timestamp ( updatable )
field contained bool ( updatable )
field disqualified timestamp ( updatable, nullable )
field audit_reputation_alpha float64 ( updatable )
field audit_reputation_beta float64 ( updatable )
field uptime_reputation_alpha float64 ( updatable )
field uptime_reputation_beta float64 ( updatable )
field exit_initiated_at utimestamp ( updatable, nullable )
field exit_loop_completed_at utimestamp ( updatable, nullable )
field exit_finished_at utimestamp ( updatable, nullable )
field exit_success bool ( updatable )
)
create node ( noreturn )
update node ( where node.id = ? )
update node (
where node.id = ?
noreturn
)
delete node ( where node.id = ? )
// "Get" query; fails if node not found
read one (
select node
where node.id = ?
)
read all (
select node.id
)
read limitoffset (
select node
where node.id >= ?
orderby asc node.id
)
read limitoffset (
select node.id node.last_net node.address node.protocol
where node.id >= ?
where node.disqualified = null
orderby asc node.id
)
read all (
select node.id node.piece_count
where node.piece_count != 0
)
read limitoffset (
select node.id node.address node.last_contact_success node.last_contact_failure
where node.last_contact_success < node.last_contact_failure
where node.disqualified = null
orderby asc node.last_contact_failure
)
read all (
select node.id node.address node.last_contact_success node.last_contact_failure
where node.last_contact_success < ?
where node.last_contact_success > node.last_contact_failure
where node.disqualified = null
orderby asc node.last_contact_success
)
//--- repairqueue ---//
model injuredsegment (
key path
field path blob
field data blob
field attempted utimestamp (updatable, nullable)
field num_healthy_pieces int (default 52)
index (
fields attempted
)
index (
fields num_healthy_pieces
)
)
//--- satellite console ---//
model user (
key id
field id blob
field email text ( updatable )
field normalized_email text ( updatable )
field full_name text ( updatable )
field short_name text ( updatable, nullable )
field password_hash blob ( updatable )
field status int ( updatable, autoinsert )
field partner_id blob ( nullable )
field created_at timestamp ( autoinsert )
)
create user ( )
update user ( where user.id = ? )
delete user ( where user.id = ? )
read one (
select user
where user.normalized_email = ?
where user.status != 0
)
read one (
select user
where user.id = ?
)
model project (
key id
field id blob
field name text
field description text ( updatable )
field usage_limit int64 ( updatable )
field rate_limit int ( nullable, updatable )
field partner_id blob ( nullable )
field owner_id blob
field created_at timestamp ( autoinsert )
)
create project ( )
update project ( where project.id = ? )
delete project ( where project.id = ? )
read one (
select project
where project.id = ?
)
read one (
select project.usage_limit
where project.id = ?
)
read all (
select project
)
read all (
select project
where project.created_at < ?
orderby asc project.created_at
)
read all (
select project
where project.owner_id = ?
orderby asc project.created_at
)
read all (
select project
join project.id = project_member.project_id
where project_member.member_id = ?
orderby asc project.name
)
read limitoffset (
select project
where project.created_at < ?
orderby asc project.created_at
)
model project_invoice_stamp (
key project_id start_date end_date
unique invoice_id
field project_id project.id cascade
field invoice_id blob
field start_date timestamp
field end_date timestamp
field created_at timestamp
)
create project_invoice_stamp ( )
read one (
select project_invoice_stamp
where project_invoice_stamp.project_id = ?
where project_invoice_stamp.start_date = ?
)
// pagination ?
read all (
select project_invoice_stamp
where project_invoice_stamp.project_id = ?
orderby desc project_invoice_stamp.start_date
)
model project_member (
key member_id project_id
field member_id user.id cascade
field project_id project.id cascade
field created_at timestamp ( autoinsert )
)
create project_member ( )
delete project_member (
where project_member.member_id = ?
where project_member.project_id = ?
)
read all (
select project_member
where project_member.member_id = ?
)
read limitoffset (
select project_member
where project_member.project_id = ?
)
model api_key (
key id
unique head
unique name project_id
field id blob
field project_id project.id cascade
field head blob
field name text (updatable)
field secret blob
field partner_id blob (nullable)
field created_at timestamp (autoinsert)
)
create api_key ( )
update api_key (
where api_key.id = ?
noreturn
)
delete api_key ( where api_key.id = ? )
read one (
select api_key
where api_key.id = ?
)
read one (
select api_key
where api_key.head = ?
)
read one (
select api_key
where api_key.name = ?
where api_key.project_id = ?
)
read all (
select api_key
where api_key.project_id = ?
orderby asc api_key.name
)
//--- tracking serial numbers ---//
model serial_number (
key id
index (
name serial_number
fields serial_number
unique
)
field id serial
field serial_number blob
field bucket_id blob
field expires_at utimestamp
index (
fields expires_at
)
)
model used_serial (
key serial_number_id storage_node_id
field serial_number_id serial_number.id cascade
field storage_node_id blob
)
// inserting a new serial number
create serial_number ( noreturn )
// finding out information about the serial number
read scalar (
select serial_number
where serial_number.serial_number = ?
)
// deleting expired serial numbers
delete serial_number (
where serial_number.expires_at <= ?
)
// for preventing duplicate serial numbers
create used_serial ( noreturn )
//
// DEPRECATED! vvvvvvvvvvvvvvvvvvvvvvvvvvv
//
model reported_serial (
key expires_at storage_node_id bucket_id action serial_number
field expires_at timestamp // ceil'd to the next day
field storage_node_id blob // involved node id
field bucket_id blob // involved project id and bucket name
field action uint // action (get, put, audit, etc. see protobuf)
field serial_number blob // identifies order (required for uniqueness)
field settled uint64
field observed_at timestamp
)
//
// DEPRECATED! ^^^^^^^^^^^^^^^^^^^^^^^^^^
//
model pending_serial_queue (
table pending_serial_queue
key storage_node_id bucket_id serial_number
field storage_node_id blob
field bucket_id blob
field serial_number blob
field action uint
field settled uint64
field expires_at timestamp
)
create pending_serial_queue ( noreturn, replace )
read paged ( select pending_serial_queue )
model consumed_serial (
key storage_node_id serial_number
index ( fields expires_at )
field storage_node_id blob
field serial_number blob
field expires_at timestamp
)
create consumed_serial ( noreturn )
delete consumed_serial ( where consumed_serial.expires_at <= ? )
read has (
select consumed_serial
where consumed_serial.storage_node_id = ?
where consumed_serial.serial_number = ?
)
// --- bucket accounting tables --- //
model bucket_bandwidth_rollup (
key bucket_name project_id interval_start action
field bucket_name blob
field project_id blob
field interval_start utimestamp
field interval_seconds uint
field action uint
field inline uint64 ( updatable )
field allocated uint64 ( updatable )
field settled uint64 ( updatable )
)
read scalar (
select bucket_bandwidth_rollup
where bucket_bandwidth_rollup.bucket_name = ?
where bucket_bandwidth_rollup.project_id = ?
where bucket_bandwidth_rollup.interval_start = ?
where bucket_bandwidth_rollup.action = ?
)
model bucket_storage_tally (
key bucket_name project_id interval_start
field bucket_name blob
field project_id blob
field interval_start utimestamp
field inline uint64
field remote uint64
field remote_segments_count uint
field inline_segments_count uint
field object_count uint
field metadata_size uint64
)
create bucket_storage_tally ( noreturn )
read first (
select bucket_storage_tally
where bucket_storage_tally.project_id = ?
orderby desc bucket_storage_tally.interval_start
)
read all (
select bucket_storage_tally
)
read all (
select bucket_storage_tally
where bucket_storage_tally.project_id = ?
where bucket_storage_tally.bucket_name = ?
where bucket_storage_tally.interval_start >= ?
where bucket_storage_tally.interval_start <= ?
orderby desc bucket_storage_tally.interval_start
)
// --- storage node accounting tables --- //
model storagenode_bandwidth_rollup (
key storagenode_id interval_start action
field storagenode_id blob
field interval_start utimestamp
field interval_seconds uint
field action uint
field allocated uint64 ( updatable, nullable, default 0 )
field settled uint64 ( updatable )
)
read scalar (
select storagenode_bandwidth_rollup
where storagenode_bandwidth_rollup.storagenode_id = ?
where storagenode_bandwidth_rollup.interval_start = ?
where storagenode_bandwidth_rollup.action = ?
)
read all (
select storagenode_bandwidth_rollup
where storagenode_bandwidth_rollup.interval_start >= ?
)
model storagenode_storage_tally (
key id
field id serial64
field node_id blob
field interval_end_time timestamp
field data_total float64
)
create storagenode_storage_tally ( noreturn )
delete storagenode_storage_tally ( where storagenode_storage_tally.id = ? )
read one (
select storagenode_storage_tally
where storagenode_storage_tally.id = ?
)
read all (
select storagenode_storage_tally
)
read all (
select storagenode_storage_tally
where storagenode_storage_tally.interval_end_time >= ?
)
//--- peer_identity ---//
model peer_identity (
key node_id
field node_id blob
field leaf_serial_number blob (updatable)
field chain blob (updatable) // x509 ASN.1 DER content
field updated_at timestamp ( autoinsert, autoupdate )
)
create peer_identity ( noreturn )
update peer_identity (
where peer_identity.node_id = ?
noreturn
)
read one (
select peer_identity
where peer_identity.node_id = ?
)
read one (
select peer_identity.leaf_serial_number
where peer_identity.node_id = ?
)
//--- satellite registration token for Vanguard release (temporary table) ---//
model registration_token (
key secret
unique owner_id
field secret blob
field owner_id blob ( updatable, nullable )
field project_limit int
field created_at timestamp ( autoinsert )
)
create registration_token ( )
read one (
select registration_token
where registration_token.secret = ?
)
read one (
select registration_token
where registration_token.owner_id = ?
)
update registration_token ( where registration_token.secret = ? )
//--- satellite reset password token ---//
model reset_password_token (
key secret
unique owner_id
field secret blob
field owner_id blob ( updatable )
field created_at timestamp ( autoinsert )
)
create reset_password_token ( )
read one (
select reset_password_token
where reset_password_token.secret = ?
)
read one (
select reset_password_token
where reset_password_token.owner_id = ?
)
delete reset_password_token ( where reset_password_token.secret = ? )
//--- offer table ---//
model offer (
key id
field id serial
field name text ( updatable )
field description text ( updatable )
field award_credit_in_cents int ( updatable )
field invitee_credit_in_cents int ( updatable )
field award_credit_duration_days int ( updatable, nullable )
field invitee_credit_duration_days int ( updatable, nullable )
field redeemable_cap int ( updatable, nullable )
field expires_at timestamp ( updatable )
field created_at timestamp ( autoinsert )
// status corresponds to the values of rewards.OfferStatus
field status int ( updatable )
// type corresponds to the values of rewards.OfferType
field type int ( updatable )
)
read one (
select offer
where offer.id = ?
)
read all (
select offer
orderby asc offer.id
)
update offer (
where offer.id = ?
noreturn
)
create offer ( )
//--- user credit table ---//
model user_credit (
key id
unique id offer_id
index (
name credits_earned_user_id_offer_id
fields id offer_id
unique
)
field id serial
field user_id user.id cascade
field offer_id offer.id restrict
field referred_by user.id setnull ( nullable )
field type text
field credits_earned_in_cents int
field credits_used_in_cents int ( updatable, autoinsert )
field expires_at timestamp ( updatable )
field created_at timestamp ( autoinsert )
)
read all (
select user_credit
where user_credit.user_id = ?
where user_credit.expires_at > ?
where user_credit.credits_used_in_cents < user_credit.credits_earned_in_cents
orderby asc user_credit.expires_at
)
read count (
select user_credit
where user_credit.referred_by = ?
)
create user_credit ()
//--- metainfo buckets ---//
model bucket_metainfo (
key id
unique name project_id
field id blob
field project_id project.id restrict
field name blob
field partner_id blob (nullable, updatable)
field path_cipher int
field created_at timestamp ( autoinsert )
field default_segment_size int (updatable)
field default_encryption_cipher_suite int (updatable)
field default_encryption_block_size int (updatable)
field default_redundancy_algorithm int (updatable)
field default_redundancy_share_size int (updatable)
field default_redundancy_required_shares int (updatable)
field default_redundancy_repair_shares int (updatable)
field default_redundancy_optimal_shares int (updatable)
field default_redundancy_total_shares int (updatable)
)
create bucket_metainfo ()
update bucket_metainfo (
where bucket_metainfo.project_id = ?
where bucket_metainfo.name = ?
)
read one (
select bucket_metainfo
where bucket_metainfo.project_id = ?
where bucket_metainfo.name = ?
)
delete bucket_metainfo (
where bucket_metainfo.project_id = ?
where bucket_metainfo.name = ?
)
read limitoffset ( // Forward
select bucket_metainfo
where bucket_metainfo.project_id = ?
where bucket_metainfo.name >= ?
orderby asc bucket_metainfo.name
)
read limitoffset ( // After
select bucket_metainfo
where bucket_metainfo.project_id = ?
where bucket_metainfo.name > ?
orderby asc bucket_metainfo.name
)
//--- graceful exit progress ---//
model graceful_exit_progress (
table graceful_exit_progress
key node_id
field node_id blob
field bytes_transferred int64 ( updatable )
field pieces_transferred int64 ( autoinsert, updatable )
field pieces_failed int64 ( autoinsert, updatable )
field updated_at utimestamp ( autoinsert, autoupdate )
)
create graceful_exit_progress ( noreturn )
update graceful_exit_progress (
where graceful_exit_progress.node_id = ?
noreturn
)
delete graceful_exit_progress ( where graceful_exit_progress.node_id = ? )
read one (
select graceful_exit_progress
where graceful_exit_progress.node_id = ?
)
//--- graceful exit transfer queue ---//
model graceful_exit_transfer_queue (
table graceful_exit_transfer_queue
key node_id path piece_num
field node_id blob
field path blob
field piece_num int
field root_piece_id blob ( nullable )
field durability_ratio float64 ( updatable )
field queued_at utimestamp ( autoinsert )
field requested_at utimestamp ( updatable, nullable )
field last_failed_at utimestamp ( updatable, nullable )
field last_failed_code int ( updatable, nullable )
field failed_count int ( updatable, nullable )
field finished_at utimestamp ( updatable, nullable )
field order_limit_send_count int ( updatable )
)
create graceful_exit_transfer_queue ( noreturn )
update graceful_exit_transfer_queue (
where graceful_exit_transfer_queue.node_id = ?
where graceful_exit_transfer_queue.path = ?
where graceful_exit_transfer_queue.piece_num = ?
noreturn
)
delete graceful_exit_transfer_queue (
where graceful_exit_transfer_queue.node_id = ?
)
delete graceful_exit_transfer_queue (
where graceful_exit_transfer_queue.node_id = ?
where graceful_exit_transfer_queue.path = ?
where graceful_exit_transfer_queue.piece_num = ?
)
delete graceful_exit_transfer_queue (
where graceful_exit_transfer_queue.node_id = ?
where graceful_exit_transfer_queue.finished_at != null
)
read one (
select graceful_exit_transfer_queue
where graceful_exit_transfer_queue.node_id = ?
where graceful_exit_transfer_queue.path = ?
where graceful_exit_transfer_queue.piece_num = ?
)
//--- downtime tracking ---//
model nodes_offline_time (
key node_id tracked_at
index (
fields node_id
)
field node_id blob
field tracked_at timestamp
field seconds int
)
create nodes_offline_time ()
read all (
select nodes_offline_time
where nodes_offline_time.node_id = ?
where nodes_offline_time.tracked_at > ?
where nodes_offline_time.tracked_at <= ?
)
//--- satellite payments ---//
model stripe_customer (
key user_id
unique customer_id
field user_id blob
field customer_id text
field created_at timestamp ( autoinsert )
)
create stripe_customer ( )
read one (
select stripe_customer.customer_id
where stripe_customer.user_id = ?
)
read limitoffset (
select stripe_customer
where stripe_customer.created_at <= ?
orderby desc stripe_customer.created_at
)
model coinpayments_transaction (
key id
field id text
field user_id blob
field address text
field amount blob
field received blob ( updatable )
field status int ( updatable )
field key text
field timeout int
field created_at timestamp ( autoinsert )
)
create coinpayments_transaction ()
update coinpayments_transaction ( where coinpayments_transaction.id = ? )
read all (
select coinpayments_transaction
where coinpayments_transaction.user_id = ?
orderby desc coinpayments_transaction.created_at
)
read limitoffset (
select coinpayments_transaction
where coinpayments_transaction.created_at <= ?
where coinpayments_transaction.status = ?
orderby desc coinpayments_transaction.created_at
)
model stripecoinpayments_apply_balance_intent (
key tx_id
field tx_id coinpayments_transaction.id cascade
field state int ( updatable )
field created_at timestamp ( autoinsert )
)
create stripecoinpayments_apply_balance_intent ()
update stripecoinpayments_apply_balance_intent (
where stripecoinpayments_apply_balance_intent.tx_id = ?
)
delete stripecoinpayments_apply_balance_intent (
where stripecoinpayments_apply_balance_intent.tx_id = ?
)
model stripecoinpayments_invoice_project_record (
key id
unique project_id period_start period_end
field id blob
field project_id blob
field storage float64
field egress int64
field objects int64
field period_start timestamp
field period_end timestamp
field state int ( updatable )
field created_at timestamp ( autoinsert )
)
create stripecoinpayments_invoice_project_record ()
update stripecoinpayments_invoice_project_record (
where stripecoinpayments_invoice_project_record.id = ?
)
delete stripecoinpayments_invoice_project_record (
where stripecoinpayments_invoice_project_record.id = ?
)
read one (
select stripecoinpayments_invoice_project_record
where stripecoinpayments_invoice_project_record.project_id = ?
where stripecoinpayments_invoice_project_record.period_start = ?
where stripecoinpayments_invoice_project_record.period_end = ?
)
read limitoffset (
select stripecoinpayments_invoice_project_record
where stripecoinpayments_invoice_project_record.created_at <= ?
where stripecoinpayments_invoice_project_record.state = ?
orderby desc stripecoinpayments_invoice_project_record.created_at
)
model stripecoinpayments_tx_conversion_rate (
key tx_id
field tx_id text
field rate blob
field created_at timestamp ( autoinsert )
)
create stripecoinpayments_tx_conversion_rate ()
read one (
select stripecoinpayments_tx_conversion_rate
where stripecoinpayments_tx_conversion_rate.tx_id = ?
)
model coupon (
key id
field id blob
field project_id blob
field user_id blob
field amount int64
field description text
field type int
field status int ( updatable )
field duration int64
field created_at timestamp ( autoinsert )
)
create coupon ()
update coupon (
where coupon.id = ?
)
delete coupon (
where coupon.id = ?
)
read one (
select coupon
where coupon.id = ?
)
read all (
select coupon
where coupon.project_id = ?
where coupon.status = 0
orderby desc coupon.created_at
)
read all (
select coupon
where coupon.user_id = ?
orderby desc coupon.created_at
)
read all (
select coupon
where coupon.user_id = ?
where coupon.status = ?
orderby desc coupon.created_at
)
read all (
select coupon
where coupon.status = ?
orderby desc coupon.created_at
)
read limitoffset (
select coupon
where coupon.created_at <= ?
where coupon.status = ?
orderby desc coupon.created_at
)
model coupon_usage (
key coupon_id period
field coupon_id blob
field amount int64
field status int ( updatable )
field period timestamp
)
create coupon_usage ()
read limitoffset (
select coupon_usage
where coupon_usage.period <= ?
where coupon_usage.status = 0
orderby desc coupon_usage.period
)
update coupon_usage (
where coupon_usage.coupon_id = ?
where coupon_usage.period = ?
)
model credit (
key transaction_id
field user_id blob
field transaction_id text
field amount int64
field created_at timestamp ( autoinsert )
)
create credit ( )
read one (
select credit
where credit.transaction_id = ?
)
read all (
select credit
where credit.user_id = ?
orderby desc credit.created_at
)
read limitoffset (
select credit
where credit.user_id = ?
where credit.created_at <= ?
orderby desc credit.created_at
)
model credits_spending (
key id
field id blob
field user_id blob
field project_id blob
field amount int64
field status int ( updatable )
field created_at timestamp ( autoinsert )
)
create credits_spending ( )
update credits_spending (
where credits_spending.id = ?
)
read all (
select credits_spending
where credits_spending.user_id = ?
orderby desc credits_spending.created_at
)
read limitoffset (
select credits_spending
where credits_spending.created_at <= ?
where credits_spending.status = ?
orderby desc credits_spending.created_at
)