satellite/satellitedb/dbx: document accounting tables

Change-Id: I94478892a37623882d306a811e272f51865e48dd
This commit is contained in:
Egon Elbre 2023-02-23 14:39:43 +02:00
parent dfe3c87b87
commit e4914e9b7a
4 changed files with 95 additions and 22 deletions

View File

@ -1,8 +1,11 @@
// accounting_timestamps just allows us to save the last time/thing that happened // accounting_timestamps saves the last time an process completed.
model accounting_timestamps ( model accounting_timestamps (
key name key name
// name is one of "LastAtRestTally", "LastBandwidthTally" or "LastRollup".
// See the details in satellite/accounting/common.go for their meaning.
field name text field name text
// value is the last time that the specific event completed.
field value timestamp ( updatable ) field value timestamp ( updatable )
) )

View File

@ -1,20 +1,30 @@
// accounting_rollup stores rolled up bandwidth information for a storage node.
// Each row corresponds to a time-period between start_time and interval_end_time.
model accounting_rollup ( model accounting_rollup (
key node_id start_time key node_id start_time
index ( fields start_time ) index ( fields start_time )
// node_id refers to the storage node id.
field node_id blob field node_id blob
// start_time is when the total measurement started.
field start_time timestamp field start_time timestamp
// put_total is the number of bytes uploaded to the storagenode.
field put_total int64 field put_total int64
// get_total is the number of bytes downloaded from the storagenode.
field get_total int64 field get_total int64
// get_audit_total is the number of bytes downloaded from the storagenode for audit.
field get_audit_total int64 field get_audit_total int64
// get_repair_total is the number of bytes downloaded from the storagenode for repair.
field get_repair_total int64 field get_repair_total int64
// put_repair_total is the number of bytes uploaded to the storagenode for repair.
field put_repair_total int64 field put_repair_total int64
// at_rest_total is bytes*hour stored on the storagenode.
field at_rest_total float64 field at_rest_total float64
// interval_end_time is when the measurement finished and a new measurement began.
field interval_end_time timestamp ( updatable, nullable ) field interval_end_time timestamp ( updatable, nullable )
) )
// --- storage node accounting tables --- // // storagenode_bandwidth_rollup contains rolled up bandwidth information for a storage node.
model storagenode_bandwidth_rollup ( model storagenode_bandwidth_rollup (
key storagenode_id interval_start action key storagenode_id interval_start action
@ -23,12 +33,19 @@ model storagenode_bandwidth_rollup (
fields interval_start fields interval_start
) )
// storagenode_id refers to the storagenode storj.NodeID.
field storagenode_id blob field storagenode_id blob
// interval_start is when the total measurement started.
field interval_start timestamp field interval_start timestamp
// interval_seconds is the duration of the measurement interval in seconds.
field interval_seconds uint field interval_seconds uint
// action is the bandwidth operation, which refers to storj.io/common/pb.PieceAction.
// It is one of put = 1, get = 2, get audit = 3, get repair = 4, put repair = 5, delete = 6 or put graceful exit = 7.
field action uint field action uint
// allocated is the number of bytes that the satellite has budgeted for.
field allocated uint64 ( updatable, nullable, default 0 ) field allocated uint64 ( updatable, nullable, default 0 )
// settled is the number of bytes of traffic that the storagenodes have reported back to the satellite.
field settled uint64 ( updatable ) field settled uint64 ( updatable )
) )
@ -51,6 +68,7 @@ read paged (
where storagenode_bandwidth_rollup.interval_start >= ? where storagenode_bandwidth_rollup.interval_start >= ?
) )
// storagenode_bandwidth_rollup_archive contains rolled up values for storagenode_bandwidth_rollup table.
model storagenode_bandwidth_rollup_archive ( model storagenode_bandwidth_rollup_archive (
key storagenode_id interval_start action key storagenode_id interval_start action
@ -59,12 +77,19 @@ model storagenode_bandwidth_rollup_archive (
fields interval_start fields interval_start
) )
// storagenode_id refers to the storagenode storj.NodeID.
field storagenode_id blob field storagenode_id blob
// interval_start is when the total measurement started.
field interval_start timestamp field interval_start timestamp
// interval_seconds is the duration of the measurement interval in seconds.
field interval_seconds uint field interval_seconds uint
// action is the bandwidth operation, which refers to storj.io/common/pb.PieceAction.
// It is one of put = 1, get = 2, get audit = 3, get repair = 4, put repair = 5, delete = 6 or put graceful exit = 7.
field action uint field action uint
// allocated is the number of bytes that the satellite has budgeted for.
field allocated uint64 ( updatable, nullable, default 0 ) field allocated uint64 ( updatable, nullable, default 0 )
// settled is the number of bytes of traffic that the storagenodes have reported back to the satellite.
field settled uint64 ( updatable ) field settled uint64 ( updatable )
) )
@ -73,21 +98,25 @@ read paged (
where storagenode_bandwidth_rollup_archive.interval_start >= ? where storagenode_bandwidth_rollup_archive.interval_start >= ?
) )
/////////////////////////////////////// // storagenode_bandwidth_rollup_phase2 contains rolled up bandwidth information for a storage node.
// orders phase2->phase3 rollout table
///////////////////////////////////////
model storagenode_bandwidth_rollup_phase2 ( model storagenode_bandwidth_rollup_phase2 (
table storagenode_bandwidth_rollups_phase2 // make the pluralization consistent table storagenode_bandwidth_rollups_phase2 // make the pluralization consistent
key storagenode_id interval_start action key storagenode_id interval_start action
// storagenode_id refers to the storagenode storj.NodeID.
field storagenode_id blob field storagenode_id blob
// interval_start is when the total measurement started.
field interval_start timestamp field interval_start timestamp
// interval_seconds is the duration of the measurement interval in seconds.
field interval_seconds uint field interval_seconds uint
// action is the bandwidth operation, which refers to storj.io/common/pb.PieceAction.
// It is one of put = 1, get = 2, get audit = 3, get repair = 4, put repair = 5, delete = 6 or put graceful exit = 7.
field action uint field action uint
// allocated is the number of bytes that the satellite has budgeted for.
field allocated uint64 ( updatable, nullable, default 0 ) field allocated uint64 ( updatable, nullable, default 0 )
// settled is the number of bytes of traffic that the storagenodes have reported back to the satellite.
field settled uint64 ( updatable ) field settled uint64 ( updatable )
) )
@ -97,6 +126,7 @@ read paged (
where storagenode_bandwidth_rollup_phase2.interval_start >= ? where storagenode_bandwidth_rollup_phase2.interval_start >= ?
) )
// storagenode_storage_tally contains the calculated at rest data for storagenode.
model storagenode_storage_tally ( model storagenode_storage_tally (
// this primary key will enforce uniqueness on interval_end_time,node_id // this primary key will enforce uniqueness on interval_end_time,node_id
// and also creates an index on interval_end_time implicitly. // and also creates an index on interval_end_time implicitly.
@ -108,8 +138,11 @@ model storagenode_storage_tally (
index ( fields node_id ) index ( fields node_id )
// node_id refers to the storagenode storj.NodeID.
field node_id blob field node_id blob
// interval_end_time is when the measurement finished and a new measurement began.
field interval_end_time timestamp field interval_end_time timestamp
// data_total is the estimated bytes*hour stored on the storagenode.
field data_total float64 field data_total float64
) )

View File

@ -1,5 +1,4 @@
// --- bucket accounting tables --- // // bucket_bandwidth_roolup contains per bucket infromation about bandwidth used.
model bucket_bandwidth_rollup ( model bucket_bandwidth_rollup (
key bucket_name project_id interval_start action key bucket_name project_id interval_start action
index ( index (
@ -11,15 +10,25 @@ model bucket_bandwidth_rollup (
fields action interval_start project_id fields action interval_start project_id
) )
field bucket_name blob field bucket_name blob
// project_id is an UUID that refers to project.id.
field project_id blob field project_id blob
// interval_start is when the measurement interval started.
field interval_start timestamp field interval_start timestamp
// interval_seconds is the duration of the measurement interval in seconds.
field interval_seconds uint field interval_seconds uint
// action is the bandwidth operation, which refers to storj.io/common/pb.PieceAction.
// It is one of put = 1, get = 2, get audit = 3, get repair = 4, put repair = 5, delete = 6 or put graceful exit = 7.
field action uint field action uint
// inline is the number of bytes of inline traffic.
field inline uint64 ( updatable ) field inline uint64 ( updatable )
// allocated is the number of bytes that the satellite has budgeted for.
// It's unlikely that the uplink will use all the bandwidth that they requested.
field allocated uint64 ( updatable ) field allocated uint64 ( updatable )
// settled is the number of bytes of traffic that the storagenodes have reported back to the satellite.
field settled uint64 ( updatable ) field settled uint64 ( updatable )
) )
@ -28,6 +37,7 @@ read paged (
where bucket_bandwidth_rollup.interval_start >= ? where bucket_bandwidth_rollup.interval_start >= ?
) )
// bucket_bandwidth_rollup_archive contains historical condensed data from bucket_bandwidth_rollup table.
model bucket_bandwidth_rollup_archive ( model bucket_bandwidth_rollup_archive (
key bucket_name project_id interval_start action key bucket_name project_id interval_start action
index ( index (
@ -39,15 +49,25 @@ model bucket_bandwidth_rollup_archive (
fields action interval_start project_id fields action interval_start project_id
) )
// bucket_name refers to buckets.name table.
field bucket_name blob field bucket_name blob
// project_id is an UUID that refers to project.id.
field project_id blob field project_id blob
// interval_start is when the measurement interval started.
field interval_start timestamp field interval_start timestamp
// interval_seconds is the duration of the measurement interval in seconds.
field interval_seconds uint field interval_seconds uint
// action is the bandwidth operation, which refers to storj.io/common/pb.PieceAction.
// It is one of put = 1, get = 2, get audit = 3, get repair = 4, put repair = 5, delete = 6 or put graceful exit = 7.
field action uint field action uint
// inline is the number of bytes of inline traffic.
field inline uint64 ( updatable ) field inline uint64 ( updatable )
// allocated is the number of bytes that the satellite has budgeted for.
// It's unlikely that the uplink will use all the bandwidth that they requested.
field allocated uint64 ( updatable ) field allocated uint64 ( updatable )
// settled is the number of bytes of traffic that the storagenodes have reported back to the satellite.
field settled uint64 ( updatable ) field settled uint64 ( updatable )
) )
@ -56,6 +76,7 @@ read paged (
where bucket_bandwidth_rollup_archive.interval_start >= ? where bucket_bandwidth_rollup_archive.interval_start >= ?
) )
// project_bandwidth_daily_rollup contains the total traffic for a given project.
model project_bandwidth_daily_rollup ( model project_bandwidth_daily_rollup (
key project_id interval_day key project_id interval_day
index ( index (
@ -63,13 +84,19 @@ model project_bandwidth_daily_rollup (
fields interval_day fields interval_day
) )
// project_id is an UUID that refers to projects.id.
field project_id blob field project_id blob
// interval_day is the total for a given day.
field interval_day date field interval_day date
// egress_allocated is the total budgeted download traffic.
field egress_allocated uint64 ( updatable ) field egress_allocated uint64 ( updatable )
// egress_settled is the total download traffic that the storagenodes have reported.
field egress_settled uint64 ( updatable ) field egress_settled uint64 ( updatable )
// egress_dead is the amount of traffic that the uplink did not use.
field egress_dead uint64 ( updatable, default 0 ) field egress_dead uint64 ( updatable, default 0 )
) )
// bucket_storage_tally is the total at rest traffic for a given date.
model bucket_storage_tally ( model bucket_storage_tally (
key bucket_name project_id interval_start key bucket_name project_id interval_start
@ -78,20 +105,32 @@ model bucket_storage_tally (
fields project_id interval_start fields project_id interval_start
) )
// bucket_name refers to buckets.name table.
field bucket_name blob field bucket_name blob
// project_id is an UUID that refers to projects.id.
field project_id blob field project_id blob
// interval_start is when the measurement started.
// The measurement is collected over several hours, so it is not consistent.
field interval_start timestamp field interval_start timestamp
// total_bytes is the total bytes at rest.
field total_bytes uint64 ( default 0) field total_bytes uint64 ( default 0)
// inline is the amount of bytes stored in inline segments.
field inline uint64 field inline uint64
// remote is the amount of bytes stored on storagenodes, pre-expansion.
field remote uint64 field remote uint64
// total_segments_count is the count of all segments.
field total_segments_count uint ( default 0) field total_segments_count uint ( default 0)
// remote_segments_count is the count of segments that refer to storagenodes.
field remote_segments_count uint field remote_segments_count uint
// inline_segments_count is the count of segments that have inline data.
field inline_segments_count uint field inline_segments_count uint
// object_count is the count of objects.
field object_count uint field object_count uint
// metadata_size is the total bytes used by encrypted metadata.
field metadata_size uint64 field metadata_size uint64
) )

View File

@ -1,5 +1,3 @@
//--- repairqueue ---//
// repair_queue contains items that should be repaired. // repair_queue contains items that should be repaired.
model repair_queue ( model repair_queue (
table repair_queue table repair_queue