From e4914e9b7a7931226dc8ca838568470b719ce879 Mon Sep 17 00:00:00 2001 From: Egon Elbre Date: Thu, 23 Feb 2023 14:39:43 +0200 Subject: [PATCH] satellite/satellitedb/dbx: document accounting tables Change-Id: I94478892a37623882d306a811e272f51865e48dd --- satellite/satellitedb/dbx/accounting.dbx | 5 +- satellite/satellitedb/dbx/accounting_node.dbx | 67 ++++++++++++++----- .../satellitedb/dbx/accounting_project.dbx | 43 +++++++++++- satellite/satellitedb/dbx/repair.dbx | 2 - 4 files changed, 95 insertions(+), 22 deletions(-) diff --git a/satellite/satellitedb/dbx/accounting.dbx b/satellite/satellitedb/dbx/accounting.dbx index 4badcf69d..fb78e503d 100644 --- a/satellite/satellitedb/dbx/accounting.dbx +++ b/satellite/satellitedb/dbx/accounting.dbx @@ -1,8 +1,11 @@ -// accounting_timestamps just allows us to save the last time/thing that happened +// accounting_timestamps saves the last time an process completed. model accounting_timestamps ( key name + // name is one of "LastAtRestTally", "LastBandwidthTally" or "LastRollup". + // See the details in satellite/accounting/common.go for their meaning. field name text + // value is the last time that the specific event completed. field value timestamp ( updatable ) ) diff --git a/satellite/satellitedb/dbx/accounting_node.dbx b/satellite/satellitedb/dbx/accounting_node.dbx index 4eaa2220b..017d5316d 100644 --- a/satellite/satellitedb/dbx/accounting_node.dbx +++ b/satellite/satellitedb/dbx/accounting_node.dbx @@ -1,20 +1,30 @@ +// accounting_rollup stores rolled up bandwidth information for a storage node. +// Each row corresponds to a time-period between start_time and interval_end_time. model accounting_rollup ( key node_id start_time index ( fields start_time ) - field node_id blob - field start_time timestamp - field put_total int64 - field get_total int64 - field get_audit_total int64 - field get_repair_total int64 - field put_repair_total int64 - field at_rest_total float64 + // node_id refers to the storage node id. + field node_id blob + // start_time is when the total measurement started. + field start_time timestamp + // put_total is the number of bytes uploaded to the storagenode. + field put_total int64 + // get_total is the number of bytes downloaded from the storagenode. + field get_total int64 + // get_audit_total is the number of bytes downloaded from the storagenode for audit. + field get_audit_total int64 + // get_repair_total is the number of bytes downloaded from the storagenode for repair. + field get_repair_total int64 + // put_repair_total is the number of bytes uploaded to the storagenode for repair. + field put_repair_total int64 + // at_rest_total is bytes*hour stored on the storagenode. + field at_rest_total float64 + // interval_end_time is when the measurement finished and a new measurement began. field interval_end_time timestamp ( updatable, nullable ) ) -// --- storage node accounting tables --- // - +// storagenode_bandwidth_rollup contains rolled up bandwidth information for a storage node. model storagenode_bandwidth_rollup ( key storagenode_id interval_start action @@ -23,12 +33,19 @@ model storagenode_bandwidth_rollup ( fields interval_start ) + // storagenode_id refers to the storagenode storj.NodeID. field storagenode_id blob + // interval_start is when the total measurement started. field interval_start timestamp + // interval_seconds is the duration of the measurement interval in seconds. field interval_seconds uint + // action is the bandwidth operation, which refers to storj.io/common/pb.PieceAction. + // It is one of put = 1, get = 2, get audit = 3, get repair = 4, put repair = 5, delete = 6 or put graceful exit = 7. field action uint + // allocated is the number of bytes that the satellite has budgeted for. field allocated uint64 ( updatable, nullable, default 0 ) + // settled is the number of bytes of traffic that the storagenodes have reported back to the satellite. field settled uint64 ( updatable ) ) @@ -51,6 +68,7 @@ read paged ( where storagenode_bandwidth_rollup.interval_start >= ? ) +// storagenode_bandwidth_rollup_archive contains rolled up values for storagenode_bandwidth_rollup table. model storagenode_bandwidth_rollup_archive ( key storagenode_id interval_start action @@ -59,12 +77,19 @@ model storagenode_bandwidth_rollup_archive ( fields interval_start ) + // storagenode_id refers to the storagenode storj.NodeID. field storagenode_id blob + // interval_start is when the total measurement started. field interval_start timestamp + // interval_seconds is the duration of the measurement interval in seconds. field interval_seconds uint + // action is the bandwidth operation, which refers to storj.io/common/pb.PieceAction. + // It is one of put = 1, get = 2, get audit = 3, get repair = 4, put repair = 5, delete = 6 or put graceful exit = 7. field action uint + // allocated is the number of bytes that the satellite has budgeted for. field allocated uint64 ( updatable, nullable, default 0 ) + // settled is the number of bytes of traffic that the storagenodes have reported back to the satellite. field settled uint64 ( updatable ) ) @@ -73,21 +98,25 @@ read paged ( where storagenode_bandwidth_rollup_archive.interval_start >= ? ) -/////////////////////////////////////// -// orders phase2->phase3 rollout table -/////////////////////////////////////// - +// storagenode_bandwidth_rollup_phase2 contains rolled up bandwidth information for a storage node. model storagenode_bandwidth_rollup_phase2 ( table storagenode_bandwidth_rollups_phase2 // make the pluralization consistent key storagenode_id interval_start action + // storagenode_id refers to the storagenode storj.NodeID. field storagenode_id blob + // interval_start is when the total measurement started. field interval_start timestamp + // interval_seconds is the duration of the measurement interval in seconds. field interval_seconds uint + // action is the bandwidth operation, which refers to storj.io/common/pb.PieceAction. + // It is one of put = 1, get = 2, get audit = 3, get repair = 4, put repair = 5, delete = 6 or put graceful exit = 7. field action uint + // allocated is the number of bytes that the satellite has budgeted for. field allocated uint64 ( updatable, nullable, default 0 ) + // settled is the number of bytes of traffic that the storagenodes have reported back to the satellite. field settled uint64 ( updatable ) ) @@ -97,6 +126,7 @@ read paged ( where storagenode_bandwidth_rollup_phase2.interval_start >= ? ) +// storagenode_storage_tally contains the calculated at rest data for storagenode. model storagenode_storage_tally ( // this primary key will enforce uniqueness on interval_end_time,node_id // and also creates an index on interval_end_time implicitly. @@ -108,9 +138,12 @@ model storagenode_storage_tally ( index ( fields node_id ) - field node_id blob - field interval_end_time timestamp - field data_total float64 + // node_id refers to the storagenode storj.NodeID. + field node_id blob + // interval_end_time is when the measurement finished and a new measurement began. + field interval_end_time timestamp + // data_total is the estimated bytes*hour stored on the storagenode. + field data_total float64 ) read all ( diff --git a/satellite/satellitedb/dbx/accounting_project.dbx b/satellite/satellitedb/dbx/accounting_project.dbx index 80c95ab6c..f1a13a93d 100644 --- a/satellite/satellitedb/dbx/accounting_project.dbx +++ b/satellite/satellitedb/dbx/accounting_project.dbx @@ -1,5 +1,4 @@ -// --- bucket accounting tables --- // - +// bucket_bandwidth_roolup contains per bucket infromation about bandwidth used. model bucket_bandwidth_rollup ( key bucket_name project_id interval_start action index ( @@ -11,15 +10,25 @@ model bucket_bandwidth_rollup ( fields action interval_start project_id ) + field bucket_name blob + // project_id is an UUID that refers to project.id. field project_id blob + // interval_start is when the measurement interval started. field interval_start timestamp + // interval_seconds is the duration of the measurement interval in seconds. field interval_seconds uint + // action is the bandwidth operation, which refers to storj.io/common/pb.PieceAction. + // It is one of put = 1, get = 2, get audit = 3, get repair = 4, put repair = 5, delete = 6 or put graceful exit = 7. field action uint + // inline is the number of bytes of inline traffic. field inline uint64 ( updatable ) + // allocated is the number of bytes that the satellite has budgeted for. + // It's unlikely that the uplink will use all the bandwidth that they requested. field allocated uint64 ( updatable ) + // settled is the number of bytes of traffic that the storagenodes have reported back to the satellite. field settled uint64 ( updatable ) ) @@ -28,6 +37,7 @@ read paged ( where bucket_bandwidth_rollup.interval_start >= ? ) +// bucket_bandwidth_rollup_archive contains historical condensed data from bucket_bandwidth_rollup table. model bucket_bandwidth_rollup_archive ( key bucket_name project_id interval_start action index ( @@ -39,15 +49,25 @@ model bucket_bandwidth_rollup_archive ( fields action interval_start project_id ) + // bucket_name refers to buckets.name table. field bucket_name blob + // project_id is an UUID that refers to project.id. field project_id blob + // interval_start is when the measurement interval started. field interval_start timestamp + // interval_seconds is the duration of the measurement interval in seconds. field interval_seconds uint + // action is the bandwidth operation, which refers to storj.io/common/pb.PieceAction. + // It is one of put = 1, get = 2, get audit = 3, get repair = 4, put repair = 5, delete = 6 or put graceful exit = 7. field action uint + // inline is the number of bytes of inline traffic. field inline uint64 ( updatable ) + // allocated is the number of bytes that the satellite has budgeted for. + // It's unlikely that the uplink will use all the bandwidth that they requested. field allocated uint64 ( updatable ) + // settled is the number of bytes of traffic that the storagenodes have reported back to the satellite. field settled uint64 ( updatable ) ) @@ -56,6 +76,7 @@ read paged ( where bucket_bandwidth_rollup_archive.interval_start >= ? ) +// project_bandwidth_daily_rollup contains the total traffic for a given project. model project_bandwidth_daily_rollup ( key project_id interval_day index ( @@ -63,13 +84,19 @@ model project_bandwidth_daily_rollup ( fields interval_day ) + // project_id is an UUID that refers to projects.id. field project_id blob + // interval_day is the total for a given day. field interval_day date + // egress_allocated is the total budgeted download traffic. field egress_allocated uint64 ( updatable ) + // egress_settled is the total download traffic that the storagenodes have reported. field egress_settled uint64 ( updatable ) + // egress_dead is the amount of traffic that the uplink did not use. field egress_dead uint64 ( updatable, default 0 ) ) +// bucket_storage_tally is the total at rest traffic for a given date. model bucket_storage_tally ( key bucket_name project_id interval_start @@ -78,20 +105,32 @@ model bucket_storage_tally ( fields project_id interval_start ) + // bucket_name refers to buckets.name table. field bucket_name blob + // project_id is an UUID that refers to projects.id. field project_id blob + // interval_start is when the measurement started. + // The measurement is collected over several hours, so it is not consistent. field interval_start timestamp + // total_bytes is the total bytes at rest. field total_bytes uint64 ( default 0) + // inline is the amount of bytes stored in inline segments. field inline uint64 + // remote is the amount of bytes stored on storagenodes, pre-expansion. field remote uint64 + // total_segments_count is the count of all segments. field total_segments_count uint ( default 0) + // remote_segments_count is the count of segments that refer to storagenodes. field remote_segments_count uint + // inline_segments_count is the count of segments that have inline data. field inline_segments_count uint + // object_count is the count of objects. field object_count uint + // metadata_size is the total bytes used by encrypted metadata. field metadata_size uint64 ) diff --git a/satellite/satellitedb/dbx/repair.dbx b/satellite/satellitedb/dbx/repair.dbx index 682adf256..9fb6c5149 100644 --- a/satellite/satellitedb/dbx/repair.dbx +++ b/satellite/satellitedb/dbx/repair.dbx @@ -1,5 +1,3 @@ -//--- repairqueue ---// - // repair_queue contains items that should be repaired. model repair_queue ( table repair_queue