Revert "satellite/db: fix long loadtime for charges endpoint"
This reverts commit 676178299f.
Reason for revert:
The new query used by this commit performs a full table scan.
It's been reverted pending a fix for that.
Change-Id: Idc53954459aa6f5a692056232b8674b11d1928ce
This commit is contained in:
Wilfred Asomani
parent
3180e09750
commit
74b3617813
@ -549,67 +549,55 @@ func (db *ProjectAccounting) GetProjectTotal(ctx context.Context, projectID uuid
|
||||
func (db *ProjectAccounting) GetProjectTotalByPartner(ctx context.Context, projectID uuid.UUID, partnerNames []string, since, before time.Time) (usages map[string]accounting.ProjectUsage, err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
since = timeTruncateDown(since)
|
||||
|
||||
storageQuery := db.db.Rebind(`
|
||||
SELECT * FROM (
|
||||
SELECT
|
||||
COALESCE(t.bucket_name, rollups.bucket_name) AS bucket_name,
|
||||
COALESCE(t.interval_start, rollups.interval_start) AS interval_start,
|
||||
COALESCE(t.total_bytes, 0) AS total_bytes,
|
||||
COALESCE(t.inline, 0) AS inline,
|
||||
COALESCE(t.remote, 0) AS remote,
|
||||
COALESCE(t.total_segments_count, 0) AS total_segments_count,
|
||||
COALESCE(t.object_count, 0) AS object_count,
|
||||
m.user_agent,
|
||||
COALESCE(rollups.egress, 0) AS egress
|
||||
FROM
|
||||
bucket_storage_tallies AS t
|
||||
FULL OUTER JOIN (
|
||||
SELECT
|
||||
bucket_name,
|
||||
SUM(settled + inline) AS egress,
|
||||
MIN(interval_start) AS interval_start
|
||||
FROM
|
||||
bucket_bandwidth_rollups
|
||||
WHERE
|
||||
project_id = $1 AND
|
||||
interval_start >= $2 AND
|
||||
interval_start < $3 AND
|
||||
action = $4
|
||||
GROUP BY
|
||||
bucket_name
|
||||
) AS rollups ON
|
||||
t.bucket_name = rollups.bucket_name
|
||||
LEFT JOIN bucket_metainfos AS m ON
|
||||
m.project_id = $1 AND
|
||||
m.name = COALESCE(t.bucket_name, rollups.bucket_name)
|
||||
WHERE
|
||||
(t.project_id IS NULL OR t.project_id = $1) AND
|
||||
COALESCE(t.interval_start, rollups.interval_start) >= $2 AND
|
||||
COALESCE(t.interval_start, rollups.interval_start) < $3
|
||||
) AS q` + db.db.impl.AsOfSystemInterval(-10) + ` ORDER BY bucket_name, interval_start DESC`)
|
||||
|
||||
usages = make(map[string]accounting.ProjectUsage)
|
||||
|
||||
storageTalliesRows, err := db.db.QueryContext(ctx, storageQuery, projectID[:], since, before, pb.PieceAction_GET)
|
||||
bucketNames, err := db.getBucketsSinceAndBefore(ctx, projectID, since, before)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var prevTallyForBucket = make(map[string]*accounting.BucketStorageTally)
|
||||
var recentBucket string
|
||||
|
||||
for storageTalliesRows.Next() {
|
||||
tally := accounting.BucketStorageTally{}
|
||||
var userAgent []byte
|
||||
var inline, remote, egress int64
|
||||
err = storageTalliesRows.Scan(&tally.BucketName, &tally.IntervalStart, &tally.TotalBytes, &inline, &remote, &tally.TotalSegmentCount, &tally.ObjectCount, &userAgent, &egress)
|
||||
if err != nil {
|
||||
return nil, errs.Combine(err, storageTalliesRows.Close())
|
||||
storageQuery := db.db.Rebind(`
|
||||
SELECT
|
||||
bucket_storage_tallies.interval_start,
|
||||
bucket_storage_tallies.total_bytes,
|
||||
bucket_storage_tallies.inline,
|
||||
bucket_storage_tallies.remote,
|
||||
bucket_storage_tallies.total_segments_count,
|
||||
bucket_storage_tallies.object_count
|
||||
FROM
|
||||
bucket_storage_tallies
|
||||
WHERE
|
||||
bucket_storage_tallies.project_id = ? AND
|
||||
bucket_storage_tallies.bucket_name = ? AND
|
||||
bucket_storage_tallies.interval_start >= ? AND
|
||||
bucket_storage_tallies.interval_start < ?
|
||||
ORDER BY bucket_storage_tallies.interval_start DESC
|
||||
`)
|
||||
|
||||
totalEgressQuery := db.db.Rebind(`
|
||||
SELECT
|
||||
COALESCE(SUM(settled) + SUM(inline), 0)
|
||||
FROM
|
||||
bucket_bandwidth_rollups
|
||||
WHERE
|
||||
project_id = ? AND
|
||||
bucket_name = ? AND
|
||||
interval_start >= ? AND
|
||||
interval_start < ? AND
|
||||
action = ?;
|
||||
`)
|
||||
|
||||
usages = make(map[string]accounting.ProjectUsage)
|
||||
|
||||
for _, bucket := range bucketNames {
|
||||
userAgentRow, err := db.db.Get_BucketMetainfo_UserAgent_By_ProjectId_And_Name(ctx,
|
||||
dbx.BucketMetainfo_ProjectId(projectID[:]),
|
||||
dbx.BucketMetainfo_Name([]byte(bucket)))
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var partner string
|
||||
if userAgent != nil {
|
||||
entries, err := useragent.ParseEntries(userAgent)
|
||||
if userAgentRow != nil && userAgentRow.UserAgent != nil {
|
||||
entries, err := useragent.ParseEntries(userAgentRow.UserAgent)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -623,35 +611,40 @@ func (db *ProjectAccounting) GetProjectTotalByPartner(ctx context.Context, proje
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if _, ok := usages[partner]; !ok {
|
||||
usages[partner] = accounting.ProjectUsage{Since: since, Before: before}
|
||||
}
|
||||
usage := usages[partner]
|
||||
|
||||
storageTalliesRows, err := db.db.QueryContext(ctx, storageQuery, projectID[:], []byte(bucket), since, before)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var prevTally *accounting.BucketStorageTally
|
||||
for storageTalliesRows.Next() {
|
||||
tally := accounting.BucketStorageTally{}
|
||||
|
||||
var inline, remote int64
|
||||
err = storageTalliesRows.Scan(&tally.IntervalStart, &tally.TotalBytes, &inline, &remote, &tally.TotalSegmentCount, &tally.ObjectCount)
|
||||
if err != nil {
|
||||
return nil, errs.Combine(err, storageTalliesRows.Close())
|
||||
}
|
||||
if tally.TotalBytes == 0 {
|
||||
tally.TotalBytes = inline + remote
|
||||
}
|
||||
|
||||
if tally.BucketName != recentBucket {
|
||||
usage.Egress += egress
|
||||
recentBucket = tally.BucketName
|
||||
}
|
||||
|
||||
if _, ok := prevTallyForBucket[tally.BucketName]; !ok {
|
||||
prevTallyForBucket[tally.BucketName] = &tally
|
||||
usages[partner] = usage
|
||||
if prevTally == nil {
|
||||
prevTally = &tally
|
||||
continue
|
||||
}
|
||||
|
||||
hours := prevTallyForBucket[tally.BucketName].IntervalStart.Sub(tally.IntervalStart).Hours()
|
||||
hours := prevTally.IntervalStart.Sub(tally.IntervalStart).Hours()
|
||||
usage.Storage += memory.Size(tally.TotalBytes).Float64() * hours
|
||||
usage.SegmentCount += float64(tally.TotalSegmentCount) * hours
|
||||
usage.ObjectCount += float64(tally.ObjectCount) * hours
|
||||
|
||||
usages[partner] = usage
|
||||
|
||||
prevTallyForBucket[tally.BucketName] = &tally
|
||||
prevTally = &tally
|
||||
}
|
||||
|
||||
err = errs.Combine(storageTalliesRows.Err(), storageTalliesRows.Close())
|
||||
@ -659,6 +652,20 @@ func (db *ProjectAccounting) GetProjectTotalByPartner(ctx context.Context, proje
|
||||
return nil, err
|
||||
}
|
||||
|
||||
totalEgressRow := db.db.QueryRowContext(ctx, totalEgressQuery, projectID[:], []byte(bucket), since, before, pb.PieceAction_GET)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var egress int64
|
||||
if err = totalEgressRow.Scan(&egress); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
usage.Egress += egress
|
||||
|
||||
usages[partner] = usage
|
||||
}
|
||||
|
||||
return usages, nil
|
||||
}
|
||||
|
||||
|
||||
@ -207,31 +207,31 @@ func Test_GetProjectTotal(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
const epsilon = 1e-8
|
||||
require.InDelta(t, float64(tallies[0].Bytes()+tallies[1].Bytes()), usage.Storage, epsilon)
|
||||
require.InDelta(t, float64(tallies[0].TotalSegmentCount+tallies[1].TotalSegmentCount), usage.SegmentCount, epsilon)
|
||||
require.InDelta(t, float64(tallies[0].ObjectCount+tallies[1].ObjectCount), usage.ObjectCount, epsilon)
|
||||
require.Equal(t, expectedEgress, usage.Egress)
|
||||
require.Equal(t, tallies[0].IntervalStart, usage.Since)
|
||||
require.Equal(t, tallies[2].IntervalStart.Add(time.Minute), usage.Before)
|
||||
require.InDelta(t, usage.Storage, float64(tallies[0].Bytes()+tallies[1].Bytes()), epsilon)
|
||||
require.InDelta(t, usage.SegmentCount, float64(tallies[0].TotalSegmentCount+tallies[1].TotalSegmentCount), epsilon)
|
||||
require.InDelta(t, usage.ObjectCount, float64(tallies[0].ObjectCount+tallies[1].ObjectCount), epsilon)
|
||||
require.Equal(t, usage.Egress, expectedEgress)
|
||||
require.Equal(t, usage.Since, tallies[0].IntervalStart)
|
||||
require.Equal(t, usage.Before, tallies[2].IntervalStart.Add(time.Minute))
|
||||
|
||||
// Ensure that GetProjectTotal treats the 'before' arg as exclusive
|
||||
usage, err = db.ProjectAccounting().GetProjectTotal(ctx, projectID, tallies[0].IntervalStart, tallies[2].IntervalStart)
|
||||
require.NoError(t, err)
|
||||
require.InDelta(t, float64(tallies[0].Bytes()), usage.Storage, epsilon)
|
||||
require.InDelta(t, float64(tallies[0].TotalSegmentCount), usage.SegmentCount, epsilon)
|
||||
require.InDelta(t, float64(tallies[0].ObjectCount), usage.ObjectCount, epsilon)
|
||||
require.Equal(t, expectedEgress, usage.Egress)
|
||||
require.Equal(t, tallies[0].IntervalStart, usage.Since)
|
||||
require.Equal(t, tallies[2].IntervalStart, usage.Before)
|
||||
require.InDelta(t, usage.Storage, float64(tallies[0].Bytes()), epsilon)
|
||||
require.InDelta(t, usage.SegmentCount, float64(tallies[0].TotalSegmentCount), epsilon)
|
||||
require.InDelta(t, usage.ObjectCount, float64(tallies[0].ObjectCount), epsilon)
|
||||
require.Equal(t, usage.Egress, expectedEgress)
|
||||
require.Equal(t, usage.Since, tallies[0].IntervalStart)
|
||||
require.Equal(t, usage.Before, tallies[2].IntervalStart)
|
||||
|
||||
usage, err = db.ProjectAccounting().GetProjectTotal(ctx, projectID, rollups[0].IntervalStart, rollups[1].IntervalStart)
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, usage.Storage)
|
||||
require.Zero(t, usage.SegmentCount)
|
||||
require.Zero(t, usage.ObjectCount)
|
||||
require.Equal(t, rollups[0].Inline+rollups[0].Settled, usage.Egress)
|
||||
require.Equal(t, rollups[0].IntervalStart, usage.Since)
|
||||
require.Equal(t, rollups[1].IntervalStart, usage.Before)
|
||||
require.Equal(t, usage.Egress, rollups[0].Inline+rollups[0].Settled)
|
||||
require.Equal(t, usage.Since, rollups[0].IntervalStart)
|
||||
require.Equal(t, usage.Before, rollups[1].IntervalStart)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user