JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/metabase/metaloop: move loop under metabase

Browse Source 
Currently the loop handling is heavily related to the metabase rather
than metainfo.

metainfo over time has become related to the "public API" for accessing
the metabase data.

Currently updates monkit.lock, because monkit monitoring does not handle
ScopeNamed correctly. Needs a followup change to monitoring check.

Change-Id: Ie50519991d718dfb872ec9a0176a82e732c97584
This commit is contained in:
Egon Elbre 2021-04-22 12:07:18 +03:00
parent ebc981291a
commit 4c9ed64f75
20 changed files with 28 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/metainfo-loop-benchmark/bench.go
View File

@ -18,8 +18,8 @@ import (
"storj.io/common/errs2"
"storj.io/common/memory"
"storj.io/storj/satellite/metabase/metaloop"
"storj.io/storj/satellite/metainfo"
"storj.io/storj/satellite/metainfo/metaloop"
)
var mon = monkit.Package()

8
monkit.lock
View File

@ -61,11 +61,11 @@ storj.io/storj/satellite/gracefulexit."graceful_exit_success" Meter
storj.io/storj/satellite/gracefulexit."graceful_exit_successful_pieces_transfer_ratio" IntVal
storj.io/storj/satellite/gracefulexit."graceful_exit_transfer_piece_fail" Meter
storj.io/storj/satellite/gracefulexit."graceful_exit_transfer_piece_success" Meter
storj.io/storj/satellite/metabase/metaloop."objectsIterated" IntVal
storj.io/storj/satellite/metabase/metaloop."objectsProcessed" IntVal
storj.io/storj/satellite/metabase/metaloop."segmentsProcessed" IntVal
storj.io/storj/satellite/metabase/metaloop.*Service.RunOnce Task
storj.io/storj/satellite/metainfo."metainfo_rate_limit_exceeded" Event
storj.io/storj/satellite/metainfo/metaloop."objectsIterated" IntVal
storj.io/storj/satellite/metainfo/metaloop."objectsProcessed" IntVal
storj.io/storj/satellite/metainfo/metaloop."segmentsProcessed" IntVal
storj.io/storj/satellite/metainfo/metaloop.*Service.RunOnce Task
storj.io/storj/satellite/metainfo/piecedeletion."delete_batch_size" IntVal
storj.io/storj/satellite/metainfo/piecedeletion."deletion_pieces_unhandled_count" IntVal
storj.io/storj/satellite/orders."download_failed_not_enough_pieces_uplink" Meter

2
private/testplanet/satellite.go
View File

@ -48,9 +48,9 @@ import (
"storj.io/storj/satellite/gracefulexit"
"storj.io/storj/satellite/inspector"
"storj.io/storj/satellite/mailservice"
"storj.io/storj/satellite/metabase/metaloop"
"storj.io/storj/satellite/metainfo"
"storj.io/storj/satellite/metainfo/expireddeletion"
"storj.io/storj/satellite/metainfo/metaloop"
"storj.io/storj/satellite/metainfo/piecedeletion"
"storj.io/storj/satellite/metrics"
"storj.io/storj/satellite/nodestats"

2
satellite/accounting/tally/tally.go
View File

@ -16,7 +16,7 @@ import (
"storj.io/common/uuid"
"storj.io/storj/satellite/accounting"
"storj.io/storj/satellite/metabase"
"storj.io/storj/satellite/metainfo/metaloop"
"storj.io/storj/satellite/metabase/metaloop"
)
// Error is a standard error class for this package.

2
satellite/audit/chore.go
View File

@ -11,7 +11,7 @@ import (
"go.uber.org/zap"
"storj.io/common/sync2"
"storj.io/storj/satellite/metainfo/metaloop"
"storj.io/storj/satellite/metabase/metaloop"
)
// Chore populates reservoirs and the audit queue.

2
satellite/audit/collector.go
View File

@ -8,7 +8,7 @@ import (
"math/rand"
"storj.io/common/storj"
"storj.io/storj/satellite/metainfo/metaloop"
"storj.io/storj/satellite/metabase/metaloop"
)
var _ metaloop.Observer = (*Collector)(nil)

2
satellite/audit/reservoir.go
View File

@ -9,7 +9,7 @@ import (
"storj.io/common/uuid"
"storj.io/storj/satellite/metabase"
"storj.io/storj/satellite/metainfo/metaloop"
"storj.io/storj/satellite/metabase/metaloop"
)
const maxReservoirSize = 3

2
satellite/core.go
View File

@ -33,9 +33,9 @@ import (
"storj.io/storj/satellite/contact"
"storj.io/storj/satellite/gc"
"storj.io/storj/satellite/gracefulexit"
"storj.io/storj/satellite/metabase/metaloop"
"storj.io/storj/satellite/metainfo"
"storj.io/storj/satellite/metainfo/expireddeletion"
"storj.io/storj/satellite/metainfo/metaloop"
"storj.io/storj/satellite/metrics"
"storj.io/storj/satellite/orders"
"storj.io/storj/satellite/overlay"

2
satellite/gc.go
View File

@ -23,8 +23,8 @@ import (
"storj.io/storj/private/lifecycle"
version_checker "storj.io/storj/private/version/checker"
"storj.io/storj/satellite/gc"
"storj.io/storj/satellite/metabase/metaloop"
"storj.io/storj/satellite/metainfo"
"storj.io/storj/satellite/metainfo/metaloop"
"storj.io/storj/satellite/overlay"
)

2
satellite/gc/piecetracker.go
View File

@ -13,7 +13,7 @@ import (
"storj.io/common/bloomfilter"
"storj.io/common/memory"
"storj.io/common/storj"
"storj.io/storj/satellite/metainfo/metaloop"
"storj.io/storj/satellite/metabase/metaloop"
)
var _ metaloop.Observer = (*PieceTracker)(nil)

2
satellite/gc/service.go
View File

@ -16,7 +16,7 @@ import (
"storj.io/common/rpc"
"storj.io/common/storj"
"storj.io/common/sync2"
"storj.io/storj/satellite/metainfo/metaloop"
"storj.io/storj/satellite/metabase/metaloop"
"storj.io/storj/satellite/overlay"
"storj.io/uplink/private/piecestore"
)

2
satellite/gracefulexit/chore.go
View File

@ -13,7 +13,7 @@ import (
"storj.io/common/storj"
"storj.io/common/sync2"
"storj.io/storj/satellite/metainfo/metaloop"
"storj.io/storj/satellite/metabase/metaloop"
"storj.io/storj/satellite/overlay"
)

2
satellite/gracefulexit/pathcollector.go
View File

@ -11,7 +11,7 @@ import (
"go.uber.org/zap"
"storj.io/common/storj"
"storj.io/storj/satellite/metainfo/metaloop"
"storj.io/storj/satellite/metabase/metaloop"
"storj.io/uplink/private/eestream"
)

11
satellite/metainfo/metaloop/service.go → satellite/metabase/metaloop/service.go
View File

@ -239,11 +239,14 @@ func (loop *Service) Close() (err error) {
return nil
}
// monMetainfo is to preserve the monitoring names.
var monMetainfo = monkit.ScopeNamed("storj.io/storj/satellite/metainfo/metaloop")
// RunOnce goes through metainfo one time and sends information to observers.
//
// It is not safe to call this concurrently with Run.
func (loop *Service) RunOnce(ctx context.Context) (err error) {
defer mon.Task()(&ctx)(&err) //mon:locked
defer monMetainfo.Task()(&ctx)(&err) //mon:locked
var observers []*observerContext
@ -355,7 +358,7 @@ func iterateObjects(ctx context.Context, metabaseDB MetabaseDB, observers []*obs
}
objectsProcessed++
mon.IntVal("objectsProcessed").Observe(objectsProcessed) //mon:locked
monMetainfo.IntVal("objectsProcessed").Observe(objectsProcessed) //mon:locked
for {
// if context has been canceled exit. Otherwise, continue
@ -383,7 +386,7 @@ func iterateObjects(ctx context.Context, metabaseDB MetabaseDB, observers []*obs
}
segmentsProcessed++
mon.IntVal("segmentsProcessed").Observe(segmentsProcessed) //mon:locked
monMetainfo.IntVal("segmentsProcessed").Observe(segmentsProcessed) //mon:locked
}
@ -420,7 +423,7 @@ func iterateObjects(ctx context.Context, metabaseDB MetabaseDB, observers []*obs
}
timer.Stop()
mon.IntVal("objectsIterated").Observe(objectsIterated) //mon:locked
monMetainfo.IntVal("objectsIterated").Observe(objectsIterated) //mon:locked
objectsProcessed++
objectsMap[entry.StreamID] = entry

2
satellite/metainfo/metaloop/service_test.go → satellite/metabase/metaloop/service_test.go
View File

@ -24,7 +24,7 @@ import (
"storj.io/storj/private/testplanet"
"storj.io/storj/satellite"
"storj.io/storj/satellite/metabase"
"storj.io/storj/satellite/metainfo/metaloop"
"storj.io/storj/satellite/metabase/metaloop"
"storj.io/uplink/private/multipart"
)

0
satellite/metainfo/metaloop/stats.go → satellite/metabase/metaloop/stats.go
View File

2
satellite/metainfo/config.go
View File

@ -18,7 +18,7 @@ import (
"storj.io/common/uuid"
"storj.io/storj/private/dbutil"
"storj.io/storj/satellite/metabase"
"storj.io/storj/satellite/metainfo/metaloop"
"storj.io/storj/satellite/metabase/metaloop"
"storj.io/storj/satellite/metainfo/piecedeletion"
)

2
satellite/metrics/chore.go
View File

@ -12,7 +12,7 @@ import (
"go.uber.org/zap"
"storj.io/common/sync2"
"storj.io/storj/satellite/metainfo/metaloop"
"storj.io/storj/satellite/metabase/metaloop"
)
var (

2
satellite/metrics/counter.go
View File

@ -7,7 +7,7 @@ import (
"context"
"storj.io/common/uuid"
"storj.io/storj/satellite/metainfo/metaloop"
"storj.io/storj/satellite/metabase/metaloop"
)
// Counter implements the metainfo loop observer interface for data science metrics collection.

2
satellite/repair/checker/checker.go
View File

@ -18,8 +18,8 @@ import (
"storj.io/common/sync2"
"storj.io/storj/satellite/internalpb"
"storj.io/storj/satellite/metabase"
"storj.io/storj/satellite/metabase/metaloop"
"storj.io/storj/satellite/metainfo"
"storj.io/storj/satellite/metainfo/metaloop"
"storj.io/storj/satellite/overlay"
"storj.io/storj/satellite/repair"
"storj.io/storj/satellite/repair/irreparable"