JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Get rid of statdb grpc stuff (#761)

Browse Source 
* remove sdbclient

* do not register statdb grpc server/remove apikey from statdb

* start audit service after statdb in captplanet
This commit is contained in:
Maximillian von Briesen 2018-12-04 13:47:58 -05:00 committed by GitHub
parent b7c45c7c16
commit 2ab15196d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 133 additions and 663 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/captplanet/run.go
View File

@ -105,8 +105,8 @@ func cmdRun(cmd *cobra.Command, args []string) (err error) {
errch <- runCfg.Satellite.Identity.Run(ctx,
grpcauth.NewAPIKeyInterceptor(),
runCfg.Satellite.Kademlia,
runCfg.Satellite.Audit,
runCfg.Satellite.StatDB,
runCfg.Satellite.Audit,
runCfg.Satellite.Overlay,
runCfg.Satellite.PointerDB,
runCfg.Satellite.Checker,

8
cmd/inspector/main.go
View File

@ -253,8 +253,8 @@ func GetStats(cmd *cobra.Command, args []string) (err error) {
}
fmt.Printf("Stats for ID %s:\n", nodeID)
fmt.Printf("AuditSuccessRatio: %f, UptimeRatio: %f, AuditCount: %d\n",
res.AuditRatio, res.UptimeRatio, res.AuditCount)
fmt.Printf("AuditSuccessRatio: %f, AuditCount: %d, UptimeRatio: %f, UptimeCount: %d,\n",
res.AuditRatio, res.AuditCount, res.UptimeRatio, res.UptimeCount)
return nil
}
@ -289,8 +289,8 @@ func GetCSVStats(cmd *cobra.Command, args []string) (err error) {
}
fmt.Printf("Stats for ID %s:\n", nodeID)
fmt.Printf("AuditSuccessRatio: %f, UptimeRatio: %f, AuditCount: %d\n",
res.AuditRatio, res.UptimeRatio, res.AuditCount)
fmt.Printf("AuditSuccessRatio: %f, AuditCount: %d, UptimeRatio: %f, UptimeCount: %d,\n",
res.AuditRatio, res.AuditCount, res.UptimeRatio, res.UptimeCount)
}
return nil
}

16
examples/statdb-client/README.md
View File

@ -1,16 +0,0 @@
# gRPC Client
This is an example gRPC client which makes requests for updating and storing storagenode stats in a relational database.
The gRPC server at `storj.io/storj/cmd/statdb/main.go` needs to be running for this to work.
To run the client:
```
go run examples/statdb-client/main.go
```
You can change the port number with a flag if necessary: `-port=<port-number>`
If changes are made to `storj.io/storj/pkg/statdb/proto/statdb.proto, the protobuf file will need to be regenerated by running `go generate` inside `pkg/statdb/proto`
If changes are made to `storj.io/storj/pkg/statdb/dbx/statdb.dbx, the dbx files will need to be regenerated by running `go generate` inside `pkg/statdb/dbx`

148
examples/statdb-client/main.go
View File

@ -1,148 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package main
import (
"context"
"flag"
"fmt"
"os"
"go.uber.org/zap"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"storj.io/storj/pkg/provider"
proto "storj.io/storj/pkg/statdb/proto"
"storj.io/storj/pkg/statdb/sdbclient"
)
var (
port string
apiKey = ""
ctx = context.Background()
)
func initializeFlags() {
flag.StringVar(&port, "port", ":7777", "port")
flag.Parse()
}
func printNodeStats(ns proto.NodeStats, logger zap.Logger) {
nodeID := ns.NodeId
latency90 := ns.Latency_90
auditSuccess := ns.AuditSuccessRatio
uptime := ns.UptimeRatio
logStr := fmt.Sprintf("NodeID: %s, Latency (90th percentile): %d, Audit Success Ratio: %g, Uptime Ratio: %g", nodeID, latency90, auditSuccess, uptime)
logger.Info(logStr)
}
func main() {
initializeFlags()
logger, _ := zap.NewDevelopment()
defer printError(logger.Sync)
idents := make([]*provider.FullIdentity, 3)
for i := range idents {
var err error
idents[i], err = provider.NewFullIdentity(ctx, 12, 4)
if err != nil {
logger.Error("Failed to create certificate authority: ", zap.Error(err))
os.Exit(1)
}
}
client, err := sdbclient.NewClient(idents[0], port, apiKey)
if err != nil {
logger.Error("Failed to create sdbclient: ", zap.Error(err))
}
logger.Debug(fmt.Sprintf("client dialed port %s", port))
// Test farmers
farmer1 := proto.Node{
Id: idents[1].ID,
UpdateAuditSuccess: false,
UpdateUptime: false,
}
farmer2 := proto.Node{
Id: idents[2].ID,
UpdateAuditSuccess: false,
UpdateUptime: false,
}
// Example Creates
err = client.Create(ctx, farmer1.Id)
if err != nil || status.Code(err) == codes.Internal {
logger.Error("failed to create", zap.Error(err))
os.Exit(1)
}
logger.Info("Farmer 1 created successfully")
err = client.Create(ctx, farmer2.Id)
if err != nil || status.Code(err) == codes.Internal {
logger.Error("failed to create", zap.Error(err))
os.Exit(1)
}
logger.Info("Farmer 2 created successfully")
// Example Updates
farmer1.AuditSuccess = true
farmer1.IsUp = true
farmer1.UpdateAuditSuccess = true
farmer1.UpdateUptime = true
nodeStats, err := client.Update(ctx, farmer1.Id, farmer1.AuditSuccess, farmer1.IsUp, nil)
if err != nil || status.Code(err) == codes.Internal {
logger.Error("failed to update", zap.Error(err))
os.Exit(1)
}
logger.Info("Farmer 1 after Update")
printNodeStats(*nodeStats, *logger)
// Example UpdateBatch
farmer1.AuditSuccess = false
farmer1.IsUp = false
farmer2.AuditSuccess = true
farmer2.IsUp = true
farmer2.UpdateAuditSuccess = true
farmer2.UpdateUptime = true
nodeList := []*proto.Node{&farmer1, &farmer2}
statsList, _, err := client.UpdateBatch(ctx, nodeList)
if err != nil || status.Code(err) == codes.Internal {
logger.Error("failed to update batch", zap.Error(err))
os.Exit(1)
}
logger.Info("Farmer stats after UpdateBatch")
for _, statsEl := range statsList {
printNodeStats(*statsEl, *logger)
}
// Example Get
nodeStats, err = client.Get(ctx, farmer1.Id)
if err != nil || status.Code(err) == codes.Internal {
logger.Error("failed to update", zap.Error(err))
os.Exit(1)
}
logger.Info("Farmer 1 after Get 1")
printNodeStats(*nodeStats, *logger)
nodeStats, err = client.Get(ctx, farmer2.Id)
if err != nil || status.Code(err) == codes.Internal {
logger.Error("failed to update", zap.Error(err))
os.Exit(1)
}
logger.Info("Farmer 2 after Get 2")
printNodeStats(*nodeStats, *logger)
}
func printError(fn func() error) {
err := fn()
if err != nil {
fmt.Println(err)
}
}

4
internal/testplanet/node.go
View File

@ -35,7 +35,7 @@ type Node struct {
Listener net.Listener
Provider *provider.Provider
Kademlia *kademlia.Kademlia
StatDB *statdb.Server
StatDB *statdb.StatDB
Overlay *overlay.Cache
Dependencies []io.Closer
@ -159,7 +159,7 @@ func (node *Node) initOverlay(planet *Planet) error {
// initStatDB creates statdb for a given planet
func (node *Node) initStatDB() error {
dbPath := fmt.Sprintf("file:memdb%d?mode=memory&cache=shared", rand.Int63())
sdb, err := statdb.NewServer("sqlite3", dbPath, "", zap.NewNop())
sdb, err := statdb.NewStatDB("sqlite3", dbPath, zap.NewNop())
if err != nil {
return err
}

41
pkg/audit/reporter.go
View File

@ -6,45 +6,40 @@ package audit
import (
"context"
"storj.io/storj/pkg/provider"
proto "storj.io/storj/pkg/statdb/proto"
"storj.io/storj/pkg/statdb/sdbclient"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/statdb"
statsproto "storj.io/storj/pkg/statdb/proto"
"storj.io/storj/pkg/storj"
)
type reporter interface {
RecordAudits(ctx context.Context, failedNodes []*proto.Node) (err error)
RecordAudits(ctx context.Context, failedNodes []*pb.Node) (err error)
}
// Reporter records audit reports in statdb and implements the reporter interface
type Reporter struct {
statdb sdbclient.Client
statdb *statdb.StatDB
maxRetries int
}
// NewReporter instantiates a reporter
func NewReporter(ctx context.Context, statDBPort string, maxRetries int, apiKey string) (reporter *Reporter, err error) {
identity, err := provider.NewFullIdentity(ctx, 12, 4)
if err != nil {
return nil, err
}
sdb := statdb.LoadFromContext(ctx)
client, err := sdbclient.NewClient(identity, statDBPort, apiKey)
if err != nil {
return nil, err
}
return &Reporter{statdb: client, maxRetries: maxRetries}, nil
return &Reporter{statdb: sdb, maxRetries: maxRetries}, nil
}
// RecordAudits saves failed audit details to statdb
func (reporter *Reporter) RecordAudits(ctx context.Context, nodes []*proto.Node) (err error) {
func (reporter *Reporter) RecordAudits(ctx context.Context, nodes []*pb.Node) (err error) {
retries := 0
for len(nodes) > 0 && retries < reporter.maxRetries {
_, failedNodes, err := reporter.statdb.UpdateBatch(ctx, nodes)
res, err := reporter.statdb.UpdateBatch(ctx, &statsproto.UpdateBatchRequest{
NodeList: nodes,
})
if err != nil {
return err
}
nodes = failedNodes
nodes = res.GetFailedNodes()
retries++
}
if retries >= reporter.maxRetries && len(nodes) > 0 {
@ -53,9 +48,9 @@ func (reporter *Reporter) RecordAudits(ctx context.Context, nodes []*proto.Node)
return nil
}
func setAuditFailStatus(ctx context.Context, failedNodes storj.NodeIDList) (failStatusNodes []*proto.Node) {
func setAuditFailStatus(ctx context.Context, failedNodes storj.NodeIDList) (failStatusNodes []*pb.Node) {
for i := range failedNodes {
setNode := &proto.Node{
setNode := &pb.Node{
Id: failedNodes[i],
AuditSuccess: false,
IsUp: true,
@ -68,9 +63,9 @@ func setAuditFailStatus(ctx context.Context, failedNodes storj.NodeIDList) (fail
}
// TODO: offline nodes should maybe be marked as failing the audit in the future
func setOfflineStatus(ctx context.Context, offlineNodeIDs storj.NodeIDList) (offlineStatusNodes []*proto.Node) {
func setOfflineStatus(ctx context.Context, offlineNodeIDs storj.NodeIDList) (offlineStatusNodes []*pb.Node) {
for i := range offlineNodeIDs {
setNode := &proto.Node{
setNode := &pb.Node{
Id: offlineNodeIDs[i],
IsUp: false,
UpdateUptime: true,
@ -80,9 +75,9 @@ func setOfflineStatus(ctx context.Context, offlineNodeIDs storj.NodeIDList) (off
return offlineStatusNodes
}
func setSuccessStatus(ctx context.Context, offlineNodeIDs storj.NodeIDList) (successStatusNodes []*proto.Node) {
func setSuccessStatus(ctx context.Context, offlineNodeIDs storj.NodeIDList) (successStatusNodes []*pb.Node) {
for i := range offlineNodeIDs {
setNode := &proto.Node{
setNode := &pb.Node{
Id: offlineNodeIDs[i],
AuditSuccess: true,
IsUp: true,

14
pkg/inspector/server.go
View File

@ -8,7 +8,7 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap"
"gopkg.in/spacemonkeygo/monkit.v2"
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/dht"
"storj.io/storj/pkg/node"
@ -29,7 +29,7 @@ var (
type Server struct {
dht dht.DHT
cache *overlay.Cache
statdb *statdb.Server
statdb *statdb.StatDB
logger *zap.Logger
metrics *monkit.Registry
identity *provider.FullIdentity
@ -141,9 +141,10 @@ func (srv *Server) GetStats(ctx context.Context, req *pb.GetStatsRequest) (*pb.G
}
return &pb.GetStatsResponse{
AuditRatio: res.Stats.AuditSuccessRatio,
UptimeRatio: res.Stats.UptimeRatio,
AuditCount: res.Stats.AuditCount,
AuditRatio: res.Stats.AuditSuccessRatio,
UptimeCount: res.Stats.UptimeCount,
UptimeRatio: res.Stats.UptimeRatio,
}, nil
}
@ -163,6 +164,9 @@ func (srv *Server) CreateStats(ctx context.Context, req *pb.CreateStatsRequest)
Stats: stats,
}
_, err := srv.statdb.Create(ctx, createReq)
if err != nil {
return nil, err
}
return &pb.CreateStatsResponse{}, err
return &pb.CreateStatsResponse{}, nil
}

4
pkg/overlay/cache.go
View File

@ -34,11 +34,11 @@ var OverlayError = errs.Class("Overlay Error")
type Cache struct {
DB storage.KeyValueStore
DHT dht.DHT
StatDB *statdb.Server
StatDB *statdb.StatDB
}
// NewOverlayCache returns a new Cache
func NewOverlayCache(db storage.KeyValueStore, dht dht.DHT, sdb *statdb.Server) *Cache {
func NewOverlayCache(db storage.KeyValueStore, dht dht.DHT, sdb *statdb.StatDB) *Cache {
return &Cache{DB: db, DHT: dht, StatDB: sdb}
}

2
pkg/overlay/config_test.go
View File

@ -19,7 +19,7 @@ func TestRun(t *testing.T) {
var kadKey kademlia.CtxKey
ctx := context.WithValue(bctx, kadKey, kad)
sdb := &statdb.Server{}
sdb := &statdb.StatDB{}
var statKey statdb.CtxKey
ctx = context.WithValue(ctx, statKey, sdb)

126
pkg/pb/inspector.pb.go
View File

@ -34,7 +34,7 @@ func (m *GetStatsRequest) Reset() { *m = GetStatsRequest{} }
func (m *GetStatsRequest) String() string { return proto.CompactTextString(m) }
func (*GetStatsRequest) ProtoMessage() {}
func (*GetStatsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_inspector_9ff12d18f09cf85c, []int{0}
return fileDescriptor_inspector_44054ed20d312407, []int{0}
}
func (m *GetStatsRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_GetStatsRequest.Unmarshal(m, b)
@ -56,8 +56,9 @@ var xxx_messageInfo_GetStatsRequest proto.InternalMessageInfo
type GetStatsResponse struct {
AuditCount int64 `protobuf:"varint,1,opt,name=audit_count,json=auditCount,proto3" json:"audit_count,omitempty"`
UptimeRatio float64 `protobuf:"fixed64,2,opt,name=uptime_ratio,json=uptimeRatio,proto3" json:"uptime_ratio,omitempty"`
AuditRatio float64 `protobuf:"fixed64,3,opt,name=audit_ratio,json=auditRatio,proto3" json:"audit_ratio,omitempty"`
AuditRatio float64 `protobuf:"fixed64,2,opt,name=audit_ratio,json=auditRatio,proto3" json:"audit_ratio,omitempty"`
UptimeCount int64 `protobuf:"varint,3,opt,name=uptime_count,json=uptimeCount,proto3" json:"uptime_count,omitempty"`
UptimeRatio float64 `protobuf:"fixed64,4,opt,name=uptime_ratio,json=uptimeRatio,proto3" json:"uptime_ratio,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -67,7 +68,7 @@ func (m *GetStatsResponse) Reset() { *m = GetStatsResponse{} }
func (m *GetStatsResponse) String() string { return proto.CompactTextString(m) }
func (*GetStatsResponse) ProtoMessage() {}
func (*GetStatsResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_inspector_9ff12d18f09cf85c, []int{1}
return fileDescriptor_inspector_44054ed20d312407, []int{1}
}
func (m *GetStatsResponse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_GetStatsResponse.Unmarshal(m, b)
@ -94,16 +95,23 @@ func (m *GetStatsResponse) GetAuditCount() int64 {
return 0
}
func (m *GetStatsResponse) GetUptimeRatio() float64 {
func (m *GetStatsResponse) GetAuditRatio() float64 {
if m != nil {
return m.UptimeRatio
return m.AuditRatio
}
return 0
}
func (m *GetStatsResponse) GetAuditRatio() float64 {
func (m *GetStatsResponse) GetUptimeCount() int64 {
if m != nil {
return m.AuditRatio
return m.UptimeCount
}
return 0
}
func (m *GetStatsResponse) GetUptimeRatio() float64 {
if m != nil {
return m.UptimeRatio
}
return 0
}
@ -124,7 +132,7 @@ func (m *CreateStatsRequest) Reset() { *m = CreateStatsRequest{} }
func (m *CreateStatsRequest) String() string { return proto.CompactTextString(m) }
func (*CreateStatsRequest) ProtoMessage() {}
func (*CreateStatsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_inspector_9ff12d18f09cf85c, []int{2}
return fileDescriptor_inspector_44054ed20d312407, []int{2}
}
func (m *CreateStatsRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_CreateStatsRequest.Unmarshal(m, b)
@ -182,7 +190,7 @@ func (m *CreateStatsResponse) Reset() { *m = CreateStatsResponse{} }
func (m *CreateStatsResponse) String() string { return proto.CompactTextString(m) }
func (*CreateStatsResponse) ProtoMessage() {}
func (*CreateStatsResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_inspector_9ff12d18f09cf85c, []int{3}
return fileDescriptor_inspector_44054ed20d312407, []int{3}
}
func (m *CreateStatsResponse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_CreateStatsResponse.Unmarshal(m, b)
@ -215,7 +223,7 @@ func (m *CountNodesResponse) Reset() { *m = CountNodesResponse{} }
func (m *CountNodesResponse) String() string { return proto.CompactTextString(m) }
func (*CountNodesResponse) ProtoMessage() {}
func (*CountNodesResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_inspector_9ff12d18f09cf85c, []int{4}
return fileDescriptor_inspector_44054ed20d312407, []int{4}
}
func (m *CountNodesResponse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_CountNodesResponse.Unmarshal(m, b)
@ -259,7 +267,7 @@ func (m *CountNodesRequest) Reset() { *m = CountNodesRequest{} }
func (m *CountNodesRequest) String() string { return proto.CompactTextString(m) }
func (*CountNodesRequest) ProtoMessage() {}
func (*CountNodesRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_inspector_9ff12d18f09cf85c, []int{5}
return fileDescriptor_inspector_44054ed20d312407, []int{5}
}
func (m *CountNodesRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_CountNodesRequest.Unmarshal(m, b)
@ -290,7 +298,7 @@ func (m *GetBucketsRequest) Reset() { *m = GetBucketsRequest{} }
func (m *GetBucketsRequest) String() string { return proto.CompactTextString(m) }
func (*GetBucketsRequest) ProtoMessage() {}
func (*GetBucketsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_inspector_9ff12d18f09cf85c, []int{6}
return fileDescriptor_inspector_44054ed20d312407, []int{6}
}
func (m *GetBucketsRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_GetBucketsRequest.Unmarshal(m, b)
@ -322,7 +330,7 @@ func (m *GetBucketsResponse) Reset() { *m = GetBucketsResponse{} }
func (m *GetBucketsResponse) String() string { return proto.CompactTextString(m) }
func (*GetBucketsResponse) ProtoMessage() {}
func (*GetBucketsResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_inspector_9ff12d18f09cf85c, []int{7}
return fileDescriptor_inspector_44054ed20d312407, []int{7}
}
func (m *GetBucketsResponse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_GetBucketsResponse.Unmarshal(m, b)
@ -361,7 +369,7 @@ func (m *GetBucketRequest) Reset() { *m = GetBucketRequest{} }
func (m *GetBucketRequest) String() string { return proto.CompactTextString(m) }
func (*GetBucketRequest) ProtoMessage() {}
func (*GetBucketRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_inspector_9ff12d18f09cf85c, []int{8}
return fileDescriptor_inspector_44054ed20d312407, []int{8}
}
func (m *GetBucketRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_GetBucketRequest.Unmarshal(m, b)
@ -393,7 +401,7 @@ func (m *GetBucketResponse) Reset() { *m = GetBucketResponse{} }
func (m *GetBucketResponse) String() string { return proto.CompactTextString(m) }
func (*GetBucketResponse) ProtoMessage() {}
func (*GetBucketResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_inspector_9ff12d18f09cf85c, []int{9}
return fileDescriptor_inspector_44054ed20d312407, []int{9}
}
func (m *GetBucketResponse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_GetBucketResponse.Unmarshal(m, b)
@ -431,7 +439,7 @@ func (m *Bucket) Reset() { *m = Bucket{} }
func (m *Bucket) String() string { return proto.CompactTextString(m) }
func (*Bucket) ProtoMessage() {}
func (*Bucket) Descriptor() ([]byte, []int) {
return fileDescriptor_inspector_9ff12d18f09cf85c, []int{10}
return fileDescriptor_inspector_44054ed20d312407, []int{10}
}
func (m *Bucket) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_Bucket.Unmarshal(m, b)
@ -469,7 +477,7 @@ func (m *BucketList) Reset() { *m = BucketList{} }
func (m *BucketList) String() string { return proto.CompactTextString(m) }
func (*BucketList) ProtoMessage() {}
func (*BucketList) Descriptor() ([]byte, []int) {
return fileDescriptor_inspector_9ff12d18f09cf85c, []int{11}
return fileDescriptor_inspector_44054ed20d312407, []int{11}
}
func (m *BucketList) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_BucketList.Unmarshal(m, b)
@ -509,7 +517,7 @@ func (m *PingNodeRequest) Reset() { *m = PingNodeRequest{} }
func (m *PingNodeRequest) String() string { return proto.CompactTextString(m) }
func (*PingNodeRequest) ProtoMessage() {}
func (*PingNodeRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_inspector_9ff12d18f09cf85c, []int{12}
return fileDescriptor_inspector_44054ed20d312407, []int{12}
}
func (m *PingNodeRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_PingNodeRequest.Unmarshal(m, b)
@ -547,7 +555,7 @@ func (m *PingNodeResponse) Reset() { *m = PingNodeResponse{} }
func (m *PingNodeResponse) String() string { return proto.CompactTextString(m) }
func (*PingNodeResponse) ProtoMessage() {}
func (*PingNodeResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_inspector_9ff12d18f09cf85c, []int{13}
return fileDescriptor_inspector_44054ed20d312407, []int{13}
}
func (m *PingNodeResponse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_PingNodeResponse.Unmarshal(m, b)
@ -844,44 +852,44 @@ var _Inspector_serviceDesc = grpc.ServiceDesc{
Metadata: "inspector.proto",
}
func init() { proto.RegisterFile("inspector.proto", fileDescriptor_inspector_9ff12d18f09cf85c) }
func init() { proto.RegisterFile("inspector.proto", fileDescriptor_inspector_44054ed20d312407) }
var fileDescriptor_inspector_9ff12d18f09cf85c = []byte{
// 564 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x94, 0xcd, 0x6e, 0x1a, 0x3f,
0x14, 0xc5, 0xff, 0x33, 0x13, 0x08, 0x5c, 0x50, 0x48, 0x4c, 0xfe, 0x12, 0x1a, 0x68, 0xa0, 0xde,
0x14, 0x75, 0x81, 0x2a, 0xba, 0xeb, 0x12, 0xaa, 0xa6, 0xb4, 0x51, 0x55, 0x4d, 0xd4, 0x4d, 0x37,
0x68, 0x82, 0x2d, 0x64, 0x41, 0xf0, 0x74, 0xec, 0x69, 0xd5, 0xc7, 0xeb, 0xae, 0xcf, 0xd0, 0x45,
0x36, 0x7d, 0x91, 0xca, 0x1f, 0xf3, 0x05, 0x83, 0x1a, 0x75, 0x87, 0xcf, 0xf9, 0x71, 0xec, 0x7b,
0xaf, 0xc7, 0xd0, 0x61, 0x3b, 0x11, 0xd1, 0x95, 0xe4, 0xf1, 0x24, 0x8a, 0xb9, 0xe4, 0xa8, 0x99,
0x09, 0x3e, 0xac, 0xf9, 0x9a, 0x1b, 0xd9, 0x87, 0x1d, 0x27, 0xd4, 0xfc, 0xc6, 0xaf, 0xa0, 0x73,
0x4d, 0xe5, 0xad, 0x0c, 0xa5, 0x08, 0xe8, 0x97, 0x84, 0x0a, 0x89, 0x9e, 0xc1, 0xa9, 0x02, 0x96,
0x8c, 0xf4, 0x9c, 0x91, 0x33, 0x6e, 0xcf, 0xce, 0x7e, 0x3e, 0x0c, 0xff, 0xfb, 0xf5, 0x30, 0xac,
0x7f, 0xe0, 0x84, 0x2e, 0x5e, 0x07, 0x75, 0x65, 0x2f, 0x08, 0xfe, 0x06, 0xe7, 0xf9, 0x7f, 0x45,
0xc4, 0x77, 0x82, 0xa2, 0x21, 0xb4, 0xc2, 0x84, 0x30, 0xb9, 0x5c, 0xf1, 0x64, 0x27, 0x75, 0x80,
0x17, 0x80, 0x96, 0xe6, 0x4a, 0x41, 0x4f, 0xa1, 0x9d, 0x44, 0x92, 0xdd, 0xd3, 0x65, 0x1c, 0x4a,
0xc6, 0x7b, 0xee, 0xc8, 0x19, 0x3b, 0x41, 0xcb, 0x68, 0x81, 0x92, 0xf2, 0x0c, 0x43, 0x78, 0x9a,
0x30, 0x19, 0x1a, 0xc0, 0xbf, 0x1d, 0x40, 0xf3, 0x98, 0x86, 0x92, 0xfe, 0xd3, 0xc1, 0xf7, 0x0f,
0xe9, 0x1e, 0x1c, 0x72, 0x02, 0x5d, 0x03, 0x88, 0x64, 0xb5, 0xa2, 0x42, 0x58, 0xd0, 0xd3, 0xe0,
0x85, 0xb6, 0x6e, 0x8d, 0xb3, 0x5f, 0x94, 0x01, 0x4f, 0x34, 0x68, 0x8b, 0x32, 0xc8, 0x0b, 0xb8,
0xb4, 0x48, 0x39, 0xb3, 0xa6, 0x51, 0x64, 0xbc, 0x62, 0x28, 0xfe, 0x1f, 0xba, 0xa5, 0x22, 0x4d,
0x87, 0xf1, 0x3b, 0x40, 0xda, 0x57, 0x35, 0xe5, 0x7d, 0xf7, 0xa1, 0xb1, 0x09, 0x09, 0xbd, 0xdf,
0xb2, 0xd0, 0x36, 0x3d, 0x5b, 0xa3, 0x1e, 0x9c, 0xf2, 0xaf, 0x34, 0xde, 0x86, 0xdf, 0x6d, 0xa9,
0xe9, 0x12, 0x77, 0xe1, 0xa2, 0x98, 0xa5, 0xdb, 0xa8, 0xc4, 0x6b, 0x2a, 0x67, 0xc9, 0x6a, 0x43,
0xb3, 0xde, 0xe2, 0xb7, 0x80, 0x8a, 0xa2, 0xdd, 0xf5, 0x12, 0x6a, 0x92, 0xcb, 0x70, 0x6b, 0xb7,
0x34, 0x0b, 0x34, 0x00, 0x8f, 0x11, 0xd1, 0x73, 0x47, 0xde, 0xb8, 0x3d, 0x83, 0x42, 0xff, 0x95,
0x8c, 0xa7, 0xfa, 0xd6, 0x98, 0xa4, 0x74, 0x72, 0x57, 0xe0, 0x1e, 0x1d, 0x9a, 0xcb, 0x08, 0xfe,
0x54, 0x38, 0x52, 0xb6, 0xf9, 0x5f, 0xfe, 0x84, 0x46, 0x50, 0x53, 0xf3, 0x36, 0x07, 0x69, 0x4d,
0x61, 0xa2, 0xaf, 0xbd, 0x02, 0x02, 0x63, 0xe0, 0xe7, 0x50, 0x37, 0x99, 0x8f, 0x60, 0x27, 0x00,
0x86, 0xbd, 0x61, 0xa2, 0xc0, 0x3b, 0xc7, 0xf8, 0xf7, 0xd0, 0xf9, 0xc8, 0x76, 0x6b, 0x2d, 0x3d,
0xae, 0x4a, 0x35, 0xa7, 0x90, 0x90, 0x98, 0x0a, 0xa1, 0xe7, 0xd4, 0x0c, 0xd2, 0x25, 0xc6, 0x70,
0x9e, 0x87, 0xd9, 0xf2, 0xcf, 0xc0, 0xe5, 0x1b, 0x9d, 0xd6, 0x08, 0x5c, 0xbe, 0x99, 0xfe, 0xf0,
0xa0, 0xb9, 0x48, 0xbf, 0x77, 0xb4, 0x00, 0xc8, 0x27, 0x8b, 0x06, 0x93, 0xfc, 0x69, 0x38, 0x18,
0xb8, 0xff, 0xe4, 0x88, 0x6b, 0x37, 0x5a, 0x00, 0xe4, 0xa3, 0x2f, 0x45, 0x1d, 0x5c, 0x93, 0x52,
0x54, 0xc5, 0x7d, 0x79, 0x03, 0xcd, 0x4c, 0x45, 0xfd, 0x2a, 0x36, 0x0d, 0x1a, 0x54, 0x9b, 0x36,
0x67, 0x0e, 0x8d, 0xb4, 0x1f, 0xc8, 0x2f, 0x90, 0x7b, 0x1d, 0xf7, 0xfb, 0x95, 0x5e, 0x1e, 0x92,
0x3e, 0x5f, 0xa5, 0x90, 0xbd, 0xf7, 0xd0, 0xef, 0x57, 0x7a, 0x36, 0xe4, 0x06, 0x5a, 0x85, 0x8f,
0x14, 0x95, 0x5a, 0x79, 0xf0, 0x42, 0xf9, 0x57, 0xc7, 0x6c, 0x93, 0x36, 0x3b, 0xf9, 0xec, 0x46,
0x77, 0x77, 0x75, 0xfd, 0x34, 0xbf, 0xfc, 0x13, 0x00, 0x00, 0xff, 0xff, 0x66, 0x18, 0xa5, 0x1e,
0xd0, 0x05, 0x00, 0x00,
var fileDescriptor_inspector_44054ed20d312407 = []byte{
// 570 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x94, 0xcb, 0x6e, 0x13, 0x3f,
0x14, 0xc6, 0xff, 0x33, 0xb9, 0x34, 0x39, 0x89, 0x9a, 0xd6, 0xe9, 0x5f, 0x8a, 0x26, 0xa1, 0x09,
0xde, 0x10, 0xb1, 0x88, 0x50, 0xd8, 0xb1, 0x4c, 0x10, 0x25, 0x50, 0x21, 0x34, 0x15, 0x1b, 0x36,
0xd5, 0x34, 0xb6, 0x22, 0x2b, 0x69, 0x3c, 0x8c, 0x3d, 0x48, 0xbc, 0x0a, 0x6f, 0xc3, 0x8e, 0x67,
0x60, 0xd1, 0x0d, 0x2f, 0x82, 0x7c, 0x99, 0x5b, 0x66, 0x22, 0x2a, 0x76, 0xf1, 0xf9, 0xbe, 0xfc,
0xec, 0xe3, 0xcf, 0x73, 0xa0, 0xc7, 0xf6, 0x22, 0xa4, 0x6b, 0xc9, 0xa3, 0x59, 0x18, 0x71, 0xc9,
0x51, 0x3b, 0x2d, 0x78, 0xb0, 0xe1, 0x1b, 0x6e, 0xca, 0x1e, 0xec, 0x39, 0xa1, 0xe6, 0x37, 0x7e,
0x05, 0xbd, 0x2b, 0x2a, 0x6f, 0x64, 0x20, 0x85, 0x4f, 0xbf, 0xc4, 0x54, 0x48, 0xf4, 0x0c, 0x4e,
0x94, 0xe1, 0x96, 0x91, 0x81, 0x33, 0x71, 0xa6, 0xdd, 0xc5, 0xe9, 0xcf, 0x87, 0xf1, 0x7f, 0xbf,
0x1e, 0xc6, 0xcd, 0x0f, 0x9c, 0xd0, 0xd5, 0x6b, 0xbf, 0xa9, 0xe4, 0x15, 0xc1, 0xdf, 0x1d, 0x38,
0xcb, 0xfe, 0x2c, 0x42, 0xbe, 0x17, 0x14, 0x8d, 0xa1, 0x13, 0xc4, 0x84, 0xc9, 0xdb, 0x35, 0x8f,
0xf7, 0x52, 0x13, 0x6a, 0x3e, 0xe8, 0xd2, 0x52, 0x55, 0x32, 0x43, 0x14, 0x48, 0xc6, 0x07, 0xee,
0xc4, 0x99, 0x3a, 0xd6, 0xe0, 0xab, 0x0a, 0x7a, 0x0a, 0xdd, 0x38, 0x94, 0xec, 0x9e, 0x5a, 0x44,
0x4d, 0x23, 0x3a, 0xa6, 0x66, 0x18, 0x99, 0xc5, 0x40, 0xea, 0x1a, 0x62, 0x2d, 0x9a, 0x82, 0x7f,
0x3b, 0x80, 0x96, 0x11, 0x0d, 0x24, 0xfd, 0xa7, 0xe6, 0x0e, 0xfb, 0x70, 0x4b, 0x7d, 0xcc, 0xa0,
0x6f, 0x0c, 0x22, 0x5e, 0xaf, 0xa9, 0x10, 0x85, 0xd3, 0x9e, 0x6b, 0xe9, 0xc6, 0x28, 0x87, 0x67,
0x36, 0xc6, 0x7a, 0xb9, 0xad, 0x17, 0x70, 0x61, 0x2d, 0x45, 0x66, 0x43, 0x5b, 0x91, 0xd1, 0xf2,
0x50, 0xfc, 0x3f, 0xf4, 0x0b, 0x4d, 0x9a, 0x10, 0xf0, 0x3b, 0x40, 0x5a, 0x57, 0x3d, 0x65, 0xd1,
0x78, 0xd0, 0xda, 0x06, 0x84, 0xde, 0xef, 0x58, 0x60, 0x73, 0x49, 0xd7, 0x68, 0x00, 0x27, 0xfc,
0x2b, 0x8d, 0x76, 0xc1, 0x37, 0xdb, 0x6a, 0xb2, 0xc4, 0x7d, 0x38, 0xcf, 0xb3, 0xf4, 0x35, 0xaa,
0xe2, 0x15, 0x95, 0x8b, 0x78, 0xbd, 0xa5, 0xe9, 0xdd, 0xe2, 0xb7, 0x80, 0xf2, 0x45, 0xbb, 0xeb,
0x05, 0x34, 0x24, 0x97, 0xc1, 0xce, 0x6e, 0x69, 0x16, 0x68, 0x04, 0x35, 0x46, 0xc4, 0xc0, 0x9d,
0xd4, 0xa6, 0xdd, 0x05, 0xe4, 0xee, 0x5f, 0x95, 0xf1, 0x5c, 0x3f, 0x2c, 0x43, 0x4a, 0x92, 0xbb,
0x04, 0xf7, 0x68, 0x68, 0x2e, 0x23, 0xf8, 0x53, 0xee, 0x48, 0xe9, 0xe6, 0x7f, 0xf9, 0x13, 0x9a,
0x40, 0x43, 0xe5, 0x6d, 0x0e, 0xd2, 0x99, 0xc3, 0x4c, 0x7f, 0x1a, 0xca, 0xe0, 0x1b, 0x01, 0x3f,
0x87, 0xa6, 0x61, 0x3e, 0xc2, 0x3b, 0x03, 0x30, 0xde, 0x6b, 0x26, 0x72, 0x7e, 0xe7, 0x98, 0xff,
0x3d, 0xf4, 0x3e, 0xb2, 0xfd, 0x46, 0x97, 0x1e, 0xd7, 0xa5, 0xca, 0x29, 0x20, 0x24, 0xa2, 0x42,
0xe8, 0x9c, 0xda, 0x7e, 0xb2, 0xc4, 0x18, 0xce, 0x32, 0x98, 0x6d, 0xff, 0x14, 0x5c, 0xbe, 0xd5,
0xb4, 0x96, 0xef, 0xf2, 0xed, 0xfc, 0x47, 0x0d, 0xda, 0xab, 0x64, 0x26, 0xa0, 0x15, 0x40, 0x96,
0x2c, 0x1a, 0xcd, 0xb2, 0xf1, 0x51, 0x0a, 0xdc, 0x7b, 0x72, 0x44, 0xb5, 0x1b, 0xad, 0x00, 0xb2,
0xe8, 0x0b, 0xa8, 0xd2, 0x33, 0x29, 0xa0, 0x2a, 0xde, 0xcb, 0x1b, 0x68, 0xa7, 0x55, 0x34, 0xac,
0xf2, 0x26, 0xa0, 0x51, 0xb5, 0x68, 0x39, 0x4b, 0x68, 0x25, 0xf7, 0x81, 0xbc, 0x9c, 0xf3, 0xe0,
0xc6, 0xbd, 0x61, 0xa5, 0x96, 0x41, 0x92, 0x09, 0x57, 0x80, 0x1c, 0xcc, 0x4c, 0x6f, 0x58, 0xa9,
0x59, 0xc8, 0x35, 0x74, 0x72, 0x1f, 0x29, 0x2a, 0x5c, 0x65, 0x69, 0x42, 0x79, 0x97, 0xc7, 0x64,
0x43, 0x5b, 0xd4, 0x3f, 0xbb, 0xe1, 0xdd, 0x5d, 0x53, 0x8f, 0xef, 0x97, 0x7f, 0x02, 0x00, 0x00,
0xff, 0xff, 0x9c, 0x78, 0xee, 0x95, 0xf4, 0x05, 0x00, 0x00,
}

5
pkg/pb/inspector.proto
View File

@ -34,8 +34,9 @@ message GetStatsRequest {
message GetStatsResponse {
int64 audit_count = 1;
double uptime_ratio = 2;
double audit_ratio = 3;
double audit_ratio = 2;
int64 uptime_count = 3;
double uptime_ratio = 4;
}
// CreateStats

10
pkg/statdb/config.go
View File

@ -9,7 +9,6 @@ import (
"go.uber.org/zap"
"storj.io/storj/pkg/provider"
pb "storj.io/storj/pkg/statdb/proto"
)
//CtxKey Used as statdb key
@ -24,25 +23,22 @@ const (
type Config struct {
DatabaseURL string `help:"the database connection string to use" default:"$CONFDIR/stats.db"`
DatabaseDriver string `help:"the database driver to use" default:"sqlite3"`
APIKey string `help:"the statdb api key to use" default:"abc123"`
}
// Run implements the provider.Responsibility interface
func (c Config) Run(ctx context.Context, server *provider.Provider) error {
ns, err := NewServer(c.DatabaseDriver, c.DatabaseURL, c.APIKey, zap.L())
ns, err := NewStatDB(c.DatabaseDriver, c.DatabaseURL, zap.L())
if err != nil {
return err
}
pb.RegisterStatDBServer(server.GRPC(), ns)
return server.Run(context.WithValue(ctx, ctxKeyStats, ns))
}
// LoadFromContext loads an existing StatDB from the Provider context
// stack if one exists.
func LoadFromContext(ctx context.Context) *Server {
if v, ok := ctx.Value(ctxKeyStats).(*Server); ok {
func LoadFromContext(ctx context.Context) *StatDB {
if v, ok := ctx.Value(ctxKeyStats).(*StatDB); ok {
return v
}
return nil

227
pkg/statdb/sdbclient/client.go
View File

@ -1,227 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package sdbclient
import (
"context"
"google.golang.org/grpc"
"gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/auth/grpcauth"
"storj.io/storj/pkg/provider"
pb "storj.io/storj/pkg/statdb/proto"
"storj.io/storj/pkg/storj"
"storj.io/storj/pkg/transport"
)
var (
mon = monkit.Package()
)
// StatDB creates a grpcClient
type StatDB struct {
client pb.StatDBClient
APIKey []byte
}
// Client services offerred for the interface
type Client interface {
Create(ctx context.Context, id storj.NodeID) error
CreateWithStats(ctx context.Context, id storj.NodeID, stats *pb.NodeStats) error
Get(ctx context.Context, id storj.NodeID) (*pb.NodeStats, error)
FindValidNodes(ctx context.Context, ids storj.NodeIDList, minStats *pb.NodeStats) (passedIDs storj.NodeIDList, err error)
Update(ctx context.Context, id storj.NodeID, auditSuccess, isUp bool,
latencyList []int64) (stats *pb.NodeStats, err error)
UpdateUptime(ctx context.Context, id storj.NodeID, isUp bool) (*pb.NodeStats, error)
UpdateAuditSuccess(ctx context.Context, id storj.NodeID, passed bool) (*pb.NodeStats, error)
UpdateBatch(ctx context.Context, nodes []*pb.Node) ([]*pb.NodeStats, []*pb.Node, error)
CreateEntryIfNotExists(ctx context.Context, id storj.NodeID) (stats *pb.NodeStats, err error)
}
// NewClient initializes a new statdb client
func NewClient(identity *provider.FullIdentity, address string, APIKey string) (Client, error) {
apiKeyInjector := grpcauth.NewAPIKeyInjector(APIKey)
tc := transport.NewClient(identity)
conn, err := tc.DialAddress(
context.Background(),
address,
grpc.WithUnaryInterceptor(apiKeyInjector),
)
if err != nil {
return nil, err
}
return &StatDB{client: pb.NewStatDBClient(conn)}, nil
}
// a compiler trick to make sure *StatDB implements Client
var _ Client = (*StatDB)(nil)
// Create is used for creating a new entry in the stats db with default reputation
func (sdb *StatDB) Create(ctx context.Context, id storj.NodeID) (err error) {
defer mon.Task()(&ctx)(&err)
node := pb.Node{
Id: id,
}
createReq := &pb.CreateRequest{
Node: &node,
}
_, err = sdb.client.Create(ctx, createReq)
return err
}
// CreateWithStats is used for creating a new entry in the stats db with a specific reputation
// stats must have AuditCount, AuditSuccessCount, UptimeCount, UptimeSuccessCount
func (sdb *StatDB) CreateWithStats(ctx context.Context, id storj.NodeID, stats *pb.NodeStats) (err error) {
defer mon.Task()(&ctx)(&err)
node := &pb.Node{
Id: id,
}
createReq := &pb.CreateRequest{
Node: node,
Stats: stats,
}
_, err = sdb.client.Create(ctx, createReq)
return err
}
// Get is used for retrieving a new entry from the stats db
func (sdb *StatDB) Get(ctx context.Context, id storj.NodeID) (stats *pb.NodeStats, err error) {
defer mon.Task()(&ctx)(&err)
getReq := &pb.GetRequest{
NodeId: id,
}
res, err := sdb.client.Get(ctx, getReq)
if err != nil {
return nil, err
}
return res.Stats, nil
}
// FindValidNodes is used for retrieving a subset of nodes that meet a minimum reputation requirement
// minStats must have AuditSuccessRatio, UptimeRatio, AuditCount
func (sdb *StatDB) FindValidNodes(ctx context.Context, ids storj.NodeIDList,
minStats *pb.NodeStats) (passedIDs storj.NodeIDList, err error) {
defer mon.Task()(&ctx)(&err)
findValidNodesReq := &pb.FindValidNodesRequest{
NodeIds: ids,
MinStats: minStats,
}
res, err := sdb.client.FindValidNodes(ctx, findValidNodesReq)
if err != nil {
return nil, err
}
return res.PassedIds, nil
}
// Update is used for updating a node's stats in the stats db
func (sdb *StatDB) Update(ctx context.Context, id storj.NodeID,
auditSuccess, isUp bool, latencyList []int64) (stats *pb.NodeStats, err error) {
defer mon.Task()(&ctx)(&err)
node := pb.Node{
Id: id,
AuditSuccess: auditSuccess,
IsUp: isUp,
LatencyList: latencyList,
UpdateAuditSuccess: true,
UpdateUptime: true,
UpdateLatency: true,
}
updateReq := &pb.UpdateRequest{
Node: &node,
}
res, err := sdb.client.Update(ctx, updateReq)
if err != nil {
return nil, err
}
return res.Stats, nil
}
// UpdateUptime is used for updating a node's uptime in statdb
func (sdb *StatDB) UpdateUptime(ctx context.Context, id storj.NodeID,
isUp bool) (stats *pb.NodeStats, err error) {
defer mon.Task()(&ctx)(&err)
node := pb.Node{
Id: id,
IsUp: isUp,
}
updateReq := &pb.UpdateUptimeRequest{
Node: &node,
}
res, err := sdb.client.UpdateUptime(ctx, updateReq)
if err != nil {
return nil, err
}
return res.Stats, nil
}
// UpdateAuditSuccess is used for updating a node's audit success in statdb
func (sdb *StatDB) UpdateAuditSuccess(ctx context.Context, id storj.NodeID,
passed bool) (stats *pb.NodeStats, err error) {
defer mon.Task()(&ctx)(&err)
node := pb.Node{
Id: id,
AuditSuccess: passed,
}
updateReq := &pb.UpdateAuditSuccessRequest{
Node: &node,
}
res, err := sdb.client.UpdateAuditSuccess(ctx, updateReq)
if err != nil {
return nil, err
}
return res.Stats, nil
}
// UpdateBatch is used for updating multiple nodes' stats in the stats db
func (sdb *StatDB) UpdateBatch(ctx context.Context, nodes []*pb.Node) (statsList []*pb.NodeStats, failedNodes []*pb.Node, err error) {
defer mon.Task()(&ctx)(&err)
updateBatchReq := &pb.UpdateBatchRequest{
NodeList: nodes,
}
res, err := sdb.client.UpdateBatch(ctx, updateBatchReq)
if err != nil {
return nil, nil, err
}
return res.StatsList, res.FailedNodes, nil
}
// CreateEntryIfNotExists creates a db entry for a node if entry doesn't already exist
func (sdb *StatDB) CreateEntryIfNotExists(ctx context.Context, id storj.NodeID) (stats *pb.NodeStats, err error) {
defer mon.Task()(&ctx)(&err)
node := &pb.Node{Id: id}
createReq := &pb.CreateEntryIfNotExistsRequest{
Node: node,
}
res, err := sdb.client.CreateEntryIfNotExists(ctx, createReq)
if err != nil {
return nil, err
}
return res.Stats, nil
}

11
pkg/statdb/sdbclient/common.go
View File

@ -1,11 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package sdbclient
import (
"github.com/zeebo/errs"
)
// Error is the sdbclient error class
var Error = errs.Class("statdb client error")

95
pkg/statdb/sdbclient/mock_client.go
View File

@ -1,95 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package sdbclient
import (
"context"
pb "storj.io/storj/pkg/statdb/proto"
"storj.io/storj/pkg/storj"
)
// MockStatDB creates a noop Mock Statdb Client
type MockStatDB struct{}
// NewMockClient initializes a new mock statdb client
func NewMockClient() Client {
return &MockStatDB{}
}
// a compiler trick to make sure *MockStatDB implements Client
var _ Client = (*MockStatDB)(nil)
// Create is used for creating a new entry in the stats db with default reputation
func (sdb *MockStatDB) Create(ctx context.Context, id storj.NodeID) (err error) {
return nil
}
// CreateWithStats is used for creating a new entry in the stats db with a specific reputation
func (sdb *MockStatDB) CreateWithStats(ctx context.Context, id storj.NodeID, stats *pb.NodeStats) (err error) {
return nil
}
// Get is used for retrieving an entry from statdb or creating a new one if one does not exist
func (sdb *MockStatDB) Get(ctx context.Context, id storj.NodeID) (stats *pb.NodeStats, err error) {
stats = &pb.NodeStats{
AuditSuccessRatio: 0,
UptimeRatio: 0,
AuditCount: 0,
}
return stats, nil
}
// FindValidNodes is used for retrieving a subset of nodes that meet a minimum reputation requirement
func (sdb *MockStatDB) FindValidNodes(ctx context.Context, iDs storj.NodeIDList, minStats *pb.NodeStats) (passedIDs storj.NodeIDList, err error) {
return nil, nil
}
// Update is used for updating a node's stats in the stats db
func (sdb *MockStatDB) Update(ctx context.Context, id storj.NodeID, auditSuccess,
isUp bool, latencyList []int64) (stats *pb.NodeStats, err error) {
stats = &pb.NodeStats{
AuditSuccessRatio: 0,
UptimeRatio: 0,
AuditCount: 0,
}
return stats, nil
}
// UpdateUptime is used for updating a node's uptime in statdb
func (sdb *MockStatDB) UpdateUptime(ctx context.Context, id storj.NodeID,
isUp bool) (stats *pb.NodeStats, err error) {
stats = &pb.NodeStats{
AuditSuccessRatio: 0,
UptimeRatio: 0,
AuditCount: 0,
}
return stats, nil
}
// UpdateAuditSuccess is used for updating a node's audit success in statdb
func (sdb *MockStatDB) UpdateAuditSuccess(ctx context.Context, id storj.NodeID,
passed bool) (stats *pb.NodeStats, err error) {
stats = &pb.NodeStats{
AuditSuccessRatio: 0,
UptimeRatio: 0,
AuditCount: 0,
}
return stats, nil
}
// UpdateBatch is used for updating multiple nodes' stats in the stats db
func (sdb *MockStatDB) UpdateBatch(ctx context.Context, nodes []*pb.Node) (statsList []*pb.NodeStats, failedNodes []*pb.Node, err error) {
return nil, nil, nil
}
// CreateEntryIfNotExists creates a db entry for a node if entry doesn't already exist
func (sdb *MockStatDB) CreateEntryIfNotExists(ctx context.Context, id storj.NodeID) (stats *pb.NodeStats, err error) {
stats = &pb.NodeStats{
AuditSuccessRatio: 0,
UptimeRatio: 0,
AuditCount: 0,
}
return stats, nil
}

71
pkg/statdb/statdb.go
View File

@ -15,7 +15,6 @@ import (
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/internal/migrate"
"storj.io/storj/pkg/auth"
dbx "storj.io/storj/pkg/statdb/dbx"
pb "storj.io/storj/pkg/statdb/proto"
"storj.io/storj/pkg/storj"
@ -27,15 +26,14 @@ var (
errUptime = errs.Class("statdb uptime error")
)
// Server implements the statdb RPC service
type Server struct {
log *zap.Logger
DB *dbx.DB
apiKey []byte
// StatDB implements the statdb RPC service
type StatDB struct {
log *zap.Logger
DB *dbx.DB
}
// NewServer creates instance of Server
func NewServer(driver, source, apiKey string, log *zap.Logger) (*Server, error) {
// NewStatDB creates instance of StatDB
func NewStatDB(driver, source string, log *zap.Logger) (*StatDB, error) {
db, err := dbx.Open(driver, source)
if err != nil {
return nil, err
@ -46,29 +44,16 @@ func NewServer(driver, source, apiKey string, log *zap.Logger) (*Server, error)
return nil, err
}
return &Server{
DB: db,
log: log,
apiKey: []byte(apiKey),
return &StatDB{
DB: db,
log: log,
}, nil
}
func (s *Server) validateAuth(ctx context.Context) error {
err := auth.ValidateAPIKey(ctx, s.apiKey)
if err != nil {
return err
}
return nil
}
// Create a db entry for the provided storagenode
func (s *Server) Create(ctx context.Context, createReq *pb.CreateRequest) (resp *pb.CreateResponse, err error) {
func (s *StatDB) Create(ctx context.Context, createReq *pb.CreateRequest) (resp *pb.CreateResponse, err error) {
defer mon.Task()(&ctx)(&err)
if err := s.validateAuth(ctx); err != nil {
return nil, err
}
var (
totalAuditCount int64
auditSuccessCount int64
@ -123,14 +108,9 @@ func (s *Server) Create(ctx context.Context, createReq *pb.CreateRequest) (resp
}
// Get a storagenode's stats from the db
func (s *Server) Get(ctx context.Context, getReq *pb.GetRequest) (resp *pb.GetResponse, err error) {
func (s *StatDB) Get(ctx context.Context, getReq *pb.GetRequest) (resp *pb.GetResponse, err error) {
defer mon.Task()(&ctx)(&err)
err = s.validateAuth(ctx)
if err != nil {
return nil, err
}
dbNode, err := s.DB.Get_Node_By_Id(ctx, dbx.Node_Id(getReq.NodeId.Bytes()))
if err != nil {
return nil, status.Errorf(codes.Internal, err.Error())
@ -148,7 +128,7 @@ func (s *Server) Get(ctx context.Context, getReq *pb.GetRequest) (resp *pb.GetRe
}
// FindValidNodes finds a subset of storagenodes that meet reputation requirements
func (s *Server) FindValidNodes(ctx context.Context, getReq *pb.FindValidNodesRequest) (resp *pb.FindValidNodesResponse, err error) {
func (s *StatDB) FindValidNodes(ctx context.Context, getReq *pb.FindValidNodesRequest) (resp *pb.FindValidNodesResponse, err error) {
defer mon.Task()(&ctx)(&err)
var passedIds storj.NodeIDList
@ -188,7 +168,7 @@ func (s *Server) FindValidNodes(ctx context.Context, getReq *pb.FindValidNodesRe
}, nil
}
func (s *Server) findValidNodesQuery(nodeIds storj.NodeIDList, auditCount int64, auditSuccess, uptime float64) (*sql.Rows, error) {
func (s *StatDB) findValidNodesQuery(nodeIds storj.NodeIDList, auditCount int64, auditSuccess, uptime float64) (*sql.Rows, error) {
args := make([]interface{}, len(nodeIds))
for i, id := range nodeIds {
args[i] = id.Bytes()
@ -207,14 +187,9 @@ func (s *Server) findValidNodesQuery(nodeIds storj.NodeIDList, auditCount int64,
}
// Update a single storagenode's stats in the db
func (s *Server) Update(ctx context.Context, updateReq *pb.UpdateRequest) (resp *pb.UpdateResponse, err error) {
func (s *StatDB) Update(ctx context.Context, updateReq *pb.UpdateRequest) (resp *pb.UpdateResponse, err error) {
defer mon.Task()(&ctx)(&err)
err = s.validateAuth(ctx)
if err != nil {
return nil, err
}
node := updateReq.GetNode()
createIfReq := &pb.CreateEntryIfNotExistsRequest{
@ -279,14 +254,9 @@ func (s *Server) Update(ctx context.Context, updateReq *pb.UpdateRequest) (resp
}
// UpdateUptime updates a single storagenode's uptime stats in the db
func (s *Server) UpdateUptime(ctx context.Context, updateReq *pb.UpdateUptimeRequest) (resp *pb.UpdateUptimeResponse, err error) {
func (s *StatDB) UpdateUptime(ctx context.Context, updateReq *pb.UpdateUptimeRequest) (resp *pb.UpdateUptimeResponse, err error) {
defer mon.Task()(&ctx)(&err)
err = s.validateAuth(ctx)
if err != nil {
return nil, err
}
node := updateReq.GetNode()
dbNode, err := s.DB.Get_Node_By_Id(ctx, dbx.Node_Id(node.Id.Bytes()))
@ -327,14 +297,9 @@ func (s *Server) UpdateUptime(ctx context.Context, updateReq *pb.UpdateUptimeReq
}
// UpdateAuditSuccess updates a single storagenode's uptime stats in the db
func (s *Server) UpdateAuditSuccess(ctx context.Context, updateReq *pb.UpdateAuditSuccessRequest) (resp *pb.UpdateAuditSuccessResponse, err error) {
func (s *StatDB) UpdateAuditSuccess(ctx context.Context, updateReq *pb.UpdateAuditSuccessRequest) (resp *pb.UpdateAuditSuccessResponse, err error) {
defer mon.Task()(&ctx)(&err)
err = s.validateAuth(ctx)
if err != nil {
return nil, err
}
node := updateReq.GetNode()
dbNode, err := s.DB.Get_Node_By_Id(ctx, dbx.Node_Id(node.Id.Bytes()))
@ -375,7 +340,7 @@ func (s *Server) UpdateAuditSuccess(ctx context.Context, updateReq *pb.UpdateAud
}
// UpdateBatch for updating multiple farmers' stats in the db
func (s *Server) UpdateBatch(ctx context.Context, updateBatchReq *pb.UpdateBatchRequest) (resp *pb.UpdateBatchResponse, err error) {
func (s *StatDB) UpdateBatch(ctx context.Context, updateBatchReq *pb.UpdateBatchRequest) (resp *pb.UpdateBatchResponse, err error) {
defer mon.Task()(&ctx)(&err)
var nodeStatsList []*pb.NodeStats
@ -402,7 +367,7 @@ func (s *Server) UpdateBatch(ctx context.Context, updateBatchReq *pb.UpdateBatch
}
// CreateEntryIfNotExists creates a statdb node entry and saves to statdb if it didn't already exist
func (s *Server) CreateEntryIfNotExists(ctx context.Context, createIfReq *pb.CreateEntryIfNotExistsRequest) (resp *pb.CreateEntryIfNotExistsResponse, err error) {
func (s *StatDB) CreateEntryIfNotExists(ctx context.Context, createIfReq *pb.CreateEntryIfNotExistsRequest) (resp *pb.CreateEntryIfNotExistsResponse, err error) {
defer mon.Task()(&ctx)(&err)
getReq := &pb.GetRequest{

12
pkg/statdb/statdb_test.go
View File

@ -13,7 +13,6 @@ import (
"go.uber.org/zap"
"storj.io/storj/internal/teststorj"
"storj.io/storj/pkg/auth"
"storj.io/storj/pkg/statdb"
dbx "storj.io/storj/pkg/statdb/dbx"
pb "storj.io/storj/pkg/statdb/proto"
@ -21,8 +20,7 @@ import (
)
var (
apiKey = []byte("")
ctx = auth.WithAPIKey(context.Background(), apiKey)
ctx = context.Background()
)
func TestCreateDoesNotExist(t *testing.T) {
@ -434,14 +432,14 @@ func getDBPath() string {
return fmt.Sprintf("file:memdb%d?mode=memory&cache=shared", rand.Int63())
}
func getServerAndDB(path string) (sdb *statdb.Server, db *dbx.DB, err error) {
sdb, err = statdb.NewServer("sqlite3", path, string(apiKey), zap.NewNop())
func getServerAndDB(path string) (sdb *statdb.StatDB, db *dbx.DB, err error) {
sdb, err = statdb.NewStatDB("sqlite3", path, zap.NewNop())
if err != nil {
return &statdb.Server{}, &dbx.DB{}, err
return &statdb.StatDB{}, &dbx.DB{}, err
}
db, err = dbx.Open("sqlite3", path)
if err != nil {
return &statdb.Server{}, &dbx.DB{}, err
return &statdb.StatDB{}, &dbx.DB{}, err
}
return sdb, db, err
}