pkg/debug: separate debug endpoints to a server
By separating Server it allows Peers to directly embed the server and provide customizations and hooks into rest of the services. Change-Id: Ic1d68740fd494d2f82c1739bd990849c561b912b
This commit is contained in:
Egon Elbre
parent
e0cb8037c1
commit
f833289e1b
1
.gitignore
vendored
1
.gitignore
vendored
@ -17,7 +17,6 @@
|
||||
|
||||
# VSCode
|
||||
/.vscode
|
||||
debug
|
||||
|
||||
# Visual Studio
|
||||
/.vs
|
||||
|
||||
9
pkg/debug/common.go
Normal file
9
pkg/debug/common.go
Normal file
@ -0,0 +1,9 @@
|
||||
// Copyright (C) 2020 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package debug
|
||||
|
||||
import "github.com/zeebo/errs"
|
||||
|
||||
// Error is default error class for debug package.
|
||||
var Error = errs.Class("debug")
|
||||
@ -1,7 +1,7 @@
|
||||
// Copyright (C) 2019 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package process
|
||||
package debug
|
||||
|
||||
import (
|
||||
"google.golang.org/grpc/status"
|
||||
141
pkg/debug/server.go
Normal file
141
pkg/debug/server.go
Normal file
@ -0,0 +1,141 @@
|
||||
// Copyright (C) 2019 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
// Package debug implements debug server for satellite and storage node.
|
||||
package debug
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/pprof"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"gopkg.in/spacemonkeygo/monkit.v2"
|
||||
"gopkg.in/spacemonkeygo/monkit.v2/present"
|
||||
|
||||
"storj.io/storj/pkg/traces"
|
||||
"storj.io/storj/private/version/checker"
|
||||
)
|
||||
|
||||
func init() {
|
||||
// zero out the http.DefaultServeMux net/http/pprof so unhelpfully
|
||||
// side-effected.
|
||||
*http.DefaultServeMux = http.ServeMux{}
|
||||
}
|
||||
|
||||
// Config defines configuration for debug server.
|
||||
type Config struct {
|
||||
Address string `internal:"true"`
|
||||
}
|
||||
|
||||
// Server provides endpoints for debugging.
|
||||
type Server struct {
|
||||
log *zap.Logger
|
||||
|
||||
listener net.Listener
|
||||
server http.Server
|
||||
mux http.ServeMux
|
||||
|
||||
registry *monkit.Registry
|
||||
}
|
||||
|
||||
// NewServer returns a new debug.Server
|
||||
func NewServer(log *zap.Logger, listener net.Listener, registry *monkit.Registry, config Config) *Server {
|
||||
server := &Server{log: log}
|
||||
|
||||
server.listener = listener
|
||||
server.server.Handler = &server.mux
|
||||
server.registry = registry
|
||||
|
||||
server.mux.HandleFunc("/debug/pprof/", pprof.Index)
|
||||
server.mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
|
||||
server.mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
|
||||
server.mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
|
||||
server.mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
|
||||
|
||||
server.mux.HandleFunc("/debug/run/trace/db", server.collectTraces)
|
||||
|
||||
server.mux.Handle("/version/", http.StripPrefix("/version", checker.NewDebugHandler(log.Named("version"))))
|
||||
server.mux.Handle("/mon/", http.StripPrefix("/mon", present.HTTP(server.registry)))
|
||||
server.mux.HandleFunc("/metrics", server.metrics)
|
||||
server.mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = fmt.Fprintln(w, "OK")
|
||||
})
|
||||
|
||||
return server
|
||||
}
|
||||
|
||||
// Run starts the debug endpoint.
|
||||
func (server *Server) Run(ctx context.Context) error {
|
||||
if server.listener == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
var group errgroup.Group
|
||||
group.Go(func() error {
|
||||
<-ctx.Done()
|
||||
return Error.Wrap(server.server.Shutdown(context.Background()))
|
||||
})
|
||||
group.Go(func() error {
|
||||
defer cancel()
|
||||
return Error.Wrap(server.server.Serve(server.listener))
|
||||
})
|
||||
return group.Wait()
|
||||
}
|
||||
|
||||
// Close closes server and underlying listener
|
||||
func (server *Server) Close() error {
|
||||
return Error.Wrap(server.server.Close())
|
||||
}
|
||||
|
||||
func (server *Server) metrics(w http.ResponseWriter, r *http.Request) {
|
||||
// writes https://prometheus.io/docs/instrumenting/exposition_formats/
|
||||
// TODO(jt): deeper monkit integration so we can expose prometheus types
|
||||
// (https://prometheus.io/docs/concepts/metric_types/)
|
||||
server.registry.Stats(func(name string, val float64) {
|
||||
metric := sanitize(name)
|
||||
_, _ = fmt.Fprintf(w, "# TYPE %s gauge\n%s %g\n",
|
||||
metric, metric, val)
|
||||
})
|
||||
}
|
||||
|
||||
// collectTraces
|
||||
func (server *Server) collectTraces(w http.ResponseWriter, r *http.Request) {
|
||||
cancel := traces.CollectTraces()
|
||||
defer cancel()
|
||||
for {
|
||||
_, err := w.Write([]byte{0})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func sanitize(val string) string {
|
||||
// https://prometheus.io/docs/concepts/data_model/
|
||||
// specifies all metric names must match [a-zA-Z_:][a-zA-Z0-9_:]*
|
||||
// Note: The colons are reserved for user defined recording rules.
|
||||
// They should not be used by exporters or direct instrumentation.
|
||||
if '0' <= val[0] && val[0] <= '9' {
|
||||
val = "_" + val
|
||||
}
|
||||
return strings.Map(func(r rune) rune {
|
||||
switch {
|
||||
case 'a' <= r && r <= 'z':
|
||||
return r
|
||||
case 'A' <= r && r <= 'Z':
|
||||
return r
|
||||
case '0' <= r && r <= '9':
|
||||
return r
|
||||
default:
|
||||
return '_'
|
||||
}
|
||||
}, val)
|
||||
}
|
||||
@ -4,105 +4,39 @@
|
||||
package process
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/pprof"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"gopkg.in/spacemonkeygo/monkit.v2"
|
||||
"gopkg.in/spacemonkeygo/monkit.v2/present"
|
||||
|
||||
"storj.io/storj/pkg/traces"
|
||||
"storj.io/storj/private/version/checker"
|
||||
"storj.io/storj/pkg/debug"
|
||||
)
|
||||
|
||||
var (
|
||||
debugAddr = flag.String("debug.addr", "127.0.0.1:0", "address to listen on for debug endpoints")
|
||||
)
|
||||
|
||||
func init() {
|
||||
// zero out the http.DefaultServeMux net/http/pprof so unhelpfully
|
||||
// side-effected.
|
||||
*http.DefaultServeMux = http.ServeMux{}
|
||||
}
|
||||
|
||||
func initDebug(logger *zap.Logger, r *monkit.Registry) (err error) {
|
||||
var mux http.ServeMux
|
||||
mux.HandleFunc("/debug/pprof/", pprof.Index)
|
||||
mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
|
||||
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
|
||||
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
|
||||
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
|
||||
|
||||
mux.HandleFunc("/debug/run/trace/db", traceDB)
|
||||
|
||||
mux.Handle("/version/", http.StripPrefix("/version", checker.NewDebugHandler(logger.Named("version"))))
|
||||
mux.Handle("/mon/", http.StripPrefix("/mon", present.HTTP(r)))
|
||||
mux.HandleFunc("/metrics", prometheus)
|
||||
mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = fmt.Fprintln(w, "OK")
|
||||
})
|
||||
if *debugAddr != "" {
|
||||
ln, err := net.Listen("tcp", *debugAddr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
logger.Debug(fmt.Sprintf("debug server listening on %s", ln.Addr().String()))
|
||||
err := (&http.Server{Handler: &mux}).Serve(ln)
|
||||
if err != nil {
|
||||
logger.Error("debug server died", zap.Error(err))
|
||||
}
|
||||
}()
|
||||
func initDebug(log *zap.Logger, r *monkit.Registry) (err error) {
|
||||
if *debugAddr == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
ln, err := net.Listen("tcp", *debugAddr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
go func() {
|
||||
server := debug.NewServer(log, ln, r, debug.Config{Address: *debugAddr})
|
||||
log.Debug(fmt.Sprintf("debug server listening on %s", ln.Addr().String()))
|
||||
err := server.Run(context.TODO())
|
||||
if err != nil {
|
||||
log.Error("debug server died", zap.Error(err))
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func sanitize(val string) string {
|
||||
// https://prometheus.io/docs/concepts/data_model/
|
||||
// specifies all metric names must match [a-zA-Z_:][a-zA-Z0-9_:]*
|
||||
// Note: The colons are reserved for user defined recording rules.
|
||||
// They should not be used by exporters or direct instrumentation.
|
||||
if '0' <= val[0] && val[0] <= '9' {
|
||||
val = "_" + val
|
||||
}
|
||||
return strings.Map(func(r rune) rune {
|
||||
switch {
|
||||
case 'a' <= r && r <= 'z':
|
||||
return r
|
||||
case 'A' <= r && r <= 'Z':
|
||||
return r
|
||||
case '0' <= r && r <= '9':
|
||||
return r
|
||||
default:
|
||||
return '_'
|
||||
}
|
||||
}, val)
|
||||
}
|
||||
|
||||
func prometheus(w http.ResponseWriter, r *http.Request) {
|
||||
// writes https://prometheus.io/docs/instrumenting/exposition_formats/
|
||||
// TODO(jt): deeper monkit integration so we can expose prometheus types
|
||||
// (https://prometheus.io/docs/concepts/metric_types/)
|
||||
monkit.Default.Stats(func(name string, val float64) {
|
||||
metric := sanitize(name)
|
||||
_, _ = fmt.Fprintf(w, "# TYPE %s gauge\n%s %g\n",
|
||||
metric, metric, val)
|
||||
})
|
||||
}
|
||||
|
||||
func traceDB(w http.ResponseWriter, r *http.Request) {
|
||||
cancel := traces.CollectTraces()
|
||||
defer cancel()
|
||||
for {
|
||||
_, err := w.Write([]byte{0})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user