f41d440944
Remove starting up messages from peers. We expect all of them to start, if they don't, then they should return an error why they don't start. The only informative message is when a service is disabled. When doing initial database setup then each migration step isn't informative, hence print only a single line with the final version. Also use shorter log scopes. Change-Id: Ic8b61411df2eeae2a36d600a0c2fbc97a84a5b93
420 lines
13 KiB
Go
420 lines
13 KiB
Go
// Copyright (C) 2019 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"text/tabwriter"
|
|
|
|
"github.com/skyrings/skyring-common/tools/uuid"
|
|
"github.com/spf13/cobra"
|
|
"github.com/zeebo/errs"
|
|
"go.uber.org/zap"
|
|
|
|
"storj.io/common/fpath"
|
|
"storj.io/common/storj"
|
|
"storj.io/storj/cmd/satellite/reports"
|
|
"storj.io/storj/pkg/cfgstruct"
|
|
"storj.io/storj/pkg/process"
|
|
"storj.io/storj/pkg/revocation"
|
|
"storj.io/storj/private/version"
|
|
"storj.io/storj/satellite"
|
|
"storj.io/storj/satellite/accounting/live"
|
|
"storj.io/storj/satellite/metainfo"
|
|
"storj.io/storj/satellite/satellitedb"
|
|
)
|
|
|
|
// Satellite defines satellite configuration
|
|
type Satellite struct {
|
|
Database string `help:"satellite database connection string" releaseDefault:"postgres://" devDefault:"postgres://"`
|
|
|
|
satellite.Config
|
|
}
|
|
|
|
var (
|
|
rootCmd = &cobra.Command{
|
|
Use: "satellite",
|
|
Short: "Satellite",
|
|
}
|
|
runCmd = &cobra.Command{
|
|
Use: "run",
|
|
Short: "Run the satellite",
|
|
RunE: cmdRun,
|
|
}
|
|
runMigrationCmd = &cobra.Command{
|
|
Use: "migration",
|
|
Short: "Run the satellite database migration",
|
|
RunE: cmdMigrationRun,
|
|
}
|
|
runAPICmd = &cobra.Command{
|
|
Use: "api",
|
|
Short: "Run the satellite API",
|
|
RunE: cmdAPIRun,
|
|
}
|
|
runRepairerCmd = &cobra.Command{
|
|
Use: "repair",
|
|
Short: "Run the repair service",
|
|
RunE: cmdRepairerRun,
|
|
}
|
|
setupCmd = &cobra.Command{
|
|
Use: "setup",
|
|
Short: "Create config files",
|
|
RunE: cmdSetup,
|
|
Annotations: map[string]string{"type": "setup"},
|
|
}
|
|
qdiagCmd = &cobra.Command{
|
|
Use: "qdiag",
|
|
Short: "Repair Queue Diagnostic Tool support",
|
|
RunE: cmdQDiag,
|
|
}
|
|
reportsCmd = &cobra.Command{
|
|
Use: "reports",
|
|
Short: "Generate a report",
|
|
}
|
|
nodeUsageCmd = &cobra.Command{
|
|
Use: "storagenode-usage [start] [end]",
|
|
Short: "Generate a node usage report for a given period to use for payments",
|
|
Long: "Generate a node usage report for a given period to use for payments. Format dates using YYYY-MM-DD. The end date is exclusive.",
|
|
Args: cobra.MinimumNArgs(2),
|
|
RunE: cmdNodeUsage,
|
|
}
|
|
partnerAttributionCmd = &cobra.Command{
|
|
Use: "partner-attribution [partner ID] [start] [end]",
|
|
Short: "Generate a partner attribution report for a given period to use for payments",
|
|
Long: "Generate a partner attribution report for a given period to use for payments. Format dates using YYYY-MM-DD. The end date is exclusive.",
|
|
Args: cobra.MinimumNArgs(3),
|
|
RunE: cmdValueAttribution,
|
|
}
|
|
gracefulExitCmd = &cobra.Command{
|
|
Use: "graceful-exit [start] [end]",
|
|
Short: "Generate a graceful exit report",
|
|
Long: "Generate a node usage report for a given period to use for payments. Format dates using YYYY-MM-DD. The end date is exclusive.",
|
|
Args: cobra.MinimumNArgs(2),
|
|
RunE: cmdGracefulExit,
|
|
}
|
|
|
|
verifyGracefulExitReceiptCmd = &cobra.Command{
|
|
Use: "verify-exit-receipt [storage node ID] [receipt]",
|
|
Short: "Verify a graceful exit receipt",
|
|
Long: "Verify a graceful exit receipt is valid.",
|
|
Args: cobra.MinimumNArgs(2),
|
|
RunE: cmdVerifyGracefulExitReceipt,
|
|
}
|
|
|
|
runCfg Satellite
|
|
setupCfg Satellite
|
|
|
|
qdiagCfg struct {
|
|
Database string `help:"satellite database connection string" releaseDefault:"postgres://" devDefault:"postgres://"`
|
|
QListLimit int `help:"maximum segments that can be requested" default:"1000"`
|
|
}
|
|
nodeUsageCfg struct {
|
|
Database string `help:"satellite database connection string" releaseDefault:"postgres://" devDefault:"postgres://"`
|
|
Output string `help:"destination of report output" default:""`
|
|
}
|
|
partnerAttribtionCfg struct {
|
|
Database string `help:"satellite database connection string" releaseDefault:"postgres://" devDefault:"postgres://"`
|
|
Output string `help:"destination of report output" default:""`
|
|
}
|
|
gracefulExitCfg struct {
|
|
Database string `help:"satellite database connection string" releaseDefault:"postgres://" devDefault:"postgres://"`
|
|
Output string `help:"destination of report output" default:""`
|
|
Completed bool `help:"whether to output (initiated and completed) or (initiated and not completed)" default:"false"`
|
|
}
|
|
verifyGracefulExitReceiptCfg struct {
|
|
}
|
|
confDir string
|
|
identityDir string
|
|
)
|
|
|
|
func init() {
|
|
defaultConfDir := fpath.ApplicationDir("storj", "satellite")
|
|
defaultIdentityDir := fpath.ApplicationDir("storj", "identity", "satellite")
|
|
cfgstruct.SetupFlag(zap.L(), rootCmd, &confDir, "config-dir", defaultConfDir, "main directory for satellite configuration")
|
|
cfgstruct.SetupFlag(zap.L(), rootCmd, &identityDir, "identity-dir", defaultIdentityDir, "main directory for satellite identity credentials")
|
|
defaults := cfgstruct.DefaultsFlag(rootCmd)
|
|
rootCmd.AddCommand(runCmd)
|
|
runCmd.AddCommand(runMigrationCmd)
|
|
runCmd.AddCommand(runAPICmd)
|
|
runCmd.AddCommand(runRepairerCmd)
|
|
rootCmd.AddCommand(setupCmd)
|
|
rootCmd.AddCommand(qdiagCmd)
|
|
rootCmd.AddCommand(reportsCmd)
|
|
reportsCmd.AddCommand(nodeUsageCmd)
|
|
reportsCmd.AddCommand(partnerAttributionCmd)
|
|
reportsCmd.AddCommand(gracefulExitCmd)
|
|
reportsCmd.AddCommand(verifyGracefulExitReceiptCmd)
|
|
process.Bind(runCmd, &runCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
|
|
process.Bind(runMigrationCmd, &runCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
|
|
process.Bind(runAPICmd, &runCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
|
|
process.Bind(runRepairerCmd, &runCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
|
|
process.Bind(setupCmd, &setupCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir), cfgstruct.SetupMode())
|
|
process.Bind(qdiagCmd, &qdiagCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
|
|
process.Bind(nodeUsageCmd, &nodeUsageCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
|
|
process.Bind(gracefulExitCmd, &gracefulExitCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
|
|
process.Bind(verifyGracefulExitReceiptCmd, &verifyGracefulExitReceiptCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
|
|
process.Bind(partnerAttributionCmd, &partnerAttribtionCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
|
|
}
|
|
|
|
func cmdRun(cmd *cobra.Command, args []string) (err error) {
|
|
// inert constructors only ====
|
|
|
|
ctx, _ := process.Ctx(cmd)
|
|
log := zap.L()
|
|
|
|
identity, err := runCfg.Identity.Load()
|
|
if err != nil {
|
|
zap.S().Fatal(err)
|
|
}
|
|
|
|
db, err := satellitedb.New(log.Named("db"), runCfg.Database)
|
|
if err != nil {
|
|
return errs.New("Error starting master database on satellite: %+v", err)
|
|
}
|
|
defer func() {
|
|
err = errs.Combine(err, db.Close())
|
|
}()
|
|
|
|
pointerDB, err := metainfo.NewStore(log.Named("pointerdb"), runCfg.Metainfo.DatabaseURL)
|
|
if err != nil {
|
|
return errs.New("Error creating revocation database: %+v", err)
|
|
}
|
|
defer func() {
|
|
err = errs.Combine(err, db.Close())
|
|
}()
|
|
|
|
revocationDB, err := revocation.NewDBFromCfg(runCfg.Server.Config)
|
|
if err != nil {
|
|
return errs.New("Error creating revocation database: %+v", err)
|
|
}
|
|
defer func() {
|
|
err = errs.Combine(err, revocationDB.Close())
|
|
}()
|
|
|
|
liveAccounting, err := live.NewCache(log.Named("live-accounting"), runCfg.LiveAccounting)
|
|
if err != nil {
|
|
return errs.New("Error creating live accounting cache: %+v", err)
|
|
}
|
|
defer func() {
|
|
err = errs.Combine(err, liveAccounting.Close())
|
|
}()
|
|
|
|
peer, err := satellite.New(log, identity, db, pointerDB, revocationDB, liveAccounting, version.Build, &runCfg.Config)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// okay, start doing stuff ====
|
|
|
|
err = peer.Version.CheckVersion(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := process.InitMetricsWithCertPath(ctx, log, nil, runCfg.Identity.CertPath); err != nil {
|
|
zap.S().Warn("Failed to initialize telemetry batcher: ", err)
|
|
}
|
|
|
|
err = db.CheckVersion()
|
|
if err != nil {
|
|
zap.S().Fatal("failed satellite database version check: ", err)
|
|
return errs.New("Error checking version for satellitedb: %+v", err)
|
|
}
|
|
|
|
runError := peer.Run(ctx)
|
|
closeError := peer.Close()
|
|
return errs.Combine(runError, closeError)
|
|
}
|
|
|
|
func cmdMigrationRun(cmd *cobra.Command, args []string) (err error) {
|
|
log := zap.L()
|
|
db, err := satellitedb.New(log.Named("migration"), runCfg.Database)
|
|
if err != nil {
|
|
return errs.New("Error creating new master database connection for satellitedb migration: %+v", err)
|
|
}
|
|
defer func() {
|
|
err = errs.Combine(err, db.Close())
|
|
}()
|
|
|
|
err = db.CreateTables()
|
|
if err != nil {
|
|
return errs.New("Error creating tables for master database on satellite: %+v", err)
|
|
}
|
|
|
|
// There should be an explicit CreateTables call for the pointerdb as well.
|
|
// This is tracked in jira ticket #3337.
|
|
pdb, err := metainfo.NewStore(log.Named("migration"), runCfg.Metainfo.DatabaseURL)
|
|
if err != nil {
|
|
return errs.New("Error creating tables for pointer database on satellite: %+v", err)
|
|
}
|
|
defer func() {
|
|
err = errs.Combine(err, pdb.Close())
|
|
}()
|
|
|
|
return nil
|
|
}
|
|
|
|
func cmdSetup(cmd *cobra.Command, args []string) (err error) {
|
|
setupDir, err := filepath.Abs(confDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
valid, _ := fpath.IsValidSetupDir(setupDir)
|
|
if !valid {
|
|
return fmt.Errorf("satellite configuration already exists (%v)", setupDir)
|
|
}
|
|
|
|
err = os.MkdirAll(setupDir, 0700)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return process.SaveConfig(cmd, filepath.Join(setupDir, "config.yaml"))
|
|
}
|
|
|
|
func cmdQDiag(cmd *cobra.Command, args []string) (err error) {
|
|
|
|
// open the master db
|
|
database, err := satellitedb.New(zap.L().Named("db"), qdiagCfg.Database)
|
|
if err != nil {
|
|
return errs.New("error connecting to master database on satellite: %+v", err)
|
|
}
|
|
defer func() {
|
|
err := database.Close()
|
|
if err != nil {
|
|
fmt.Printf("error closing connection to master database on satellite: %+v\n", err)
|
|
}
|
|
}()
|
|
|
|
list, err := database.RepairQueue().SelectN(context.Background(), qdiagCfg.QListLimit)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// initialize the table header (fields)
|
|
const padding = 3
|
|
w := tabwriter.NewWriter(os.Stdout, 0, 0, padding, ' ', tabwriter.AlignRight|tabwriter.Debug)
|
|
fmt.Fprintln(w, "Path\tLost Pieces\t")
|
|
|
|
// populate the row fields
|
|
for _, v := range list {
|
|
fmt.Fprint(w, v.GetPath(), "\t", v.GetLostPieces(), "\t")
|
|
}
|
|
|
|
// display the data
|
|
return w.Flush()
|
|
}
|
|
|
|
func cmdVerifyGracefulExitReceipt(cmd *cobra.Command, args []string) (err error) {
|
|
ctx, _ := process.Ctx(cmd)
|
|
|
|
identity, err := runCfg.Identity.Load()
|
|
if err != nil {
|
|
zap.S().Fatal(err)
|
|
}
|
|
|
|
// Check the node ID is valid
|
|
nodeID, err := storj.NodeIDFromString(args[0])
|
|
if err != nil {
|
|
return errs.Combine(err, errs.New("Invalid node ID."))
|
|
}
|
|
|
|
return verifyGracefulExitReceipt(ctx, identity, nodeID, args[1])
|
|
}
|
|
|
|
func cmdGracefulExit(cmd *cobra.Command, args []string) (err error) {
|
|
ctx, _ := process.Ctx(cmd)
|
|
|
|
start, end, err := reports.ParseRange(args[0], args[1])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// send output to stdout
|
|
if gracefulExitCfg.Output == "" {
|
|
return generateGracefulExitCSV(ctx, gracefulExitCfg.Completed, start, end, os.Stdout)
|
|
}
|
|
|
|
// send output to file
|
|
file, err := os.Create(gracefulExitCfg.Output)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer func() {
|
|
err = errs.Combine(err, file.Close())
|
|
}()
|
|
|
|
return generateGracefulExitCSV(ctx, gracefulExitCfg.Completed, start, end, file)
|
|
}
|
|
|
|
func cmdNodeUsage(cmd *cobra.Command, args []string) (err error) {
|
|
ctx, _ := process.Ctx(cmd)
|
|
|
|
start, end, err := reports.ParseRange(args[0], args[1])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// send output to stdout
|
|
if nodeUsageCfg.Output == "" {
|
|
return generateNodeUsageCSV(ctx, start, end, os.Stdout)
|
|
}
|
|
|
|
// send output to file
|
|
file, err := os.Create(nodeUsageCfg.Output)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer func() {
|
|
err = errs.Combine(err, file.Close())
|
|
}()
|
|
|
|
return generateNodeUsageCSV(ctx, start, end, file)
|
|
}
|
|
|
|
func cmdValueAttribution(cmd *cobra.Command, args []string) (err error) {
|
|
ctx, _ := process.Ctx(cmd)
|
|
log := zap.L().Named("satellite-cli")
|
|
// Parse the UUID
|
|
partnerID, err := uuid.Parse(args[0])
|
|
if err != nil {
|
|
return errs.Combine(errs.New("Invalid Partner ID format. %s", args[0]), err)
|
|
}
|
|
|
|
start, end, err := reports.ParseRange(args[1], args[2])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// send output to stdout
|
|
if partnerAttribtionCfg.Output == "" {
|
|
return reports.GenerateAttributionCSV(ctx, partnerAttribtionCfg.Database, *partnerID, start, end, os.Stdout)
|
|
}
|
|
|
|
// send output to file
|
|
file, err := os.Create(partnerAttribtionCfg.Output)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer func() {
|
|
err = errs.Combine(err, file.Close())
|
|
if err != nil {
|
|
log.Sugar().Errorf("error closing the file %v after retrieving partner value attribution data: %+v", partnerAttribtionCfg.Output, err)
|
|
}
|
|
}()
|
|
|
|
return reports.GenerateAttributionCSV(ctx, partnerAttribtionCfg.Database, *partnerID, start, end, file)
|
|
}
|
|
|
|
func main() {
|
|
process.Exec(rootCmd)
|
|
}
|