From 1966340b2ad847a318fcfe53f35e3f312c2ba549 Mon Sep 17 00:00:00 2001 From: JT Olio Date: Fri, 2 Jun 2023 10:17:23 -0400 Subject: [PATCH] storagenode: report fastopen support Change-Id: Ic707bce32729540391d5139e7f3a82c693bcb834 --- private/server/fastopen_linux.go | 10 +++++++++- private/server/fastopen_other.go | 3 ++- private/server/fastopen_windows.go | 4 +++- private/server/server.go | 25 +++++++++++++++++-------- storagenode/contact/service.go | 8 ++++++-- storagenode/peer.go | 1 + 6 files changed, 38 insertions(+), 13 deletions(-) diff --git a/private/server/fastopen_linux.go b/private/server/fastopen_linux.go index 252e6583c..92458981e 100644 --- a/private/server/fastopen_linux.go +++ b/private/server/fastopen_linux.go @@ -22,21 +22,26 @@ func setTCPFastOpen(fd uintptr, queue int) error { } var tryInitFastOpenOnce sync.Once +var initFastOpenPossiblyEnabled bool -func tryInitFastOpen(log *zap.Logger) { +// tryInitFastOpen returns true if fastopen support is possibly enabled. +func tryInitFastOpen(log *zap.Logger) bool { tryInitFastOpenOnce.Do(func() { data, err := os.ReadFile(tcpFastOpenSysctlPath) if err != nil { log.Sugar().Infof("kernel support for tcp fast open unknown") + initFastOpenPossiblyEnabled = true return } fastOpenFlags, err := strconv.Atoi(strings.TrimSpace(string(data))) if err != nil { log.Sugar().Infof("kernel support for tcp fast open unparsable") + initFastOpenPossiblyEnabled = true return } if fastOpenFlags&0x2 != 0 { log.Sugar().Infof("existing kernel support for server-side tcp fast open detected") + initFastOpenPossiblyEnabled = true return } err = os.WriteFile(tcpFastOpenSysctlPath, []byte(fmt.Sprint(fastOpenFlags|0x2)), 0o644) @@ -48,8 +53,11 @@ func tryInitFastOpen(log *zap.Logger) { // enable standard fast open with standard cookies for both clients and // servers, so it's probably the right advice. log.Sugar().Infof("enable with: sysctl -w net.ipv4.tcp_fastopen=3") + initFastOpenPossiblyEnabled = false return } log.Sugar().Infof("kernel support for server-side tcp fast open enabled.") + initFastOpenPossiblyEnabled = true }) + return initFastOpenPossiblyEnabled } diff --git a/private/server/fastopen_other.go b/private/server/fastopen_other.go index 2a0c8f1f2..edeb5af07 100644 --- a/private/server/fastopen_other.go +++ b/private/server/fastopen_other.go @@ -12,4 +12,5 @@ import ( func setTCPFastOpen(fd uintptr, queue int) error { return nil } -func tryInitFastOpen(*zap.Logger) {} +// tryInitFastOpen returns true if fastopen support is possibly enabled. +func tryInitFastOpen(*zap.Logger) bool { return false } diff --git a/private/server/fastopen_windows.go b/private/server/fastopen_windows.go index 988864d6c..34cc27b3d 100644 --- a/private/server/fastopen_windows.go +++ b/private/server/fastopen_windows.go @@ -15,9 +15,11 @@ func setTCPFastOpen(fd uintptr, queue int) error { return syscall.SetsockoptInt(syscall.Handle(fd), syscall.IPPROTO_TCP, tcpFastOpenServer, 1) } -func tryInitFastOpen(*zap.Logger) { +// tryInitFastOpen returns true if fastopen support is possibly enabled. +func tryInitFastOpen(*zap.Logger) bool { // should we log or check something along the lines of // netsh int tcp set global fastopen=enabled // netsh int tcp set global fastopenfallback=disabled // ? + return false } diff --git a/private/server/server.go b/private/server/server.go index 4c4650bb7..468422f70 100644 --- a/private/server/server.go +++ b/private/server/server.go @@ -72,6 +72,7 @@ type Server struct { tlsOptions *tlsopts.Options noiseConf noise.Config config Config + fastOpen bool publicTCPListener net.Listener publicUDPConn *net.UDPConn @@ -115,14 +116,16 @@ func New(log *zap.Logger, tlsOptions *tlsopts.Options, config Config) (_ *Server listenConfig := net.ListenConfig{} if config.TCPFastOpen { - tryInitFastOpen(log) - listenConfig.Control = func(network, address string, c syscall.RawConn) error { - return c.Control(func(fd uintptr) { - err := setTCPFastOpen(fd, config.TCPFastOpenQueue) - if err != nil { - log.Sugar().Infof("failed to set tcp fast open for this socket: %v", err) - } - }) + server.fastOpen = tryInitFastOpen(log) + if server.fastOpen { + listenConfig.Control = func(network, address string, c syscall.RawConn) error { + return c.Control(func(fd uintptr) { + err := setTCPFastOpen(fd, config.TCPFastOpenQueue) + if err != nil { + log.Sugar().Infof("failed to set tcp fast open for this socket: %v", err) + } + }) + } } } @@ -232,6 +235,12 @@ func (p *Server) DebounceLimit() int { return debounceLimit } +// FastOpen returns true if FastOpen is possibly open. false means we +// know FastOpen is off. +func (p *Server) FastOpen() bool { + return p.fastOpen +} + // Close shuts down the server. func (p *Server) Close() error { p.mu.Lock() diff --git a/storagenode/contact/service.go b/storagenode/contact/service.go index 73bfbb871..44b3498f8 100644 --- a/storagenode/contact/service.go +++ b/storagenode/contact/service.go @@ -49,6 +49,7 @@ type NodeInfo struct { Operator pb.NodeOperator NoiseKeyAttestation *pb.NoiseKeyAttestation DebounceLimit int + FastOpen bool } // Service is the contact service between storage nodes and satellites. @@ -96,7 +97,6 @@ func (service *Service) pingSatellite(ctx context.Context, satellite storj.NodeI interval := initialBackOff attempts := 0 for { - mon.Meter("satellite_contact_request").Mark(1) //mon:locked err := service.pingSatelliteOnce(ctx, satellite) @@ -117,7 +117,6 @@ func (service *Service) pingSatellite(ctx context.Context, satellite storj.NodeI return nil } } - } func (service *Service) pingSatelliteOnce(ctx context.Context, id storj.NodeID) (err error) { @@ -130,6 +129,10 @@ func (service *Service) pingSatelliteOnce(ctx context.Context, id storj.NodeID) defer func() { err = errs.Combine(err, conn.Close()) }() self := service.Local() + var features uint64 + if self.FastOpen { + features |= uint64(pb.NodeAddress_TCP_FASTOPEN_ENABLED) + } resp, err := pb.NewDRPCNodeClient(conn).CheckIn(ctx, &pb.CheckInRequest{ Address: self.Address, Version: &self.Version, @@ -137,6 +140,7 @@ func (service *Service) pingSatelliteOnce(ctx context.Context, id storj.NodeID) Operator: &self.Operator, NoiseKeyAttestation: self.NoiseKeyAttestation, DebounceLimit: int32(self.DebounceLimit), + Features: features, }) service.quicStats.SetStatus(false) if err != nil { diff --git a/storagenode/peer.go b/storagenode/peer.go index 03c24049f..6e1147d34 100644 --- a/storagenode/peer.go +++ b/storagenode/peer.go @@ -441,6 +441,7 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB, revocationDB exten Version: *pbVersion, NoiseKeyAttestation: noiseKeyAttestation, DebounceLimit: peer.Server.DebounceLimit(), + FastOpen: peer.Server.FastOpen(), } peer.Contact.PingStats = new(contact.PingStats) peer.Contact.QUICStats = contact.NewQUICStats(peer.Server.IsQUICEnabled())