JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cmd/storj-sim: fix prefix writer infinite loop bug

Browse Source 
Fix case where line is larger than maxline and writer ended up in an
infinite loop trying to buffer more data.

Change-Id: I243da738b331279d6bf27255778b5798e7f37f95
This commit is contained in:
paul cannon 2020-09-10 08:37:16 -05:00 committed by Egon Elbre
parent 789b07e226
commit 5b9fd4e50a
3 changed files with 99 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
cmd/storj-sim/prefix.go
View File

@ -15,13 +15,18 @@ import (
type PrefixWriter struct {
root *prefixWriter
maxline int
nowFunc func() time.Time
mu sync.Mutex
prefixlen int
dst io.Writer
}
const maxIDLength = 10
const (
maxIDLength = 10
timeFormat = "15:04:05.000"
emptyTimeField = " "
)
func max(a, b int) int {
if a > b {
@ -31,10 +36,11 @@ func max(a, b int) int {
}
// NewPrefixWriter creates a writer than can prefix all lines written to it.
func NewPrefixWriter(defaultPrefix string, dst io.Writer) *PrefixWriter {
func NewPrefixWriter(defaultPrefix string, maxLineLen int, dst io.Writer) *PrefixWriter {
writer := &PrefixWriter{
maxline: 10000, // disable maxline cutting
maxline: maxLineLen,
dst: dst,
nowFunc: time.Now,
}
writer.root = writer.Prefixed(defaultPrefix).(*prefixWriter)
return writer
@ -115,24 +121,30 @@ func (writer *prefixWriter) Write(data []byte) (int, error) {
prefix := writer.prefix
id := writer.id
timeText := time.Now().Format("15:04:05.000")
timeText := writer.nowFunc().Format(timeFormat)
for len(buffer) > 0 {
pos := bytes.IndexByte(buffer, '\n') + 1
breakline := false
if pos <= 0 {
pos := bytes.IndexByte(buffer, '\n')
insertbreak := false
// did not find a linebreak
if pos < 0 {
// wait for more data, if we haven't reached maxline
if len(buffer) < writer.maxline {
return len(data), nil
}
}
// try to find a nice place where to break the line
if pos < 0 || pos > writer.maxline {
pos = writer.maxline
pos = writer.maxline - 1
for p := pos; p >= writer.maxline*2/3; p-- {
// is there a space we can break on?
if buffer[p] == ' ' {
pos = p
break
}
}
breakline = true
insertbreak = true
}
_, err := fmt.Fprintf(writer.dst, "%-*s %-*s %s | ", writer.prefixlen, prefix, maxIDLength, id, timeText)
@ -142,21 +154,22 @@ func (writer *prefixWriter) Write(data []byte) (int, error) {
_, err = writer.dst.Write(buffer[:pos])
buffer = buffer[pos:]
if err != nil {
return len(data), err
}
_, err = writer.dst.Write([]byte{'\n'})
if err != nil {
return len(data), err
}
if breakline {
_, err = writer.dst.Write([]byte{'\n'})
if err != nil {
return len(data), err
}
// remove the linebreak from buffer, if it's not an insert
if !insertbreak && len(buffer) > 0 {
buffer = buffer[1:]
}
prefix = ""
id = ""
timeText = " "
timeText = emptyTimeField
}
return len(data), nil

68
cmd/storj-sim/prefix_test.go
View File

@ -4,15 +4,18 @@
package main
import (
"bytes"
"io/ioutil"
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
)
func TestPrefixWriter(t *testing.T) {
root := NewPrefixWriter("", ioutil.Discard)
root := NewPrefixWriter("", storjSimMaxLineLen, ioutil.Discard)
alpha := root.Prefixed("alpha")
beta := root.Prefixed("beta")
@ -34,3 +37,66 @@ func TestPrefixWriter(t *testing.T) {
return err
})
}
func TestPrefixWriterWithLongLine(t *testing.T) {
const (
maxLineLen = 46
inputString = "It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity,"
prefix = "dickens"
nodeID = "TWO-CITIES"
)
dst := &bytes.Buffer{}
w := NewPrefixWriter(prefix, maxLineLen, dst)
w.nowFunc = func() time.Time { return time.Unix(1599750982, 123456789).UTC() }
_, err := w.Write([]byte("Hi. Node " + nodeID + " started\n"))
require.NoError(t, err)
n, err := w.Write([]byte(inputString))
require.NoError(t, err)
require.Equal(t, len(inputString), n)
// then write the newline separately
n, err = w.Write([]byte("\n"))
require.NoError(t, err)
require.Equal(t, 1, n)
expected := `
dickens TWO-CITIES 15:16:22.123 | Hi. Node TWO-CITIES started
dickens TWO-CITIES 15:16:22.123 | It was the best of times, it was the worst of
| times, it was the age of wisdom, it was the
| age of foolishness, it was the epoch of
dickens TWO-CITIES 15:16:22.123 | belief, it was the epoch of incredulity,
`
require.Equal(t, strings.TrimLeft(expected, "\n"), dst.String())
}
func TestPrefixWriterWithLongLineAndNewline(t *testing.T) {
const (
maxLineLen = 46
inputString = "It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity,\n"
prefix = "dickens"
nodeID = "TWO-CITIES"
)
dst := &bytes.Buffer{}
w := NewPrefixWriter(prefix, maxLineLen, dst)
w.nowFunc = func() time.Time { return time.Unix(1599750982, 123456789).UTC() }
_, err := w.Write([]byte("Hi. Node " + nodeID + " started\n"))
require.NoError(t, err)
n, err := w.Write([]byte(inputString))
require.NoError(t, err)
require.Equal(t, len(inputString), n)
expected := `
dickens TWO-CITIES 15:16:22.123 | Hi. Node TWO-CITIES started
dickens TWO-CITIES 15:16:22.123 | It was the best of times, it was the worst of
| times, it was the age of wisdom, it was the
| age of foolishness, it was the epoch of
| belief, it was the epoch of incredulity,
`
require.Equal(t, strings.TrimLeft(expected, "\n"), dst.String())
}

4
cmd/storj-sim/process.go
View File

@ -32,10 +32,12 @@ type Processes struct {
MaxStartupWait time.Duration
}
const storjSimMaxLineLen = 10000
// NewProcesses returns a group of processes.
func NewProcesses(dir string) *Processes {
return &Processes{
Output: NewPrefixWriter("sim", os.Stdout),
Output: NewPrefixWriter("sim", storjSimMaxLineLen, os.Stdout),
Directory: dir,
List: nil,
MaxStartupWait: time.Minute,