diff --git a/.gitignore b/.gitignore index c08f3dabd..fbc46df64 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,4 @@ protos/google/* satellite_* storagenode_* uplink_* +*.svg \ No newline at end of file diff --git a/cmd/s3-benchmark/main.go b/cmd/s3-benchmark/main.go new file mode 100644 index 000000000..25584114d --- /dev/null +++ b/cmd/s3-benchmark/main.go @@ -0,0 +1,298 @@ +package main + +import ( + "bytes" + "errors" + "flag" + "fmt" + "image/color" + "io" + "io/ioutil" + "log" + "math/rand" + "os" + "text/tabwriter" + "time" + + "github.com/loov/hrtime" + "github.com/loov/plot" + minio "github.com/minio/minio-go" +) + +func main() { + endpoint := flag.String("endpoint", "127.0.0.1:7777", "endpoint address") + accesskey := flag.String("accesskey", "insecure-dev-access-key", "access key") + secretkey := flag.String("secretkey", "insecure-dev-secret-key", "secret key") + useSSL := flag.Bool("use-ssl", true, "use ssl") + location := flag.String("location", "", "bucket location") + count := flag.Int("count", 50, "run each benchmark n times") + plotname := flag.String("plot", "", "plot results") + + sizes := &Sizes{ + Default: []Size{{1 << 10}, {256 << 10}, {1 << 20}, {32 << 20}, {63 << 20}}, + } + flag.Var(sizes, "size", "sizes to test with") + + flag.Parse() + + client, err := minio.New(*endpoint, *accesskey, *secretkey, *useSSL) + if err != nil { + log.Fatal(err) + } + + bucket := time.Now().Format("bucket-2006-01-02-150405") + log.Println("Creating bucket", bucket) + err = client.MakeBucket(bucket, *location) + if err != nil { + log.Fatal("failed to create bucket: ", bucket, ": ", err) + } + + defer func() { + log.Println("Removing bucket") + err := client.RemoveBucket(bucket) + if err != nil { + log.Fatal("failed to remove bucket: ", bucket) + } + }() + + measurements := []Measurement{} + for _, size := range sizes.Sizes() { + measurement, err := Benchmark(client, bucket, size, *count) + if err != nil { + log.Fatal(err) + } + measurements = append(measurements, measurement) + } + + fmt.Print("\n\n") + w := tabwriter.NewWriter(os.Stdout, 0, 0, 4, ' ', 0) + fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", + "Size", "", + "Avg", "", + "Max", "", + "P50", "", "P90", "", "P99", "", + ) + fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", + "", "", + "s", "MB/s", + "s", "MB/s", + "s", "MB/s", "s", "MB/s", "s", "MB/s", + ) + for _, m := range measurements { + m.PrintStats(w) + } + _ = w.Flush() + + if *plotname != "" { + p := plot.New() + p.X.Min = 0 + p.X.Max = 10 + p.X.MajorTicks = 10 + p.X.MinorTicks = 10 + + speed := plot.NewAxisGroup() + speed.Y.Min = 0 + speed.Y.Max = 1 + speed.X.Min = 0 + speed.X.Max = 30 + speed.X.MajorTicks = 10 + speed.X.MinorTicks = 10 + + rows := plot.NewVStack() + rows.Margin = plot.R(5, 5, 5, 5) + p.Add(rows) + + for _, m := range measurements { + row := plot.NewHFlex() + rows.Add(row) + row.Add(35, plot.NewTextbox(m.Size.String())) + + plots := plot.NewVStack() + row.Add(0, plots) + + { // time plotting + uploadTime := plot.NewDensity("s", asSeconds(m.Upload)) + uploadTime.Stroke = color.NRGBA{0, 200, 0, 255} + downloadTime := plot.NewDensity("s", asSeconds(m.Download)) + downloadTime.Stroke = color.NRGBA{0, 0, 200, 255} + deleteTime := plot.NewDensity("s", asSeconds(m.Delete)) + deleteTime.Stroke = color.NRGBA{200, 0, 0, 255} + + flexTime := plot.NewHFlex() + plots.Add(flexTime) + flexTime.Add(70, plot.NewTextbox("time (s)")) + flexTime.AddGroup(0, + plot.NewGrid(), + uploadTime, + downloadTime, + deleteTime, + plot.NewTickLabels(), + ) + } + + { // speed plotting + uploadSpeed := plot.NewDensity("MB/s", asSpeed(m.Upload, m.Size.bytes)) + uploadSpeed.Stroke = color.NRGBA{0, 200, 0, 255} + downloadSpeed := plot.NewDensity("MB/s", asSpeed(m.Download, m.Size.bytes)) + downloadSpeed.Stroke = color.NRGBA{0, 0, 200, 255} + + flexSpeed := plot.NewHFlex() + plots.Add(flexSpeed) + + speedGroup := plot.NewAxisGroup() + speedGroup.X, speedGroup.Y = speed.X, speed.Y + speedGroup.AddGroup( + plot.NewGrid(), + uploadSpeed, + downloadSpeed, + plot.NewTickLabels(), + ) + + flexSpeed.Add(70, plot.NewTextbox("speed (MB/s)")) + flexSpeed.AddGroup(0, speedGroup) + } + } + + svgcanvas := plot.NewSVG(1500, 150*float64(len(measurements))) + p.Draw(svgcanvas) + + err := ioutil.WriteFile(*plotname, svgcanvas.Bytes(), 0755) + if err != nil { + log.Fatal(err) + } + } +} + +func asSeconds(durations []time.Duration) []float64 { + xs := make([]float64, 0, len(durations)) + for _, dur := range durations { + xs = append(xs, dur.Seconds()) + } + return xs +} + +func asSpeed(durations []time.Duration, size int64) []float64 { + const MB = 1 << 20 + xs := make([]float64, 0, len(durations)) + for _, dur := range durations { + xs = append(xs, (float64(size)/MB)/dur.Seconds()) + } + return xs +} + +// Measurement contains measurements for different requests +type Measurement struct { + Size Size + Upload []time.Duration + Download []time.Duration + Delete []time.Duration +} + +// PrintStats prints important valueas about the measurement +func (m *Measurement) PrintStats(w io.Writer) { + const binCount = 10 + + upload := hrtime.NewDurationHistogram(m.Upload, binCount) + download := hrtime.NewDurationHistogram(m.Download, binCount) + delete := hrtime.NewDurationHistogram(m.Delete, binCount) + + hists := []struct { + L string + H *hrtime.Histogram + }{ + {"Upload", upload}, + {"Download", download}, + {"Delete", delete}, + } + + sec := func(ns float64) string { + return fmt.Sprintf("%.2f", ns/1e9) + } + speed := func(ns float64) string { + return fmt.Sprintf("%.2f", (float64(m.Size.bytes)/(1<<20))/(ns/1e9)) + } + + for _, hist := range hists { + if hist.L == "Delete" { + fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", + m.Size, hist.L, + sec(hist.H.Average), "", + sec(hist.H.Maximum), "", + sec(hist.H.P50), "", + sec(hist.H.P90), "", + sec(hist.H.P99), "", + ) + continue + } + fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", + m.Size, hist.L, + sec(hist.H.Average), speed(hist.H.Average), + sec(hist.H.Maximum), speed(hist.H.Maximum), + sec(hist.H.P50), speed(hist.H.P50), + sec(hist.H.P90), speed(hist.H.P90), + sec(hist.H.P99), speed(hist.H.P99), + ) + } +} + +// Benchmark runs benchmarks on bucket with given size +func Benchmark(client *minio.Client, bucket string, size Size, count int) (Measurement, error) { + log.Print("Benchmarking size ", size.String(), " ") + + data := make([]byte, size.bytes) + result := make([]byte, size.bytes) + + defer fmt.Println() + + measurement := Measurement{} + measurement.Size = size + for k := 0; k < count; k++ { + fmt.Print(".") + + rand.Read(data[:]) + { // uploading + start := hrtime.Now() + _, err := client.PutObject(bucket, "data", bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ + ContentType: "application/octet-stream", + }) + finish := hrtime.Now() + if err != nil { + return measurement, fmt.Errorf("upload failed: %v", err) + } + measurement.Upload = append(measurement.Upload, (finish - start)) + } + + { // downloading + start := hrtime.Now() + reader, err := client.GetObject(bucket, "data", minio.GetObjectOptions{}) + if err != nil { + return measurement, fmt.Errorf("get object failed: %v", err) + } + + var n int + n, err = reader.Read(result) + if err != nil && err != io.EOF { + return measurement, fmt.Errorf("download failed: %v", err) + } + finish := hrtime.Now() + + if !bytes.Equal(data, result[:n]) { + return measurement, errors.New("upload/download do not match") + } + + measurement.Download = append(measurement.Download, (finish - start)) + } + + { // deleting + start := hrtime.Now() + err := client.RemoveObject(bucket, "data") + if err != nil { + return measurement, fmt.Errorf("delete failed: %v", err) + } + finish := hrtime.Now() + measurement.Delete = append(measurement.Delete, (finish - start)) + } + } + + return measurement, nil +} diff --git a/cmd/s3-benchmark/size.go b/cmd/s3-benchmark/size.go new file mode 100644 index 000000000..db1e1ee57 --- /dev/null +++ b/cmd/s3-benchmark/size.go @@ -0,0 +1,105 @@ +package main + +import ( + "errors" + "strconv" + "strings" +) + +// Sizes implements flag.Value for collecting byte counts +type Sizes struct { + Default []Size + Custom []Size +} + +// Sizes returns the loaded values +func (sizes Sizes) Sizes() []Size { + if len(sizes.Custom) > 0 { + return sizes.Custom + } + return sizes.Default +} + +// String converts values to a string +func (sizes Sizes) String() string { + sz := sizes.Sizes() + xs := make([]string, len(sz)) + for i, size := range sz { + xs[i] = size.String() + } + return strings.Join(xs, " ") +} + +// Set adds values from byte values +func (sizes *Sizes) Set(s string) error { + for _, x := range strings.Fields(s) { + var size Size + if err := size.Set(x); err != nil { + return err + } + sizes.Custom = append(sizes.Custom, size) + } + return nil +} + +// Size represents a value of bytes +type Size struct { + bytes int64 +} + +type unit struct { + suffix string + scale float64 +} + +var units = []unit{ + {"T", 1 << 40}, + {"G", 1 << 30}, + {"M", 1 << 20}, + {"K", 1 << 10}, + {"B", 1}, + {"", 0}, +} + +// String converts size to a string +func (size Size) String() string { + if size.bytes <= 0 { + return "0" + } + + v := float64(size.bytes) + for _, unit := range units { + if v >= unit.scale { + r := strconv.FormatFloat(v/unit.scale, 'f', 1, 64) + r = strings.TrimSuffix(r, "0") + r = strings.TrimSuffix(r, ".") + return r + unit.suffix + } + } + return strconv.Itoa(int(size.bytes)) + "B" +} + +// Set updates value from string +func (size *Size) Set(s string) error { + if s == "" { + return errors.New("empty size") + } + + value, suffix := s[:len(s)-1], s[len(s)-1] + if '0' <= suffix && suffix <= '9' { + suffix = 'B' + value = s + } + + for _, unit := range units { + if unit.suffix == string(suffix) { + v, err := strconv.ParseFloat(value, 64) + if err != nil { + return err + } + size.bytes = int64(v * unit.scale) + return nil + } + } + return errors.New("unknown suffix " + string(suffix)) +} diff --git a/go.mod b/go.mod index cbc5313c5..e5bcbd458 100644 --- a/go.mod +++ b/go.mod @@ -73,6 +73,8 @@ require ( github.com/klauspost/reedsolomon v0.0.0-20180704173009-925cb01d6510 // indirect github.com/kurin/blazer v0.5.1 // indirect github.com/lib/pq v0.0.0-20180523175426-90697d60dd84 // indirect + github.com/loov/hrtime v0.0.0-20180911122900-a9e82bc6c180 + github.com/loov/plot v0.0.0-20180510142208-e59891ae1271 github.com/magiconair/properties v1.7.6 // indirect github.com/mailru/easyjson v0.0.0-20180730094502-03f2033d19d5 // indirect github.com/marstr/guid v1.1.0 // indirect @@ -88,7 +90,7 @@ require ( github.com/minio/lsync v0.0.0-20180328070428-f332c3883f63 // indirect github.com/minio/mc v0.0.0-20180820172331-a1110bc0223c // indirect github.com/minio/minio v0.0.0-20180508161510-54cd29b51c38 - github.com/minio/minio-go v6.0.3-0.20180613230128-10531abd0af1+incompatible // indirect + github.com/minio/minio-go v6.0.6+incompatible github.com/minio/sha256-simd v0.0.0-20171213220625-ad98a36ba0da // indirect github.com/minio/sio v0.0.0-20180327104954-6a41828a60f0 // indirect github.com/mitchellh/go-homedir v0.0.0-20180801233206-58046073cbff // indirect diff --git a/go.sum b/go.sum index fa5c9d4b3..8019b406b 100644 --- a/go.sum +++ b/go.sum @@ -170,6 +170,12 @@ github.com/kurin/blazer v0.5.1 h1:mBc4i1uhHJEqU0KvzOgpMHhkwf+EcXvxjWEUS7HG+eY= github.com/kurin/blazer v0.5.1/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU= github.com/lib/pq v0.0.0-20180523175426-90697d60dd84 h1:it29sI2IM490luSc3RAhp5WuCYnc6RtbfLVAB7nmC5M= github.com/lib/pq v0.0.0-20180523175426-90697d60dd84/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/loov/hrtime v0.0.0-20180518045357-97b83e345e8d h1:uULUNPSLCzWImBtst3qmWBVa40z6/LjU9MyQhnVrxW4= +github.com/loov/hrtime v0.0.0-20180518045357-97b83e345e8d/go.mod h1:2871C3urfEJnq/bpTYjFdMOdgxVd8otLLEL6vMNy/Iw= +github.com/loov/hrtime v0.0.0-20180911122900-a9e82bc6c180 h1:kLwg5eA/kaWQ/RwANTH7Gg+VdxmdjbcSWyaS/1VQGkA= +github.com/loov/hrtime v0.0.0-20180911122900-a9e82bc6c180/go.mod h1:2871C3urfEJnq/bpTYjFdMOdgxVd8otLLEL6vMNy/Iw= +github.com/loov/plot v0.0.0-20180510142208-e59891ae1271 h1:51ToN6N0TDtCruf681gufYuEhO9qFHQzM3RFTS/n6XE= +github.com/loov/plot v0.0.0-20180510142208-e59891ae1271/go.mod h1:3yy5HBPbe5e1UmEffbO0n0g6A8h6ChHaCTeundr6H60= github.com/magiconair/properties v1.7.6 h1:U+1DqNen04MdEPgFiIwdOUiqZ8qPa37xgogX/sd3+54= github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180730094502-03f2033d19d5 h1:0x4qcEHDpruK6ML/m/YSlFUUu0UpRD3I2PHsNCuGnyA= @@ -210,6 +216,8 @@ github.com/minio/minio-go v6.0.3-0.20180613230128-10531abd0af1+incompatible h1:N github.com/minio/minio-go v6.0.3-0.20180613230128-10531abd0af1+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= github.com/minio/minio-go v6.0.5+incompatible h1:qxQQW40lV2vuE9i6yYmt90GSJlT1YrMenWrjM6nZh0Q= github.com/minio/minio-go v6.0.5+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= +github.com/minio/minio-go v6.0.6+incompatible h1:AOPYom8W/kjdsjlsCVYwfb5BELGmkMP7EXhocAm5iME= +github.com/minio/minio-go v6.0.6+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= github.com/minio/sha256-simd v0.0.0-20171213220625-ad98a36ba0da h1:tazA5y1hWYJO8VSYbU36yBhXeIvruLXMUKu6WBtcJck= github.com/minio/sha256-simd v0.0.0-20171213220625-ad98a36ba0da/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sio v0.0.0-20180327104954-6a41828a60f0 h1:ys4bbOlPvaUBlA0byjm6TqydsXZu614ZIUTfF+4MRY0=