2eb660d4b7
* Moving retrieve into multiple goroutines * Make sure we pass nil errors into err channel * restore tests * incorporate locks in retrieve.go * deserialize data only if we have something to deserealize when receiving bandwidth allocation in server store * Adding logic for retrieve to be more efficient * Add channel? * hmm * implement Throttle concurrency primitive * using throttle * Remove unused variables * Egon comments addressed * Get ba total correct * Consume without waiting * incrementally increase signing size * Get downloads working with throttle * Removed logging * Make sure we handle errors properly * Fix tests > > Co-authored-by: Kaloyan <kaloyan@storj.io> * Can't Fatalf in goroutine * Add missing returns to tests * add capacity to channel, smarter allocations * rename things and don't use size as limit * replace things with sync2.Throttle * fix compilation errors * add note about security * fix ordering * Max length is actually 64 bytes for piece ID * Max length is actually 64 bytes for piece ID * fix limit * error comes from pending allocs, so no need to relog * Optimize throughput * TODO * Deleted allocation manager * Return when someone sends a smaller bandwidth allocation than the previous message * review comments
100 lines
1.9 KiB
Go
100 lines
1.9 KiB
Go
// Copyright (C) 2018 Storj Labs, Inc.
|
|
// See LICENSE for copying information
|
|
|
|
package sync2
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"math/rand"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func ExampleThrottle() {
|
|
throttle := NewThrottle()
|
|
var wg sync.WaitGroup
|
|
|
|
// consumer
|
|
go func() {
|
|
defer wg.Done()
|
|
totalConsumed := int64(0)
|
|
for {
|
|
available, err := throttle.ConsumeOrWait(8)
|
|
if err != nil {
|
|
return
|
|
}
|
|
fmt.Println("- consuming ", available, " total=", totalConsumed)
|
|
totalConsumed += available
|
|
|
|
// do work for available amount
|
|
time.Sleep(time.Duration(available) * time.Millisecond)
|
|
}
|
|
}()
|
|
|
|
// producer
|
|
go func() {
|
|
defer wg.Done()
|
|
|
|
step := int64(8)
|
|
for total := int64(64); total >= 0; total -= step {
|
|
err := throttle.ProduceAndWaitUntilBelow(step, step*3)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
fmt.Println("+ producing", step, " left=", total)
|
|
time.Sleep(time.Duration(rand.Intn(8)) * time.Millisecond)
|
|
}
|
|
|
|
throttle.Fail(io.EOF)
|
|
}()
|
|
|
|
wg.Wait()
|
|
|
|
fmt.Println("done", throttle.Err())
|
|
}
|
|
|
|
func TestThrottleBasic(t *testing.T) {
|
|
throttle := NewThrottle()
|
|
var stage int64
|
|
c := make(chan error, 1)
|
|
|
|
// consumer
|
|
go func() {
|
|
consume, _ := throttle.ConsumeOrWait(4)
|
|
if v := atomic.LoadInt64(&stage); v != 1 || consume != 4 {
|
|
c <- fmt.Errorf("did not block in time: %d / %d", v, consume)
|
|
return
|
|
}
|
|
|
|
consume, _ = throttle.ConsumeOrWait(4)
|
|
if v := atomic.LoadInt64(&stage); v != 1 || consume != 4 {
|
|
c <- fmt.Errorf("did not block in time: %d / %d", v, consume)
|
|
return
|
|
}
|
|
atomic.AddInt64(&stage, 2)
|
|
c <- nil
|
|
}()
|
|
|
|
// slowly produce
|
|
time.Sleep(time.Millisecond)
|
|
// set stage to 1
|
|
atomic.AddInt64(&stage, 1)
|
|
_ = throttle.Produce(8)
|
|
// wait until consumer consumes twice
|
|
_ = throttle.WaitUntilBelow(3)
|
|
// wait slightly for updating stage
|
|
time.Sleep(time.Millisecond)
|
|
|
|
if v := atomic.LoadInt64(&stage); v != 3 {
|
|
t.Fatalf("did not unblock in time: %d", v)
|
|
}
|
|
|
|
if err := <-c; err != nil {
|
|
t.Fatalf(err.Error())
|
|
}
|
|
}
|