2019-03-27 10:24:35 +00:00
|
|
|
// Copyright (C) 2019 Storj Labs, Inc.
|
|
|
|
|
// See LICENSE for copying information.
|
|
|
|
|
|
|
|
|
|
package orders_test
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"testing"
|
2019-04-01 21:14:58 +01:00
|
|
|
"time"
|
2019-03-27 10:24:35 +00:00
|
|
|
|
|
|
|
|
"github.com/stretchr/testify/require"
|
2019-04-22 10:07:50 +01:00
|
|
|
|
2019-12-27 11:48:47 +00:00
|
|
|
"storj.io/common/memory"
|
|
|
|
|
"storj.io/common/pb"
|
|
|
|
|
"storj.io/common/storj"
|
|
|
|
|
"storj.io/common/testcontext"
|
|
|
|
|
"storj.io/common/testrand"
|
2019-11-14 19:46:15 +00:00
|
|
|
"storj.io/storj/private/testplanet"
|
2019-08-15 20:05:43 +01:00
|
|
|
"storj.io/storj/satellite"
|
2020-06-11 19:31:45 +01:00
|
|
|
"storj.io/storj/satellite/satellitedb"
|
|
|
|
|
"storj.io/storj/satellite/satellitedb/dbx"
|
2019-08-15 20:05:43 +01:00
|
|
|
"storj.io/storj/satellite/satellitedb/satellitedbtest"
|
2019-03-27 10:24:35 +00:00
|
|
|
)
|
|
|
|
|
|
2019-04-01 21:14:58 +01:00
|
|
|
func TestSendingReceivingOrders(t *testing.T) {
|
|
|
|
|
// test happy path
|
|
|
|
|
testplanet.Run(t, testplanet.Config{
|
|
|
|
|
SatelliteCount: 1, StorageNodeCount: 6, UplinkCount: 1,
|
2020-01-21 10:38:41 +00:00
|
|
|
Reconfigure: testplanet.Reconfigure{
|
|
|
|
|
Satellite: testplanet.ReconfigureRS(2, 3, 4, 4),
|
|
|
|
|
},
|
2019-04-01 21:14:58 +01:00
|
|
|
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
storagenode: live tracking of order window usage
This change accomplishes multiple things:
1. Instead of having a max in flight time, which means
we effectively have a minimum bandwidth for uploads
and downloads, we keep track of what windows have
active requests happening in them.
2. We don't double check when we save the order to see if it
is too old: by then, it's too late. A malicious uplink
could just submit orders outside of the grace window and
receive all the data, but the node would just not commit
it, so the uplink gets free traffic. Because the endpoints
also check for the order being too old, this would be a
very tight race that depends on knowledge of the node system
clock, but best to not have the race exist. Instead, we piggy
back off of the in flight tracking and do the check when
we start to handle the order, and commit at the end.
3. Change the functions that send orders and list unsent
orders to accept a time at which that operation is
happening. This way, in tests, we can pretend we're
listing or sending far into the future after the windows
are available to send, rather than exposing test functions
to modify internal state about the grace period to get
the desired effect. This brings tests closer to actual
usage in production.
4. Change the calculation for if an order is allowed to be
enqueued due to the grace period to just look at the
order creation time, rather than some computation involving
the window it will be in. In this way, you can easily
answer the question of "will this order be accepted?" by
asking "is it older than X?" where X is the grace period.
5. Increases the frequency we check to send up orders to once
every 5 minutes instead of once every hour because we already
have hour-long buffering due to the windows. This decreases
the maximum latency that an order will be reported back to
the satellite by 55 minutes.
Change-Id: Ie08b90d139d45ee89b82347e191a2f8db1b88036
2020-08-12 20:01:43 +01:00
|
|
|
now := time.Now()
|
|
|
|
|
tomorrow := now.Add(24 * time.Hour)
|
|
|
|
|
|
2019-09-11 23:37:01 +01:00
|
|
|
planet.Satellites[0].Audit.Worker.Loop.Pause()
|
2019-04-01 21:14:58 +01:00
|
|
|
for _, storageNode := range planet.StorageNodes {
|
2019-08-22 15:33:14 +01:00
|
|
|
storageNode.Storage2.Orders.Sender.Pause()
|
2019-04-01 21:14:58 +01:00
|
|
|
}
|
2019-03-27 10:24:35 +00:00
|
|
|
|
2019-06-26 11:38:51 +01:00
|
|
|
expectedData := testrand.Bytes(50 * memory.KiB)
|
2019-04-01 21:14:58 +01:00
|
|
|
|
2020-01-21 10:38:41 +00:00
|
|
|
err := planet.Uplinks[0].Upload(ctx, planet.Satellites[0], "testbucket", "test/path", expectedData)
|
2019-04-01 21:14:58 +01:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
2020-11-18 13:38:03 +00:00
|
|
|
// Wait for storage nodes to propagate all information.
|
|
|
|
|
require.NoError(t, planet.WaitForStorageNodeEndpoints(ctx))
|
|
|
|
|
|
2019-04-01 21:14:58 +01:00
|
|
|
sumBeforeSend := 0
|
|
|
|
|
for _, storageNode := range planet.StorageNodes {
|
2020-07-01 23:05:01 +01:00
|
|
|
// change settle buffer so orders can be sent
|
2020-10-15 19:57:02 +01:00
|
|
|
unsentMap, err := storageNode.OrdersStore.ListUnsentBySatellite(ctx, tomorrow)
|
2019-04-01 21:14:58 +01:00
|
|
|
require.NoError(t, err)
|
2020-07-01 23:05:01 +01:00
|
|
|
for _, satUnsent := range unsentMap {
|
|
|
|
|
sumBeforeSend += len(satUnsent.InfoList)
|
|
|
|
|
}
|
2019-04-01 21:14:58 +01:00
|
|
|
}
|
|
|
|
|
require.NotZero(t, sumBeforeSend)
|
|
|
|
|
|
|
|
|
|
sumUnsent := 0
|
|
|
|
|
sumArchived := 0
|
|
|
|
|
|
|
|
|
|
for _, storageNode := range planet.StorageNodes {
|
storagenode: live tracking of order window usage
This change accomplishes multiple things:
1. Instead of having a max in flight time, which means
we effectively have a minimum bandwidth for uploads
and downloads, we keep track of what windows have
active requests happening in them.
2. We don't double check when we save the order to see if it
is too old: by then, it's too late. A malicious uplink
could just submit orders outside of the grace window and
receive all the data, but the node would just not commit
it, so the uplink gets free traffic. Because the endpoints
also check for the order being too old, this would be a
very tight race that depends on knowledge of the node system
clock, but best to not have the race exist. Instead, we piggy
back off of the in flight tracking and do the check when
we start to handle the order, and commit at the end.
3. Change the functions that send orders and list unsent
orders to accept a time at which that operation is
happening. This way, in tests, we can pretend we're
listing or sending far into the future after the windows
are available to send, rather than exposing test functions
to modify internal state about the grace period to get
the desired effect. This brings tests closer to actual
usage in production.
4. Change the calculation for if an order is allowed to be
enqueued due to the grace period to just look at the
order creation time, rather than some computation involving
the window it will be in. In this way, you can easily
answer the question of "will this order be accepted?" by
asking "is it older than X?" where X is the grace period.
5. Increases the frequency we check to send up orders to once
every 5 minutes instead of once every hour because we already
have hour-long buffering due to the windows. This decreases
the maximum latency that an order will be reported back to
the satellite by 55 minutes.
Change-Id: Ie08b90d139d45ee89b82347e191a2f8db1b88036
2020-08-12 20:01:43 +01:00
|
|
|
storageNode.Storage2.Orders.SendOrders(ctx, tomorrow)
|
2019-04-01 21:14:58 +01:00
|
|
|
|
2020-10-15 19:57:02 +01:00
|
|
|
unsentMap, err := storageNode.OrdersStore.ListUnsentBySatellite(ctx, tomorrow)
|
2019-04-01 21:14:58 +01:00
|
|
|
require.NoError(t, err)
|
2020-07-01 23:05:01 +01:00
|
|
|
for _, satUnsent := range unsentMap {
|
|
|
|
|
sumUnsent += len(satUnsent.InfoList)
|
|
|
|
|
}
|
2019-03-27 10:24:35 +00:00
|
|
|
|
2020-07-01 23:05:01 +01:00
|
|
|
archivedInfos, err := storageNode.OrdersStore.ListArchived()
|
2019-04-01 21:14:58 +01:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
sumArchived += len(archivedInfos)
|
|
|
|
|
}
|
2019-03-27 10:24:35 +00:00
|
|
|
|
2019-04-01 21:14:58 +01:00
|
|
|
require.Zero(t, sumUnsent)
|
|
|
|
|
require.Equal(t, sumBeforeSend, sumArchived)
|
|
|
|
|
})
|
|
|
|
|
}
|
2019-03-27 10:24:35 +00:00
|
|
|
|
2019-04-01 21:14:58 +01:00
|
|
|
func TestUnableToSendOrders(t *testing.T) {
|
|
|
|
|
// test sending when satellite is unavailable
|
|
|
|
|
testplanet.Run(t, testplanet.Config{
|
|
|
|
|
SatelliteCount: 1, StorageNodeCount: 6, UplinkCount: 1,
|
2020-01-21 10:38:41 +00:00
|
|
|
Reconfigure: testplanet.Reconfigure{
|
|
|
|
|
Satellite: testplanet.ReconfigureRS(2, 3, 4, 4),
|
|
|
|
|
},
|
2019-04-01 21:14:58 +01:00
|
|
|
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
storagenode: live tracking of order window usage
This change accomplishes multiple things:
1. Instead of having a max in flight time, which means
we effectively have a minimum bandwidth for uploads
and downloads, we keep track of what windows have
active requests happening in them.
2. We don't double check when we save the order to see if it
is too old: by then, it's too late. A malicious uplink
could just submit orders outside of the grace window and
receive all the data, but the node would just not commit
it, so the uplink gets free traffic. Because the endpoints
also check for the order being too old, this would be a
very tight race that depends on knowledge of the node system
clock, but best to not have the race exist. Instead, we piggy
back off of the in flight tracking and do the check when
we start to handle the order, and commit at the end.
3. Change the functions that send orders and list unsent
orders to accept a time at which that operation is
happening. This way, in tests, we can pretend we're
listing or sending far into the future after the windows
are available to send, rather than exposing test functions
to modify internal state about the grace period to get
the desired effect. This brings tests closer to actual
usage in production.
4. Change the calculation for if an order is allowed to be
enqueued due to the grace period to just look at the
order creation time, rather than some computation involving
the window it will be in. In this way, you can easily
answer the question of "will this order be accepted?" by
asking "is it older than X?" where X is the grace period.
5. Increases the frequency we check to send up orders to once
every 5 minutes instead of once every hour because we already
have hour-long buffering due to the windows. This decreases
the maximum latency that an order will be reported back to
the satellite by 55 minutes.
Change-Id: Ie08b90d139d45ee89b82347e191a2f8db1b88036
2020-08-12 20:01:43 +01:00
|
|
|
now := time.Now()
|
|
|
|
|
tomorrow := now.Add(24 * time.Hour)
|
|
|
|
|
|
2019-09-11 23:37:01 +01:00
|
|
|
planet.Satellites[0].Audit.Worker.Loop.Pause()
|
2019-04-01 21:14:58 +01:00
|
|
|
for _, storageNode := range planet.StorageNodes {
|
2019-08-22 15:33:14 +01:00
|
|
|
storageNode.Storage2.Orders.Sender.Pause()
|
2019-04-01 21:14:58 +01:00
|
|
|
}
|
2019-03-27 10:24:35 +00:00
|
|
|
|
2019-06-26 11:38:51 +01:00
|
|
|
expectedData := testrand.Bytes(50 * memory.KiB)
|
2019-03-27 10:24:35 +00:00
|
|
|
|
2020-01-21 10:38:41 +00:00
|
|
|
err := planet.Uplinks[0].Upload(ctx, planet.Satellites[0], "testbucket", "test/path", expectedData)
|
2019-04-01 21:14:58 +01:00
|
|
|
require.NoError(t, err)
|
2019-03-27 10:24:35 +00:00
|
|
|
|
2020-11-18 13:38:03 +00:00
|
|
|
// Wait for storage nodes to propagate all information.
|
|
|
|
|
require.NoError(t, planet.WaitForStorageNodeEndpoints(ctx))
|
|
|
|
|
|
2019-04-01 21:14:58 +01:00
|
|
|
sumBeforeSend := 0
|
|
|
|
|
for _, storageNode := range planet.StorageNodes {
|
2020-10-15 19:57:02 +01:00
|
|
|
unsentMap, err := storageNode.OrdersStore.ListUnsentBySatellite(ctx, tomorrow)
|
2019-04-01 21:14:58 +01:00
|
|
|
require.NoError(t, err)
|
2020-07-01 23:05:01 +01:00
|
|
|
for _, satUnsent := range unsentMap {
|
|
|
|
|
sumBeforeSend += len(satUnsent.InfoList)
|
|
|
|
|
}
|
2019-04-01 21:14:58 +01:00
|
|
|
}
|
|
|
|
|
require.NotZero(t, sumBeforeSend)
|
2019-03-27 10:24:35 +00:00
|
|
|
|
2019-04-01 21:14:58 +01:00
|
|
|
err = planet.StopPeer(planet.Satellites[0])
|
2019-03-27 10:24:35 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
2019-04-01 21:14:58 +01:00
|
|
|
sumUnsent := 0
|
|
|
|
|
sumArchived := 0
|
|
|
|
|
for _, storageNode := range planet.StorageNodes {
|
storagenode: live tracking of order window usage
This change accomplishes multiple things:
1. Instead of having a max in flight time, which means
we effectively have a minimum bandwidth for uploads
and downloads, we keep track of what windows have
active requests happening in them.
2. We don't double check when we save the order to see if it
is too old: by then, it's too late. A malicious uplink
could just submit orders outside of the grace window and
receive all the data, but the node would just not commit
it, so the uplink gets free traffic. Because the endpoints
also check for the order being too old, this would be a
very tight race that depends on knowledge of the node system
clock, but best to not have the race exist. Instead, we piggy
back off of the in flight tracking and do the check when
we start to handle the order, and commit at the end.
3. Change the functions that send orders and list unsent
orders to accept a time at which that operation is
happening. This way, in tests, we can pretend we're
listing or sending far into the future after the windows
are available to send, rather than exposing test functions
to modify internal state about the grace period to get
the desired effect. This brings tests closer to actual
usage in production.
4. Change the calculation for if an order is allowed to be
enqueued due to the grace period to just look at the
order creation time, rather than some computation involving
the window it will be in. In this way, you can easily
answer the question of "will this order be accepted?" by
asking "is it older than X?" where X is the grace period.
5. Increases the frequency we check to send up orders to once
every 5 minutes instead of once every hour because we already
have hour-long buffering due to the windows. This decreases
the maximum latency that an order will be reported back to
the satellite by 55 minutes.
Change-Id: Ie08b90d139d45ee89b82347e191a2f8db1b88036
2020-08-12 20:01:43 +01:00
|
|
|
storageNode.Storage2.Orders.SendOrders(ctx, tomorrow)
|
2019-04-01 21:14:58 +01:00
|
|
|
|
2020-10-15 19:57:02 +01:00
|
|
|
unsentMap, err := storageNode.OrdersStore.ListUnsentBySatellite(ctx, tomorrow)
|
2019-04-01 21:14:58 +01:00
|
|
|
require.NoError(t, err)
|
2020-07-01 23:05:01 +01:00
|
|
|
for _, satUnsent := range unsentMap {
|
|
|
|
|
sumUnsent += len(satUnsent.InfoList)
|
|
|
|
|
}
|
2019-04-01 21:14:58 +01:00
|
|
|
|
2020-07-01 23:05:01 +01:00
|
|
|
archivedInfos, err := storageNode.OrdersStore.ListArchived()
|
2019-04-01 21:14:58 +01:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
sumArchived += len(archivedInfos)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
require.Zero(t, sumArchived)
|
|
|
|
|
require.Equal(t, sumBeforeSend, sumUnsent)
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestUploadDownloadBandwidth(t *testing.T) {
|
|
|
|
|
testplanet.Run(t, testplanet.Config{
|
|
|
|
|
SatelliteCount: 1, StorageNodeCount: 6, UplinkCount: 1,
|
2020-01-21 10:38:41 +00:00
|
|
|
Reconfigure: testplanet.Reconfigure{
|
|
|
|
|
Satellite: testplanet.ReconfigureRS(2, 3, 4, 4),
|
|
|
|
|
},
|
2019-04-01 21:14:58 +01:00
|
|
|
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
2020-02-14 00:03:41 +00:00
|
|
|
now := time.Now()
|
storagenode: live tracking of order window usage
This change accomplishes multiple things:
1. Instead of having a max in flight time, which means
we effectively have a minimum bandwidth for uploads
and downloads, we keep track of what windows have
active requests happening in them.
2. We don't double check when we save the order to see if it
is too old: by then, it's too late. A malicious uplink
could just submit orders outside of the grace window and
receive all the data, but the node would just not commit
it, so the uplink gets free traffic. Because the endpoints
also check for the order being too old, this would be a
very tight race that depends on knowledge of the node system
clock, but best to not have the race exist. Instead, we piggy
back off of the in flight tracking and do the check when
we start to handle the order, and commit at the end.
3. Change the functions that send orders and list unsent
orders to accept a time at which that operation is
happening. This way, in tests, we can pretend we're
listing or sending far into the future after the windows
are available to send, rather than exposing test functions
to modify internal state about the grace period to get
the desired effect. This brings tests closer to actual
usage in production.
4. Change the calculation for if an order is allowed to be
enqueued due to the grace period to just look at the
order creation time, rather than some computation involving
the window it will be in. In this way, you can easily
answer the question of "will this order be accepted?" by
asking "is it older than X?" where X is the grace period.
5. Increases the frequency we check to send up orders to once
every 5 minutes instead of once every hour because we already
have hour-long buffering due to the windows. This decreases
the maximum latency that an order will be reported back to
the satellite by 55 minutes.
Change-Id: Ie08b90d139d45ee89b82347e191a2f8db1b88036
2020-08-12 20:01:43 +01:00
|
|
|
tomorrow := now.Add(24 * time.Hour)
|
2020-02-14 00:03:41 +00:00
|
|
|
beforeRollup := now.Add(-time.Hour - time.Second)
|
|
|
|
|
afterRollup := now.Add(time.Hour + time.Second)
|
|
|
|
|
bucketName := "testbucket"
|
|
|
|
|
|
2019-09-11 23:37:01 +01:00
|
|
|
planet.Satellites[0].Audit.Worker.Loop.Pause()
|
2019-04-01 21:14:58 +01:00
|
|
|
|
|
|
|
|
for _, storageNode := range planet.StorageNodes {
|
2019-08-22 15:33:14 +01:00
|
|
|
storageNode.Storage2.Orders.Sender.Pause()
|
2019-04-01 21:14:58 +01:00
|
|
|
}
|
|
|
|
|
|
2019-06-26 11:38:51 +01:00
|
|
|
expectedData := testrand.Bytes(50 * memory.KiB)
|
2019-04-01 21:14:58 +01:00
|
|
|
|
2020-02-14 00:03:41 +00:00
|
|
|
err := planet.Uplinks[0].Upload(ctx, planet.Satellites[0], bucketName, "test/path", expectedData)
|
2019-04-01 21:14:58 +01:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
2020-02-14 00:03:41 +00:00
|
|
|
data, err := planet.Uplinks[0].Download(ctx, planet.Satellites[0], bucketName, "test/path")
|
2019-04-01 21:14:58 +01:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
require.Equal(t, expectedData, data)
|
|
|
|
|
|
storagenode: live tracking of order window usage
This change accomplishes multiple things:
1. Instead of having a max in flight time, which means
we effectively have a minimum bandwidth for uploads
and downloads, we keep track of what windows have
active requests happening in them.
2. We don't double check when we save the order to see if it
is too old: by then, it's too late. A malicious uplink
could just submit orders outside of the grace window and
receive all the data, but the node would just not commit
it, so the uplink gets free traffic. Because the endpoints
also check for the order being too old, this would be a
very tight race that depends on knowledge of the node system
clock, but best to not have the race exist. Instead, we piggy
back off of the in flight tracking and do the check when
we start to handle the order, and commit at the end.
3. Change the functions that send orders and list unsent
orders to accept a time at which that operation is
happening. This way, in tests, we can pretend we're
listing or sending far into the future after the windows
are available to send, rather than exposing test functions
to modify internal state about the grace period to get
the desired effect. This brings tests closer to actual
usage in production.
4. Change the calculation for if an order is allowed to be
enqueued due to the grace period to just look at the
order creation time, rather than some computation involving
the window it will be in. In this way, you can easily
answer the question of "will this order be accepted?" by
asking "is it older than X?" where X is the grace period.
5. Increases the frequency we check to send up orders to once
every 5 minutes instead of once every hour because we already
have hour-long buffering due to the windows. This decreases
the maximum latency that an order will be reported back to
the satellite by 55 minutes.
Change-Id: Ie08b90d139d45ee89b82347e191a2f8db1b88036
2020-08-12 20:01:43 +01:00
|
|
|
// Wait for the download to end and so the orders will be saved
|
|
|
|
|
require.NoError(t, planet.WaitForStorageNodeEndpoints(ctx))
|
2019-08-14 10:02:58 +01:00
|
|
|
|
2019-04-01 21:14:58 +01:00
|
|
|
var expectedBucketBandwidth int64
|
|
|
|
|
expectedStorageBandwidth := make(map[storj.NodeID]int64)
|
|
|
|
|
for _, storageNode := range planet.StorageNodes {
|
2020-10-15 19:57:02 +01:00
|
|
|
infos, err := storageNode.OrdersStore.ListUnsentBySatellite(ctx, tomorrow)
|
2019-04-01 21:14:58 +01:00
|
|
|
require.NoError(t, err)
|
storagenode: live tracking of order window usage
This change accomplishes multiple things:
1. Instead of having a max in flight time, which means
we effectively have a minimum bandwidth for uploads
and downloads, we keep track of what windows have
active requests happening in them.
2. We don't double check when we save the order to see if it
is too old: by then, it's too late. A malicious uplink
could just submit orders outside of the grace window and
receive all the data, but the node would just not commit
it, so the uplink gets free traffic. Because the endpoints
also check for the order being too old, this would be a
very tight race that depends on knowledge of the node system
clock, but best to not have the race exist. Instead, we piggy
back off of the in flight tracking and do the check when
we start to handle the order, and commit at the end.
3. Change the functions that send orders and list unsent
orders to accept a time at which that operation is
happening. This way, in tests, we can pretend we're
listing or sending far into the future after the windows
are available to send, rather than exposing test functions
to modify internal state about the grace period to get
the desired effect. This brings tests closer to actual
usage in production.
4. Change the calculation for if an order is allowed to be
enqueued due to the grace period to just look at the
order creation time, rather than some computation involving
the window it will be in. In this way, you can easily
answer the question of "will this order be accepted?" by
asking "is it older than X?" where X is the grace period.
5. Increases the frequency we check to send up orders to once
every 5 minutes instead of once every hour because we already
have hour-long buffering due to the windows. This decreases
the maximum latency that an order will be reported back to
the satellite by 55 minutes.
Change-Id: Ie08b90d139d45ee89b82347e191a2f8db1b88036
2020-08-12 20:01:43 +01:00
|
|
|
for _, unsentInfo := range infos {
|
|
|
|
|
for _, orderInfo := range unsentInfo.InfoList {
|
|
|
|
|
expectedBucketBandwidth += orderInfo.Order.Amount
|
|
|
|
|
expectedStorageBandwidth[storageNode.ID()] += orderInfo.Order.Amount
|
2019-04-01 21:14:58 +01:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for _, storageNode := range planet.StorageNodes {
|
storagenode: live tracking of order window usage
This change accomplishes multiple things:
1. Instead of having a max in flight time, which means
we effectively have a minimum bandwidth for uploads
and downloads, we keep track of what windows have
active requests happening in them.
2. We don't double check when we save the order to see if it
is too old: by then, it's too late. A malicious uplink
could just submit orders outside of the grace window and
receive all the data, but the node would just not commit
it, so the uplink gets free traffic. Because the endpoints
also check for the order being too old, this would be a
very tight race that depends on knowledge of the node system
clock, but best to not have the race exist. Instead, we piggy
back off of the in flight tracking and do the check when
we start to handle the order, and commit at the end.
3. Change the functions that send orders and list unsent
orders to accept a time at which that operation is
happening. This way, in tests, we can pretend we're
listing or sending far into the future after the windows
are available to send, rather than exposing test functions
to modify internal state about the grace period to get
the desired effect. This brings tests closer to actual
usage in production.
4. Change the calculation for if an order is allowed to be
enqueued due to the grace period to just look at the
order creation time, rather than some computation involving
the window it will be in. In this way, you can easily
answer the question of "will this order be accepted?" by
asking "is it older than X?" where X is the grace period.
5. Increases the frequency we check to send up orders to once
every 5 minutes instead of once every hour because we already
have hour-long buffering due to the windows. This decreases
the maximum latency that an order will be reported back to
the satellite by 55 minutes.
Change-Id: Ie08b90d139d45ee89b82347e191a2f8db1b88036
2020-08-12 20:01:43 +01:00
|
|
|
storageNode.Storage2.Orders.SendOrders(ctx, tomorrow)
|
2019-04-01 21:14:58 +01:00
|
|
|
}
|
2020-10-20 19:54:17 +01:00
|
|
|
planet.Satellites[0].Orders.Chore.Loop.TriggerWait()
|
2019-04-01 21:14:58 +01:00
|
|
|
|
|
|
|
|
ordersDB := planet.Satellites[0].DB.Orders()
|
|
|
|
|
|
2020-08-28 16:55:59 +01:00
|
|
|
bucketBandwidth, err := ordersDB.GetBucketBandwidth(ctx, planet.Uplinks[0].Projects[0].ID, []byte(bucketName), beforeRollup, afterRollup)
|
2019-03-27 10:24:35 +00:00
|
|
|
require.NoError(t, err)
|
2019-04-01 21:14:58 +01:00
|
|
|
require.Equal(t, expectedBucketBandwidth, bucketBandwidth)
|
2019-03-27 10:24:35 +00:00
|
|
|
|
2019-04-01 21:14:58 +01:00
|
|
|
for _, storageNode := range planet.StorageNodes {
|
2020-02-14 00:03:41 +00:00
|
|
|
nodeBandwidth, err := ordersDB.GetStorageNodeBandwidth(ctx, storageNode.ID(), beforeRollup, afterRollup)
|
2019-04-01 21:14:58 +01:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
require.Equal(t, expectedStorageBandwidth[storageNode.ID()], nodeBandwidth)
|
|
|
|
|
}
|
|
|
|
|
})
|
2019-03-27 10:24:35 +00:00
|
|
|
}
|
2019-04-22 10:07:50 +01:00
|
|
|
|
2020-01-24 04:57:00 +00:00
|
|
|
func TestMultiProjectUploadDownloadBandwidth(t *testing.T) {
|
|
|
|
|
testplanet.Run(t, testplanet.Config{
|
|
|
|
|
SatelliteCount: 1, StorageNodeCount: 6, UplinkCount: 2,
|
|
|
|
|
Reconfigure: testplanet.Reconfigure{
|
|
|
|
|
Satellite: testplanet.ReconfigureRS(2, 3, 4, 4),
|
|
|
|
|
},
|
|
|
|
|
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
2020-07-01 23:05:01 +01:00
|
|
|
// stop any async flushes because we want to be sure when some values are
|
|
|
|
|
// written to avoid races
|
|
|
|
|
planet.Satellites[0].Orders.Chore.Loop.Pause()
|
|
|
|
|
|
2020-02-14 00:03:41 +00:00
|
|
|
now := time.Now()
|
storagenode: live tracking of order window usage
This change accomplishes multiple things:
1. Instead of having a max in flight time, which means
we effectively have a minimum bandwidth for uploads
and downloads, we keep track of what windows have
active requests happening in them.
2. We don't double check when we save the order to see if it
is too old: by then, it's too late. A malicious uplink
could just submit orders outside of the grace window and
receive all the data, but the node would just not commit
it, so the uplink gets free traffic. Because the endpoints
also check for the order being too old, this would be a
very tight race that depends on knowledge of the node system
clock, but best to not have the race exist. Instead, we piggy
back off of the in flight tracking and do the check when
we start to handle the order, and commit at the end.
3. Change the functions that send orders and list unsent
orders to accept a time at which that operation is
happening. This way, in tests, we can pretend we're
listing or sending far into the future after the windows
are available to send, rather than exposing test functions
to modify internal state about the grace period to get
the desired effect. This brings tests closer to actual
usage in production.
4. Change the calculation for if an order is allowed to be
enqueued due to the grace period to just look at the
order creation time, rather than some computation involving
the window it will be in. In this way, you can easily
answer the question of "will this order be accepted?" by
asking "is it older than X?" where X is the grace period.
5. Increases the frequency we check to send up orders to once
every 5 minutes instead of once every hour because we already
have hour-long buffering due to the windows. This decreases
the maximum latency that an order will be reported back to
the satellite by 55 minutes.
Change-Id: Ie08b90d139d45ee89b82347e191a2f8db1b88036
2020-08-12 20:01:43 +01:00
|
|
|
tomorrow := now.Add(24 * time.Hour)
|
2020-02-14 00:03:41 +00:00
|
|
|
beforeRollup := now.Add(-time.Hour - time.Second)
|
|
|
|
|
afterRollup := now.Add(time.Hour + time.Second)
|
|
|
|
|
|
2020-01-24 04:57:00 +00:00
|
|
|
planet.Satellites[0].Audit.Worker.Loop.Pause()
|
|
|
|
|
|
|
|
|
|
for _, storageNode := range planet.StorageNodes {
|
|
|
|
|
storageNode.Storage2.Orders.Sender.Pause()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Upload some data to two different projects in different buckets.
|
|
|
|
|
firstExpectedData := testrand.Bytes(50 * memory.KiB)
|
|
|
|
|
err := planet.Uplinks[0].Upload(ctx, planet.Satellites[0], "testbucket0", "test/path", firstExpectedData)
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
data, err := planet.Uplinks[0].Download(ctx, planet.Satellites[0], "testbucket0", "test/path")
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
require.Equal(t, firstExpectedData, data)
|
|
|
|
|
|
|
|
|
|
secondExpectedData := testrand.Bytes(100 * memory.KiB)
|
|
|
|
|
err = planet.Uplinks[1].Upload(ctx, planet.Satellites[0], "testbucket1", "test/path", secondExpectedData)
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
data, err = planet.Uplinks[1].Download(ctx, planet.Satellites[0], "testbucket1", "test/path")
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
require.Equal(t, secondExpectedData, data)
|
|
|
|
|
|
2020-11-18 13:38:03 +00:00
|
|
|
// Wait for storage nodes to propagate all information.
|
|
|
|
|
require.NoError(t, planet.WaitForStorageNodeEndpoints(ctx))
|
|
|
|
|
|
2020-01-24 04:57:00 +00:00
|
|
|
// Have the nodes send up the orders.
|
|
|
|
|
for _, storageNode := range planet.StorageNodes {
|
storagenode: live tracking of order window usage
This change accomplishes multiple things:
1. Instead of having a max in flight time, which means
we effectively have a minimum bandwidth for uploads
and downloads, we keep track of what windows have
active requests happening in them.
2. We don't double check when we save the order to see if it
is too old: by then, it's too late. A malicious uplink
could just submit orders outside of the grace window and
receive all the data, but the node would just not commit
it, so the uplink gets free traffic. Because the endpoints
also check for the order being too old, this would be a
very tight race that depends on knowledge of the node system
clock, but best to not have the race exist. Instead, we piggy
back off of the in flight tracking and do the check when
we start to handle the order, and commit at the end.
3. Change the functions that send orders and list unsent
orders to accept a time at which that operation is
happening. This way, in tests, we can pretend we're
listing or sending far into the future after the windows
are available to send, rather than exposing test functions
to modify internal state about the grace period to get
the desired effect. This brings tests closer to actual
usage in production.
4. Change the calculation for if an order is allowed to be
enqueued due to the grace period to just look at the
order creation time, rather than some computation involving
the window it will be in. In this way, you can easily
answer the question of "will this order be accepted?" by
asking "is it older than X?" where X is the grace period.
5. Increases the frequency we check to send up orders to once
every 5 minutes instead of once every hour because we already
have hour-long buffering due to the windows. This decreases
the maximum latency that an order will be reported back to
the satellite by 55 minutes.
Change-Id: Ie08b90d139d45ee89b82347e191a2f8db1b88036
2020-08-12 20:01:43 +01:00
|
|
|
storageNode.Storage2.Orders.SendOrders(ctx, tomorrow)
|
2020-01-24 04:57:00 +00:00
|
|
|
}
|
2020-07-01 23:05:01 +01:00
|
|
|
// flush rollups write cache
|
|
|
|
|
planet.Satellites[0].Orders.Chore.Loop.TriggerWait()
|
2020-01-24 04:57:00 +00:00
|
|
|
|
|
|
|
|
// Query and ensure that there's no data recorded for the bucket from the other project
|
|
|
|
|
ordersDB := planet.Satellites[0].DB.Orders()
|
2020-04-16 13:12:46 +01:00
|
|
|
uplink0Project := planet.Uplinks[0].Projects[0].ID
|
|
|
|
|
uplink1Project := planet.Uplinks[1].Projects[0].ID
|
2020-01-24 04:57:00 +00:00
|
|
|
|
2020-02-14 00:03:41 +00:00
|
|
|
wrongBucketBandwidth, err := ordersDB.GetBucketBandwidth(ctx, uplink0Project, []byte("testbucket1"), beforeRollup, afterRollup)
|
2020-01-24 04:57:00 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
require.Equal(t, int64(0), wrongBucketBandwidth)
|
2020-02-14 00:03:41 +00:00
|
|
|
rightBucketBandwidth, err := ordersDB.GetBucketBandwidth(ctx, uplink0Project, []byte("testbucket0"), beforeRollup, afterRollup)
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
require.Greater(t, rightBucketBandwidth, int64(0))
|
|
|
|
|
|
|
|
|
|
wrongBucketBandwidth, err = ordersDB.GetBucketBandwidth(ctx, uplink1Project, []byte("testbucket0"), beforeRollup, afterRollup)
|
2020-01-24 04:57:00 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
require.Equal(t, int64(0), wrongBucketBandwidth)
|
2020-02-14 00:03:41 +00:00
|
|
|
rightBucketBandwidth, err = ordersDB.GetBucketBandwidth(ctx, uplink1Project, []byte("testbucket1"), beforeRollup, afterRollup)
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
require.Greater(t, rightBucketBandwidth, int64(0))
|
2020-01-24 04:57:00 +00:00
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
2020-06-11 19:31:45 +01:00
|
|
|
func TestUpdateStoragenodeBandwidthSettleWithWindow(t *testing.T) {
|
|
|
|
|
satellitedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db satellite.DB) {
|
|
|
|
|
ordersDB := db.Orders()
|
|
|
|
|
now := time.Now().UTC()
|
|
|
|
|
projectID := testrand.UUID()
|
|
|
|
|
bucketname := "testbucket"
|
|
|
|
|
snID := storj.NodeID{1}
|
|
|
|
|
windowTime := now.AddDate(0, 0, -1)
|
|
|
|
|
actionAmounts := map[int32]int64{
|
|
|
|
|
int32(pb.PieceAction_GET): 100,
|
|
|
|
|
int32(pb.PieceAction_PUT): 200,
|
|
|
|
|
int32(pb.PieceAction_DELETE): 300,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// confirm there aren't any records in the storagenodebandwidth or bucketbandwidth table
|
|
|
|
|
// at the beginning of the test
|
|
|
|
|
storagenodeID := storj.NodeID{1}
|
|
|
|
|
snbw, err := ordersDB.GetStorageNodeBandwidth(ctx, storagenodeID, time.Time{}, now)
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
require.Equal(t, int64(0), snbw)
|
|
|
|
|
bucketbw, err := ordersDB.GetBucketBandwidth(ctx, projectID, []byte(bucketname), time.Time{}, now)
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
require.Equal(t, int64(0), bucketbw)
|
|
|
|
|
|
|
|
|
|
// test: process an order from a storagenode that has not been processed before
|
|
|
|
|
status, alreadyProcesed, err := ordersDB.UpdateStoragenodeBandwidthSettleWithWindow(ctx, snID, actionAmounts, windowTime)
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
require.Equal(t, pb.SettlementWithWindowResponse_ACCEPTED, status)
|
|
|
|
|
require.Equal(t, false, alreadyProcesed)
|
|
|
|
|
// confirm a record for storagenode bandwidth has been created
|
|
|
|
|
snbw, err = ordersDB.GetStorageNodeBandwidth(ctx, storagenodeID, time.Time{}, now)
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
require.Equal(t, int64(600), snbw)
|
|
|
|
|
|
|
|
|
|
// test: process an order from a storagenode that has already been processed
|
|
|
|
|
// and the orders match the orders that were already processed
|
|
|
|
|
status, alreadyProcesed, err = ordersDB.UpdateStoragenodeBandwidthSettleWithWindow(ctx, snID, actionAmounts, windowTime)
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
require.Equal(t, pb.SettlementWithWindowResponse_ACCEPTED, status)
|
|
|
|
|
require.Equal(t, true, alreadyProcesed)
|
|
|
|
|
// confirm that no more records were created for storagenode bandwidth
|
|
|
|
|
snbw, err = ordersDB.GetStorageNodeBandwidth(ctx, storagenodeID, time.Time{}, now)
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
require.Equal(t, int64(600), snbw)
|
|
|
|
|
|
|
|
|
|
// test: process an order from a storagenode that has already been processed
|
|
|
|
|
// and the orders DO NOT match the orders that were already processed
|
|
|
|
|
actionAmounts2 := map[int32]int64{
|
|
|
|
|
int32(pb.PieceAction_GET): 50,
|
|
|
|
|
int32(pb.PieceAction_PUT): 25,
|
|
|
|
|
int32(pb.PieceAction_DELETE): 100,
|
|
|
|
|
}
|
|
|
|
|
status, alreadyProcesed, err = ordersDB.UpdateStoragenodeBandwidthSettleWithWindow(ctx, snID, actionAmounts2, windowTime)
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
require.Equal(t, pb.SettlementWithWindowResponse_REJECTED, status)
|
|
|
|
|
require.Equal(t, false, alreadyProcesed)
|
|
|
|
|
// confirm that no more records were created for storagenode bandwidth
|
|
|
|
|
snbw, err = ordersDB.GetStorageNodeBandwidth(ctx, storagenodeID, time.Time{}, now)
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
require.Equal(t, int64(600), snbw)
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestSettledAmountsMatch(t *testing.T) {
|
2021-01-22 13:51:29 +00:00
|
|
|
testCases := []struct {
|
2020-06-11 19:31:45 +01:00
|
|
|
name string
|
|
|
|
|
rows []*dbx.StoragenodeBandwidthRollup
|
|
|
|
|
orderActionAmounts map[int32]int64
|
|
|
|
|
expected bool
|
|
|
|
|
}{
|
|
|
|
|
{"zero value", []*dbx.StoragenodeBandwidthRollup{}, map[int32]int64{}, true},
|
|
|
|
|
{"nil value", nil, nil, false},
|
|
|
|
|
{"more rows amount", []*dbx.StoragenodeBandwidthRollup{{Action: uint(pb.PieceAction_PUT), Settled: 100}, {Action: uint(pb.PieceAction_GET), Settled: 200}}, map[int32]int64{1: 200}, false},
|
|
|
|
|
{"equal", []*dbx.StoragenodeBandwidthRollup{{Action: uint(pb.PieceAction_PUT), Settled: 100}, {Action: uint(pb.PieceAction_GET), Settled: 200}}, map[int32]int64{1: 100, 2: 200}, true},
|
|
|
|
|
{"more orders amount", []*dbx.StoragenodeBandwidthRollup{{Action: uint(pb.PieceAction_PUT), Settled: 100}}, map[int32]int64{1: 200, 0: 100}, false},
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for _, tt := range testCases {
|
2020-07-16 16:40:29 +01:00
|
|
|
tt := tt
|
2020-06-11 19:31:45 +01:00
|
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
|
|
|
matches := satellitedb.SettledAmountsMatch(tt.rows, tt.orderActionAmounts)
|
|
|
|
|
require.Equal(t, tt.expected, matches)
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|
