storj/satellite/console/consoleweb/consoleql/query_test.go

474 lines
14 KiB
Go
Raw Normal View History

2019-01-24 16:26:36 +00:00
// Copyright (C) 2019 Storj Labs, Inc.
2019-01-08 14:54:35 +00:00
// See LICENSE for copying information.
2019-01-24 16:26:36 +00:00
package consoleql_test
2019-01-08 14:54:35 +00:00
import (
"fmt"
"testing"
"time"
"github.com/graphql-go/graphql"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
2019-01-31 13:01:13 +00:00
"go.uber.org/zap/zaptest"
2019-01-08 14:54:35 +00:00
"storj.io/common/testcontext"
"storj.io/common/testrand"
"storj.io/storj/private/testplanet"
"storj.io/storj/private/testredis"
"storj.io/storj/satellite/accounting"
"storj.io/storj/satellite/accounting/live"
"storj.io/storj/satellite/analytics"
"storj.io/storj/satellite/console"
"storj.io/storj/satellite/console/consoleauth"
2019-01-24 16:26:36 +00:00
"storj.io/storj/satellite/console/consoleweb/consoleql"
"storj.io/storj/satellite/console/restkeys"
"storj.io/storj/satellite/mailservice"
"storj.io/storj/satellite/payments"
"storj.io/storj/satellite/payments/paymentsconfig"
"storj.io/storj/satellite/payments/stripecoinpayments"
"storj.io/storj/satellite/rewards"
2019-01-08 14:54:35 +00:00
)
func TestGraphqlQuery(t *testing.T) {
testplanet.Run(t, testplanet.Config{SatelliteCount: 1}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
sat := planet.Satellites[0]
db := sat.DB
2019-01-31 13:01:13 +00:00
log := zaptest.NewLogger(t)
2019-01-08 14:54:35 +00:00
partnersService := rewards.NewPartnersService(
log.Named("partners"),
rewards.DefaultPartnersDB,
)
analyticsService := analytics.NewService(log, analytics.Config{}, "test-satellite")
redis, err := testredis.Mini(ctx)
satellite/accounting: refactor live accounting to hold current estimated totals live accounting used to be a cache to store writes before they are picked up during the tally iteration, after which the cache is cleared. This created a window in which users could potentially exceed the storage limit. This PR refactors live accounting to hold current estimations of space used per project. This should also reduce DB load since we no longer need to query the satellite DB when checking space used for limiting. The mechanism by which the new live accounting system works is as follows: During the upload of any segment, the size of that segment is added to its respective project total in live accounting. At the beginning of the tally iteration we record the current values in live accounting as `initialLiveTotals`. At the end of the tally iteration we again record the current totals in live accounting as `latestLiveTotals`. The metainfo loop observer in tally allows us to get the project totals from what it observed in metainfo DB which are stored in `tallyProjectTotals`. However, for any particular segment uploaded during the metainfo loop, the observer may or may not have seen it. Thus, we take half of the difference between `latestLiveTotals` and `initialLiveTotals`, and add that to the total that was found during tally and set that as the new live accounting total. Initially, live accounting was storing the total stored amount across all nodes rather than the segment size, which is inconsistent with how we record amounts stored in the project accounting DB, so we have refactored live accounting to record segment size Change-Id: Ie48bfdef453428fcdc180b2d781a69d58fd927fb
2019-10-31 17:27:38 +00:00
require.NoError(t, err)
defer ctx.Check(redis.Close)
satellite/accounting: refactor live accounting to hold current estimated totals live accounting used to be a cache to store writes before they are picked up during the tally iteration, after which the cache is cleared. This created a window in which users could potentially exceed the storage limit. This PR refactors live accounting to hold current estimations of space used per project. This should also reduce DB load since we no longer need to query the satellite DB when checking space used for limiting. The mechanism by which the new live accounting system works is as follows: During the upload of any segment, the size of that segment is added to its respective project total in live accounting. At the beginning of the tally iteration we record the current values in live accounting as `initialLiveTotals`. At the end of the tally iteration we again record the current totals in live accounting as `latestLiveTotals`. The metainfo loop observer in tally allows us to get the project totals from what it observed in metainfo DB which are stored in `tallyProjectTotals`. However, for any particular segment uploaded during the metainfo loop, the observer may or may not have seen it. Thus, we take half of the difference between `latestLiveTotals` and `initialLiveTotals`, and add that to the total that was found during tally and set that as the new live accounting total. Initially, live accounting was storing the total stored amount across all nodes rather than the segment size, which is inconsistent with how we record amounts stored in the project accounting DB, so we have refactored live accounting to record segment size Change-Id: Ie48bfdef453428fcdc180b2d781a69d58fd927fb
2019-10-31 17:27:38 +00:00
cache, err := live.OpenCache(ctx, log.Named("cache"), live.Config{StorageBackend: "redis://" + redis.Addr() + "?db=0"})
require.NoError(t, err)
projectLimitCache := accounting.NewProjectLimitCache(db.ProjectAccounting(), 0, 0, 0, accounting.ProjectLimitConfig{CacheCapacity: 100})
satellite/accounting: add cache for getting project storage and bw limits This PR adds the following items: 1) an in-memory read-only cache thats stores project limit info for projectIDs This cache is stored in-memory since this is expected to be a small amount of data. In this implementation we are only storing in the cache projects that have been accessed. Currently for the largest Satellite (eu-west) there is about 4500 total projects. So storing the storage limit (int64) and the bandwidth limit (int64), this would end up being about 200kb (including the 32 byte project ID) if all 4500 projectIDs were in the cache. So this all fits in memory for the time being. At some point it may not as usage grows, but that seems years out. The cache is a read only cache. When requests come in to upload/download a file, we will read from the cache what the current limits are for that project. If the cache does not contain the projectID, it will get the info from the database (satellitedb project table), then add it to the cache. The only time the values in the cache are modified is when either a) the project ID is not in the cache, or b) the item in the cache has expired (default 10mins), then the data gets refreshed out of the database. This occurs by default every 10 mins. This means that if we update the usage limits in the database, that change might not show up in the cache for 10 mins which mean it will not be reflected to limit end users uploading/downloading files for that time period.. Change-Id: I3fd7056cf963676009834fcbcf9c4a0922ca4a8f
2020-09-09 20:20:44 +01:00
projectUsage := accounting.NewService(db.ProjectAccounting(), cache, projectLimitCache, *sat.Metabase.DB, 5*time.Minute, -10*time.Second)
// TODO maybe switch this test to testplanet to avoid defining config and Stripe service
pc := paymentsconfig.Config{
StorageTBPrice: "10",
EgressTBPrice: "45",
SegmentPrice: "0.0000022",
}
paymentsService, err := stripecoinpayments.NewService(
log.Named("payments.stripe:service"),
stripecoinpayments.NewStripeMock(
testrand.NodeID(),
db.StripeCoinPayments().Customers(),
db.Console().Users(),
),
pc.StripeCoinPayments,
db.StripeCoinPayments(),
db.Console().Projects(),
db.ProjectAccounting(),
pc.StorageTBPrice,
pc.EgressTBPrice,
pc.SegmentPrice,
pc.BonusRate)
require.NoError(t, err)
2019-01-24 16:26:36 +00:00
service, err := console.NewService(
log.Named("console"),
2019-01-24 16:26:36 +00:00
db.Console(),
restkeys.NewService(db.OIDC().OAuthTokens(), planet.Satellites[0].Config.RESTKeys),
db.ProjectAccounting(),
projectUsage,
sat.API.Buckets.Service,
partnersService,
paymentsService.Accounts(),
analyticsService,
consoleauth.NewService(consoleauth.Config{
TokenExpirationTime: 24 * time.Hour,
}, &consoleauth.Hmac{Secret: []byte("my-suppa-secret-key")}),
console.Config{
PasswordCost: console.TestPasswordCost,
DefaultProjectLimit: 5,
},
2019-01-24 16:26:36 +00:00
)
require.NoError(t, err)
2019-01-08 14:54:35 +00:00
mailService, err := mailservice.New(log, &discardSender{}, "testdata")
require.NoError(t, err)
defer ctx.Check(mailService.Close)
rootObject := make(map[string]interface{})
rootObject["origin"] = "http://doesntmatter.com/"
rootObject[consoleql.ActivationPath] = "?activationToken="
rootObject[consoleql.LetUsKnowURL] = "letUsKnowURL"
rootObject[consoleql.ContactInfoURL] = "contactInfoURL"
rootObject[consoleql.TermsAndConditionsURL] = "termsAndConditionsURL"
2019-01-24 16:26:36 +00:00
creator := consoleql.TypeCreator{}
err = creator.Create(log, service, mailService)
require.NoError(t, err)
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
schema, err := graphql.NewSchema(graphql.SchemaConfig{
Query: creator.RootQuery(),
Mutation: creator.RootMutation(),
})
require.NoError(t, err)
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
createUser := console.CreateUser{
FullName: "John",
ShortName: "",
Email: "mtest@mail.test",
Password: "123a123",
SignupPromoCode: "promo1",
2019-01-24 16:26:36 +00:00
}
2019-01-08 14:54:35 +00:00
regToken, err := service.CreateRegToken(ctx, 2)
require.NoError(t, err)
rootUser, err := service.CreateUser(ctx, createUser, regToken.Secret)
require.NoError(t, err)
2019-01-08 14:54:35 +00:00
couponType, err := paymentsService.Accounts().Setup(ctx, rootUser.ID, rootUser.Email, rootUser.SignupPromoCode)
var signupCouponType payments.CouponType = payments.SignupCoupon
require.NoError(t, err)
assert.Equal(t, signupCouponType, couponType)
t.Run("Activation", func(t *testing.T) {
activationToken, err := service.GenerateActivationToken(
ctx,
rootUser.ID,
"mtest@mail.test",
)
require.NoError(t, err)
_, err = service.ActivateAccount(ctx, activationToken)
require.NoError(t, err)
rootUser.Email = "mtest@mail.test"
})
token, err := service.Token(ctx, console.AuthUser{Email: createUser.Email, Password: createUser.Password})
require.NoError(t, err)
2019-01-08 14:54:35 +00:00
sauth, err := service.Authorize(consoleauth.WithAPIKey(ctx, []byte(token)))
require.NoError(t, err)
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
authCtx := console.WithAuth(ctx, sauth)
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
testQuery := func(t *testing.T, query string) interface{} {
result := graphql.Do(graphql.Params{
Schema: schema,
Context: authCtx,
RequestString: query,
RootObject: rootObject,
2019-01-24 16:26:36 +00:00
})
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
for _, err := range result.Errors {
assert.NoError(t, err)
}
require.False(t, result.HasErrors())
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
return result.Data
}
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
createdProject, err := service.CreateProject(authCtx, console.ProjectInfo{
Name: "TestProject",
2019-01-24 16:26:36 +00:00
})
require.NoError(t, err)
2019-01-08 14:54:35 +00:00
// "query {project(id:\"%s\"){id,name,members(offset:0, limit:50){user{fullName,shortName,email}},apiKeys{name,id,createdAt,projectID}}}"
2019-01-24 16:26:36 +00:00
t.Run("Project query base info", func(t *testing.T) {
query := fmt.Sprintf(
"query {project(id:\"%s\"){id,name,description,createdAt}}",
createdProject.ID.String(),
)
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
result := testQuery(t, query)
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
data := result.(map[string]interface{})
project := data[consoleql.ProjectQuery].(map[string]interface{})
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
assert.Equal(t, createdProject.ID.String(), project[consoleql.FieldID])
assert.Equal(t, createdProject.Name, project[consoleql.FieldName])
assert.Equal(t, createdProject.Description, project[consoleql.FieldDescription])
2019-01-08 14:54:35 +00:00
createdAt := time.Time{}
2019-01-24 16:26:36 +00:00
err := createdAt.UnmarshalText([]byte(project[consoleql.FieldCreatedAt].(string)))
2019-01-08 14:54:35 +00:00
assert.NoError(t, err)
assert.True(t, createdProject.CreatedAt.Equal(createdAt))
2019-01-24 16:26:36 +00:00
})
regTokenUser1, err := service.CreateRegToken(ctx, 2)
require.NoError(t, err)
2019-01-24 16:26:36 +00:00
user1, err := service.CreateUser(authCtx, console.CreateUser{
FullName: "Mickey Last",
ShortName: "Last",
Password: "123a123",
Email: "muu1@mail.test",
}, regTokenUser1.Secret)
require.NoError(t, err)
t.Run("Activation", func(t *testing.T) {
activationToken1, err := service.GenerateActivationToken(
ctx,
user1.ID,
"muu1@mail.test",
)
require.NoError(t, err)
_, err = service.ActivateAccount(ctx, activationToken1)
require.NoError(t, err)
user1.Email = "muu1@mail.test"
})
2019-01-08 14:54:35 +00:00
regTokenUser2, err := service.CreateRegToken(ctx, 2)
require.NoError(t, err)
2019-01-24 16:26:36 +00:00
user2, err := service.CreateUser(authCtx, console.CreateUser{
FullName: "Dubas Name",
ShortName: "Name",
Email: "muu2@mail.test",
Password: "123a123",
}, regTokenUser2.Secret)
require.NoError(t, err)
t.Run("Activation", func(t *testing.T) {
activationToken2, err := service.GenerateActivationToken(
ctx,
user2.ID,
"muu2@mail.test",
)
require.NoError(t, err)
_, err = service.ActivateAccount(ctx, activationToken2)
require.NoError(t, err)
user2.Email = "muu2@mail.test"
})
2019-01-08 14:54:35 +00:00
users, err := service.AddProjectMembers(authCtx, createdProject.ID, []string{
2019-01-24 16:26:36 +00:00
user1.Email,
user2.Email,
})
require.NoError(t, err)
assert.Equal(t, 2, len(users))
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
t.Run("Project query team members", func(t *testing.T) {
query := fmt.Sprintf(
"query {project(id: \"%s\") {members( cursor: { limit: %d, search: \"%s\", page: %d, order: %d, orderDirection: %d } ) { projectMembers{ user { id, fullName, shortName, email, createdAt }, joinedAt }, search, limit, order, offset, pageCount, currentPage, totalCount } } }",
2019-01-24 16:26:36 +00:00
createdProject.ID.String(),
5,
"",
1,
1,
2)
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
result := testQuery(t, query)
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
data := result.(map[string]interface{})
project := data[consoleql.ProjectQuery].(map[string]interface{})
members := project[consoleql.FieldMembers].(map[string]interface{})
projectMembers := members[consoleql.FieldProjectMembers].([]interface{})
2019-01-08 14:54:35 +00:00
assert.Equal(t, 3, len(projectMembers))
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
testUser := func(t *testing.T, actual map[string]interface{}, expected *console.User) {
assert.Equal(t, expected.Email, actual[consoleql.FieldEmail])
assert.Equal(t, expected.FullName, actual[consoleql.FieldFullName])
assert.Equal(t, expected.ShortName, actual[consoleql.FieldShortName])
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
createdAt := time.Time{}
err := createdAt.UnmarshalText([]byte(actual[consoleql.FieldCreatedAt].(string)))
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
assert.NoError(t, err)
assert.True(t, expected.CreatedAt.Equal(createdAt))
2019-01-24 16:26:36 +00:00
}
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
var foundRoot, foundU1, foundU2 bool
for _, entry := range projectMembers {
2019-01-24 16:26:36 +00:00
member := entry.(map[string]interface{})
user := member[consoleql.UserType].(map[string]interface{})
id := user[consoleql.FieldID].(string)
switch id {
case rootUser.ID.String():
foundRoot = true
testUser(t, user, rootUser)
case user1.ID.String():
foundU1 = true
testUser(t, user, user1)
case user2.ID.String():
foundU2 = true
testUser(t, user, user2)
}
}
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
assert.True(t, foundRoot)
assert.True(t, foundU1)
assert.True(t, foundU2)
})
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
keyInfo1, _, err := service.CreateAPIKey(authCtx, createdProject.ID, "key1")
require.NoError(t, err)
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
keyInfo2, _, err := service.CreateAPIKey(authCtx, createdProject.ID, "key2")
require.NoError(t, err)
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
t.Run("Project query api keys", func(t *testing.T) {
query := fmt.Sprintf(
"query {project(id: \"%s\") {apiKeys( cursor: { limit: %d, search: \"%s\", page: %d, order: %d, orderDirection: %d } ) { apiKeys { id, name, createdAt, projectID }, search, limit, order, offset, pageCount, currentPage, totalCount } } }",
2019-01-24 16:26:36 +00:00
createdProject.ID.String(),
5,
"",
1,
1,
2)
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
result := testQuery(t, query)
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
data := result.(map[string]interface{})
project := data[consoleql.ProjectQuery].(map[string]interface{})
keys := project[consoleql.FieldAPIKeys].(map[string]interface{})
apiKeys := keys[consoleql.FieldAPIKeys].([]interface{})
2019-01-08 14:54:35 +00:00
assert.Equal(t, 2, len(apiKeys))
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
testAPIKey := func(t *testing.T, actual map[string]interface{}, expected *console.APIKeyInfo) {
assert.Equal(t, expected.Name, actual[consoleql.FieldName])
assert.Equal(t, expected.ProjectID.String(), actual[consoleql.FieldProjectID])
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
createdAt := time.Time{}
err := createdAt.UnmarshalText([]byte(actual[consoleql.FieldCreatedAt].(string)))
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
assert.NoError(t, err)
assert.True(t, expected.CreatedAt.Equal(createdAt))
2019-01-24 16:26:36 +00:00
}
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
var foundKey1, foundKey2 bool
2019-01-08 14:54:35 +00:00
for _, entry := range apiKeys {
2019-01-24 16:26:36 +00:00
key := entry.(map[string]interface{})
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
id := key[consoleql.FieldID].(string)
switch id {
case keyInfo1.ID.String():
foundKey1 = true
testAPIKey(t, key, keyInfo1)
case keyInfo2.ID.String():
foundKey2 = true
testAPIKey(t, key, keyInfo2)
}
}
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
assert.True(t, foundKey1)
assert.True(t, foundKey2)
})
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
project2, err := service.CreateProject(authCtx, console.ProjectInfo{
Name: "Project2",
Description: "Test desc",
2019-01-24 16:26:36 +00:00
})
require.NoError(t, err)
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
t.Run("MyProjects query", func(t *testing.T) {
query := "query {myProjects{id,name,description,createdAt}}"
result := testQuery(t, query)
data := result.(map[string]interface{})
projectsList := data[consoleql.MyProjectsQuery].([]interface{})
assert.Equal(t, 2, len(projectsList))
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
testProject := func(t *testing.T, actual map[string]interface{}, expected *console.Project) {
assert.Equal(t, expected.Name, actual[consoleql.FieldName])
assert.Equal(t, expected.Description, actual[consoleql.FieldDescription])
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
createdAt := time.Time{}
err := createdAt.UnmarshalText([]byte(actual[consoleql.FieldCreatedAt].(string)))
assert.NoError(t, err)
assert.True(t, expected.CreatedAt.Equal(createdAt))
2019-01-08 14:54:35 +00:00
}
2019-01-24 16:26:36 +00:00
var foundProj1, foundProj2 bool
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
for _, entry := range projectsList {
project := entry.(map[string]interface{})
2019-01-08 14:54:35 +00:00
2019-01-24 16:26:36 +00:00
id := project[consoleql.FieldID].(string)
switch id {
case createdProject.ID.String():
foundProj1 = true
testProject(t, project, createdProject)
case project2.ID.String():
foundProj2 = true
testProject(t, project, project2)
}
}
2019-01-08 14:54:35 +00:00
assert.True(t, foundProj1)
assert.True(t, foundProj2)
})
t.Run("OwnedProjects query", func(t *testing.T) {
query := fmt.Sprintf(
"query {ownedProjects( cursor: { limit: %d, page: %d } ) {projects{id, name, ownerId, description, createdAt, memberCount}, limit, offset, pageCount, currentPage, totalCount } }",
5,
1,
)
result := testQuery(t, query)
data := result.(map[string]interface{})
projectsPage := data[consoleql.OwnedProjectsQuery].(map[string]interface{})
projectsList := projectsPage[consoleql.FieldProjects].([]interface{})
assert.Len(t, projectsList, 2)
assert.EqualValues(t, 1, projectsPage[consoleql.FieldCurrentPage])
assert.EqualValues(t, 0, projectsPage[consoleql.OffsetArg])
assert.EqualValues(t, 5, projectsPage[consoleql.LimitArg])
assert.EqualValues(t, 1, projectsPage[consoleql.FieldPageCount])
assert.EqualValues(t, 2, projectsPage[consoleql.FieldTotalCount])
testProject := func(t *testing.T, actual map[string]interface{}, expected *console.Project, expectedNumMembers int) {
assert.Equal(t, expected.Name, actual[consoleql.FieldName])
assert.Equal(t, expected.Description, actual[consoleql.FieldDescription])
createdAt := time.Time{}
err := createdAt.UnmarshalText([]byte(actual[consoleql.FieldCreatedAt].(string)))
assert.NoError(t, err)
assert.True(t, expected.CreatedAt.Equal(createdAt))
assert.EqualValues(t, expectedNumMembers, actual[consoleql.FieldMemberCount])
}
var foundProj1, foundProj2 bool
for _, entry := range projectsList {
project := entry.(map[string]interface{})
id := project[consoleql.FieldID].(string)
switch id {
case createdProject.ID.String():
foundProj1 = true
testProject(t, project, createdProject, 3)
case project2.ID.String():
foundProj2 = true
testProject(t, project, project2, 1)
}
}
2019-01-24 16:26:36 +00:00
assert.True(t, foundProj1)
assert.True(t, foundProj2)
})
2019-01-08 14:54:35 +00:00
})
}