satellite/metainfo: strip the uplink version from the UserAgent and bound its size
The UserAgent should be stored as is, with the exception of removing the trailing version from any libuplink user agents Change-Id: If17ef2fc4b59480a3477300f2585a07d64cc2bf4
This commit is contained in:
dlamarmorgan
parent
27091826ba
commit
1dd537953e
4
go.mod
4
go.mod
@ -47,11 +47,11 @@ require (
|
||||
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e
|
||||
gopkg.in/segmentio/analytics-go.v3 v3.1.0
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
|
||||
storj.io/common v0.0.0-20211021003554-f155346ccc11
|
||||
storj.io/common v0.0.0-20211102144601-401a79f0706a
|
||||
storj.io/drpc v0.0.26
|
||||
storj.io/monkit-jaeger v0.0.0-20210426161729-debb1cbcbbd7
|
||||
storj.io/private v0.0.0-20211029202355-a7eae71c382a
|
||||
storj.io/uplink v1.7.1-0.20211012154306-65bb87992c7c
|
||||
storj.io/uplink v1.7.1-0.20211031201307-b30e004c1ccb
|
||||
)
|
||||
|
||||
require (
|
||||
|
||||
10
go.sum
10
go.sum
@ -879,9 +879,9 @@ sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2
|
||||
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
|
||||
storj.io/common v0.0.0-20200424175742-65ac59022f4f/go.mod h1:pZyXiIE7bGETIRXtfs0nICqMwp7PM8HqnDuyUeldNA0=
|
||||
storj.io/common v0.0.0-20210805073808-8e0feb09e92a/go.mod h1:mhZYWpTojKsACxWE66RfXNz19zbyr/uEDVWHJH8dHog=
|
||||
storj.io/common v0.0.0-20211008091048-1c580b970007/go.mod h1:objobGrIWQwhmTSpSm6Y7ykd40wZjB7CezNfic5YLKg=
|
||||
storj.io/common v0.0.0-20211021003554-f155346ccc11 h1:3UnnSkF6gC3G+MGP4jY40oFUgV4dKcwQ8ZGSpG63wsY=
|
||||
storj.io/common v0.0.0-20211021003554-f155346ccc11/go.mod h1:aJ/LW3PhU3liWU0cjQJ4EA/fXrvz6XFMfqzP+PeReOA=
|
||||
storj.io/common v0.0.0-20211019072056-34a5992b4856/go.mod h1:objobGrIWQwhmTSpSm6Y7ykd40wZjB7CezNfic5YLKg=
|
||||
storj.io/common v0.0.0-20211102144601-401a79f0706a h1:6AF6LcFbZ8PtQX6omT6npU8Yhh1g7OG2emZKuLgS5+o=
|
||||
storj.io/common v0.0.0-20211102144601-401a79f0706a/go.mod h1:a2Kw7Uipu929OFANfWKLHRoD0JfhgssikEvimd6hbSQ=
|
||||
storj.io/drpc v0.0.11/go.mod h1:TiFc2obNjL9/3isMW1Rpxjy8V9uE0B2HMeMFGiiI7Iw=
|
||||
storj.io/drpc v0.0.24/go.mod h1:ofQUDPQbbIymRDKE0tms48k8bLP5Y+dsI9CbXGv3gko=
|
||||
storj.io/drpc v0.0.26 h1:T6jJzjby7QUa/2XHR1qMxTCENpDHEw4/o+kfDfZQqQI=
|
||||
@ -891,5 +891,5 @@ storj.io/monkit-jaeger v0.0.0-20210426161729-debb1cbcbbd7 h1:zi0w9zoBfvuqysSAqxJ
|
||||
storj.io/monkit-jaeger v0.0.0-20210426161729-debb1cbcbbd7/go.mod h1:gj4vuCeyCRjRmH8LIrgoyU9Dc9uR6H+/GcDUXmTbf80=
|
||||
storj.io/private v0.0.0-20211029202355-a7eae71c382a h1:cWPChMGma5Cw5rdGuKrlc+XFxjisRVAXfa5Ny9/nxzw=
|
||||
storj.io/private v0.0.0-20211029202355-a7eae71c382a/go.mod h1:BoSaGSvsC8C6Gy0FyjrHfsElJA623hLsNIyexs6vGno=
|
||||
storj.io/uplink v1.7.1-0.20211012154306-65bb87992c7c h1:yhjQBOnEO1F6zTRkye2oSPgjQTa6Cf9h/rx/ssPFNGE=
|
||||
storj.io/uplink v1.7.1-0.20211012154306-65bb87992c7c/go.mod h1:FrGJM81ojBOv0qk2pZENtma+9ecQHNdfNnd3KAAwDwI=
|
||||
storj.io/uplink v1.7.1-0.20211031201307-b30e004c1ccb h1:+WaWPmWvm12StirZ/b2xgbL9lnB7UmRhvC8fckNN1ZI=
|
||||
storj.io/uplink v1.7.1-0.20211031201307-b30e004c1ccb/go.mod h1:XvtDUEQplm3nG76Qo+uR59J7Rrwm9Q40XOwGpPSpF2g=
|
||||
|
||||
@ -5,7 +5,6 @@ package metainfo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"go.uber.org/zap"
|
||||
@ -21,6 +20,9 @@ import (
|
||||
"storj.io/storj/satellite/console"
|
||||
)
|
||||
|
||||
// MaxUserAgentLength is the maximum allowable length of the User Agent.
|
||||
const MaxUserAgentLength = 500
|
||||
|
||||
// ensureAttribution ensures that the bucketName has the partner information specified by keyInfo partner ID or the header user agent.
|
||||
// PartnerID from keyInfo is a value associated with registered user and prevails over header user agent.
|
||||
//
|
||||
@ -54,65 +56,54 @@ func (endpoint *Endpoint) ensureAttribution(ctx context.Context, header *pb.Requ
|
||||
}
|
||||
}
|
||||
|
||||
err := endpoint.tryUpdateBucketAttribution(ctx, header, keyInfo.ProjectID, bucketName, partnerID, userAgent)
|
||||
userAgent, err := TrimUserAgent(userAgent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = endpoint.tryUpdateBucketAttribution(ctx, header, keyInfo.ProjectID, bucketName, partnerID, userAgent)
|
||||
if errs2.IsRPC(err, rpcstatus.NotFound) || errs2.IsRPC(err, rpcstatus.AlreadyExists) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// ResolvePartnerID returns partnerIDBytes as parsed or UUID corresponding to header.UserAgent.
|
||||
// returns empty uuid when neither is defined.
|
||||
func (endpoint *Endpoint) ResolvePartnerID(ctx context.Context, header *pb.RequestHeader) (uuid.UUID, error) {
|
||||
if header == nil {
|
||||
return uuid.UUID{}, rpcstatus.Error(rpcstatus.InvalidArgument, "header is nil")
|
||||
}
|
||||
|
||||
if len(header.UserAgent) == 0 {
|
||||
return uuid.UUID{}, nil
|
||||
}
|
||||
|
||||
entries, err := useragent.ParseEntries(header.UserAgent)
|
||||
// TrimUserAgent returns userAgentBytes that consist of only the product portion of the user agent, and is bounded by
|
||||
// the maxUserAgentLength.
|
||||
func TrimUserAgent(userAgent []byte) ([]byte, error) {
|
||||
userAgentEntries, err := useragent.ParseEntries(userAgent)
|
||||
if err != nil {
|
||||
return uuid.UUID{}, rpcstatus.Errorf(rpcstatus.InvalidArgument, "invalid user agent %q: %v", string(header.UserAgent), err)
|
||||
return userAgent, Error.New("error while parsing user agent: %w", err)
|
||||
}
|
||||
entries = removeUplinkUserAgent(entries)
|
||||
|
||||
// no user agent defined
|
||||
if len(entries) == 0 {
|
||||
return uuid.UUID{}, nil
|
||||
// strip comments, libraries, and empty products from the user agent
|
||||
newEntries := userAgentEntries[:0]
|
||||
for _, e := range userAgentEntries {
|
||||
switch product := e.Product; product {
|
||||
case "uplink", "common", "drpc", "":
|
||||
default:
|
||||
e.Comment = ""
|
||||
newEntries = append(newEntries, e)
|
||||
}
|
||||
}
|
||||
userAgent, err = useragent.EncodeEntries(newEntries)
|
||||
if err != nil {
|
||||
return userAgent, Error.New("error while encoding user agent entries: %w", err)
|
||||
}
|
||||
|
||||
// Use the first partner product entry as the PartnerID.
|
||||
for _, entry := range entries {
|
||||
if entry.Product != "" {
|
||||
partner, err := endpoint.partners.ByUserAgent(ctx, entry.Product)
|
||||
if err != nil || partner.UUID.IsZero() {
|
||||
continue
|
||||
// bound the user agent length
|
||||
if len(userAgent) > MaxUserAgentLength && len(newEntries) > 0 {
|
||||
// try to preserve the first entry
|
||||
if (len(newEntries[0].Product) + len(newEntries[0].Version)) <= MaxUserAgentLength {
|
||||
userAgent, err = useragent.EncodeEntries(newEntries[:1])
|
||||
if err != nil {
|
||||
return userAgent, Error.New("error while encoding first user agent entry: %w", err)
|
||||
}
|
||||
|
||||
return partner.UUID, nil
|
||||
} else {
|
||||
// first entry is too large, truncate
|
||||
userAgent = userAgent[:MaxUserAgentLength]
|
||||
}
|
||||
}
|
||||
|
||||
return uuid.UUID{}, nil
|
||||
}
|
||||
|
||||
func removeUplinkUserAgent(entries []useragent.Entry) []useragent.Entry {
|
||||
var xs []useragent.Entry
|
||||
for i := 0; i < len(entries); i++ {
|
||||
// If it's "uplink" then skip it.
|
||||
if strings.EqualFold(entries[i].Product, uplinkProduct) {
|
||||
// also skip any associated comments
|
||||
for i+1 < len(entries) && entries[i+1].Comment != "" {
|
||||
i++
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
xs = append(xs, entries[i])
|
||||
}
|
||||
return xs
|
||||
return userAgent, nil
|
||||
}
|
||||
|
||||
func (endpoint *Endpoint) tryUpdateBucketAttribution(ctx context.Context, header *pb.RequestHeader, projectID uuid.UUID, bucketName []byte, partnerID uuid.UUID, userAgent []byte) error {
|
||||
|
||||
@ -14,65 +14,53 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"storj.io/common/memory"
|
||||
"storj.io/common/pb"
|
||||
"storj.io/common/testcontext"
|
||||
"storj.io/common/testrand"
|
||||
"storj.io/common/uuid"
|
||||
"storj.io/storj/private/testplanet"
|
||||
"storj.io/storj/satellite/attribution"
|
||||
"storj.io/storj/satellite/console"
|
||||
"storj.io/storj/satellite/metainfo"
|
||||
"storj.io/uplink"
|
||||
)
|
||||
|
||||
func TestResolvePartnerID(t *testing.T) {
|
||||
testplanet.Run(t, testplanet.Config{
|
||||
SatelliteCount: 1, StorageNodeCount: 0, UplinkCount: 0,
|
||||
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
||||
endpoint := planet.Satellites[0].Metainfo.Endpoint
|
||||
|
||||
zenkoPartnerID, err := uuid.FromString("8cd605fa-ad00-45b6-823e-550eddc611d6")
|
||||
func TestTrimUserAgent(t *testing.T) {
|
||||
oversizeProduct := testrand.RandAlphaNumeric(metainfo.MaxUserAgentLength)
|
||||
oversizeVersion := testrand.RandNumeric(metainfo.MaxUserAgentLength)
|
||||
for _, tt := range []struct {
|
||||
userAgent []byte
|
||||
strippedUserAgent []byte
|
||||
}{
|
||||
{userAgent: []byte("not-a-partner"), strippedUserAgent: []byte("not-a-partner")},
|
||||
{userAgent: []byte("Zenko"), strippedUserAgent: []byte("Zenko")},
|
||||
{userAgent: []byte("Zenko uplink/v1.0.0"), strippedUserAgent: []byte("Zenko")},
|
||||
{userAgent: []byte("Zenko uplink/v1.0.0 (drpc/v0.10.0 common/v0.0.0-00010101000000-000000000000)"), strippedUserAgent: []byte("Zenko")},
|
||||
{userAgent: []byte("Zenko uplink/v1.0.0 (drpc/v0.10.0) (common/v0.0.0-00010101000000-000000000000)"), strippedUserAgent: []byte("Zenko")},
|
||||
{userAgent: []byte("uplink/v1.0.0 (drpc/v0.10.0 common/v0.0.0-00010101000000-000000000000)"), strippedUserAgent: []byte("")},
|
||||
{userAgent: []byte("uplink/v1.0.0"), strippedUserAgent: []byte("")},
|
||||
{userAgent: []byte("uplink/v1.0.0 Zenko/v3"), strippedUserAgent: []byte("Zenko/v3")},
|
||||
// oversize alphanumeric as 2nd entry product should use just the first entry
|
||||
{userAgent: append([]byte("Zenko/v3 "), oversizeProduct...), strippedUserAgent: []byte("Zenko/v3")},
|
||||
// all comments (small or oversize) should be completely removed
|
||||
{userAgent: append([]byte("Zenko ("), append(oversizeVersion, []byte(")")...)...), strippedUserAgent: []byte("Zenko")},
|
||||
// oversize version should truncate
|
||||
{userAgent: append([]byte("Zenko/v"), oversizeVersion...), strippedUserAgent: []byte("Zenko/v" + string(oversizeVersion[:len(oversizeVersion)-len("Zenko/v")]))},
|
||||
// oversize product names should truncate
|
||||
{userAgent: append([]byte("Zenko"), oversizeProduct...), strippedUserAgent: []byte("Zenko" + string(oversizeProduct[:len(oversizeProduct)-len("Zenko")]))},
|
||||
} {
|
||||
userAgent, err := metainfo.TrimUserAgent(tt.userAgent)
|
||||
require.NoError(t, err)
|
||||
|
||||
// no header
|
||||
_, err = endpoint.ResolvePartnerID(ctx, nil)
|
||||
assert.Equal(t, tt.strippedUserAgent, userAgent)
|
||||
}
|
||||
for _, tt := range []struct {
|
||||
userAgent []byte
|
||||
strippedUserAgent []byte
|
||||
}{
|
||||
{userAgent: nil, strippedUserAgent: nil},
|
||||
{userAgent: []byte(""), strippedUserAgent: []byte("")},
|
||||
} {
|
||||
_, err := metainfo.TrimUserAgent(tt.userAgent)
|
||||
require.Error(t, err)
|
||||
|
||||
partnerID, err := endpoint.ResolvePartnerID(ctx, &pb.RequestHeader{
|
||||
UserAgent: []byte("not-a-partner"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uuid.UUID{}, partnerID)
|
||||
|
||||
partnerID, err = endpoint.ResolvePartnerID(ctx, &pb.RequestHeader{
|
||||
UserAgent: []byte("Zenko"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, zenkoPartnerID, partnerID)
|
||||
|
||||
partnerID, err = endpoint.ResolvePartnerID(ctx, &pb.RequestHeader{
|
||||
UserAgent: []byte("Zenko uplink/v1.0.0"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, zenkoPartnerID, partnerID)
|
||||
|
||||
partnerID, err = endpoint.ResolvePartnerID(ctx, &pb.RequestHeader{
|
||||
UserAgent: []byte("Zenko uplink/v1.0.0 (drpc/v0.10.0 common/v0.0.0-00010101000000-000000000000)"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, zenkoPartnerID, partnerID)
|
||||
|
||||
partnerID, err = endpoint.ResolvePartnerID(ctx, &pb.RequestHeader{
|
||||
UserAgent: []byte("Zenko uplink/v1.0.0 (drpc/v0.10.0) (common/v0.0.0-00010101000000-000000000000)"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, zenkoPartnerID, partnerID)
|
||||
|
||||
partnerID, err = endpoint.ResolvePartnerID(ctx, &pb.RequestHeader{
|
||||
UserAgent: []byte("uplink/v1.0.0 (drpc/v0.10.0 common/v0.0.0-00010101000000-000000000000)"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uuid.UUID{}, partnerID)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketAttribution(t *testing.T) {
|
||||
@ -90,6 +78,7 @@ func TestBucketAttribution(t *testing.T) {
|
||||
{signupPartner: []byte("Minio"), userAgent: nil, expectedAttribution: []byte("Minio")},
|
||||
{signupPartner: []byte("Minio"), userAgent: []byte("Minio"), expectedAttribution: []byte("Minio")},
|
||||
{signupPartner: []byte("Minio"), userAgent: []byte("Zenko"), expectedAttribution: []byte("Minio")},
|
||||
{signupPartner: nil, userAgent: []byte("rclone/1.0 uplink/v1.6.1-0.20211005203254-bb2eda8c28d3"), expectedAttribution: []byte("rclone/1.0")},
|
||||
{signupPartner: nil, userAgent: []byte("Zenko"), expectedAttribution: []byte("Zenko")},
|
||||
} {
|
||||
errTag := fmt.Sprintf("%d. %+v", i, tt)
|
||||
|
||||
@ -10,10 +10,10 @@ require (
|
||||
github.com/spf13/pflag v1.0.5
|
||||
github.com/stretchr/testify v1.7.0
|
||||
go.uber.org/zap v1.17.0
|
||||
storj.io/common v0.0.0-20211021003554-f155346ccc11
|
||||
storj.io/common v0.0.0-20211102144601-401a79f0706a
|
||||
storj.io/gateway-mt v1.14.4-0.20211015103214-01eddbc864fb
|
||||
storj.io/private v0.0.0-20211029202355-a7eae71c382a
|
||||
storj.io/storj v0.12.1-0.20210916114455-b2d724962c24
|
||||
storj.io/storj v0.12.1-0.20211102170500-1de8a695e84a
|
||||
)
|
||||
|
||||
require (
|
||||
@ -231,5 +231,5 @@ require (
|
||||
storj.io/gateway v1.3.1-0.20211004141903-f55ba9105164 // indirect
|
||||
storj.io/minio v0.0.0-20211007171754-df6c27823c8a // indirect
|
||||
storj.io/monkit-jaeger v0.0.0-20210426161729-debb1cbcbbd7 // indirect
|
||||
storj.io/uplink v1.7.1-0.20211012154306-65bb87992c7c // indirect
|
||||
storj.io/uplink v1.7.1-0.20211031201307-b30e004c1ccb // indirect
|
||||
)
|
||||
|
||||
@ -1395,9 +1395,9 @@ storj.io/common v0.0.0-20200424175742-65ac59022f4f/go.mod h1:pZyXiIE7bGETIRXtfs0
|
||||
storj.io/common v0.0.0-20210805073808-8e0feb09e92a/go.mod h1:mhZYWpTojKsACxWE66RfXNz19zbyr/uEDVWHJH8dHog=
|
||||
storj.io/common v0.0.0-20210916151047-6aaeb34bb916/go.mod h1:objobGrIWQwhmTSpSm6Y7ykd40wZjB7CezNfic5YLKg=
|
||||
storj.io/common v0.0.0-20211006105453-d3fff091f9d2/go.mod h1:objobGrIWQwhmTSpSm6Y7ykd40wZjB7CezNfic5YLKg=
|
||||
storj.io/common v0.0.0-20211008091048-1c580b970007/go.mod h1:objobGrIWQwhmTSpSm6Y7ykd40wZjB7CezNfic5YLKg=
|
||||
storj.io/common v0.0.0-20211021003554-f155346ccc11 h1:3UnnSkF6gC3G+MGP4jY40oFUgV4dKcwQ8ZGSpG63wsY=
|
||||
storj.io/common v0.0.0-20211021003554-f155346ccc11/go.mod h1:aJ/LW3PhU3liWU0cjQJ4EA/fXrvz6XFMfqzP+PeReOA=
|
||||
storj.io/common v0.0.0-20211019072056-34a5992b4856/go.mod h1:objobGrIWQwhmTSpSm6Y7ykd40wZjB7CezNfic5YLKg=
|
||||
storj.io/common v0.0.0-20211102144601-401a79f0706a h1:6AF6LcFbZ8PtQX6omT6npU8Yhh1g7OG2emZKuLgS5+o=
|
||||
storj.io/common v0.0.0-20211102144601-401a79f0706a/go.mod h1:a2Kw7Uipu929OFANfWKLHRoD0JfhgssikEvimd6hbSQ=
|
||||
storj.io/dotworld v0.0.0-20210324183515-0d11aeccd840/go.mod h1:KU9YvEgRrMMiWLvH8pzn1UkoCoxggKIPvQxmNdx7aXQ=
|
||||
storj.io/drpc v0.0.11/go.mod h1:TiFc2obNjL9/3isMW1Rpxjy8V9uE0B2HMeMFGiiI7Iw=
|
||||
storj.io/drpc v0.0.24/go.mod h1:ofQUDPQbbIymRDKE0tms48k8bLP5Y+dsI9CbXGv3gko=
|
||||
@ -1418,5 +1418,5 @@ storj.io/private v0.0.0-20211029202355-a7eae71c382a h1:cWPChMGma5Cw5rdGuKrlc+XFx
|
||||
storj.io/private v0.0.0-20211029202355-a7eae71c382a/go.mod h1:BoSaGSvsC8C6Gy0FyjrHfsElJA623hLsNIyexs6vGno=
|
||||
storj.io/uplink v1.6.0/go.mod h1:zqj/LFDxa6RMaSRSHOmukg3mMgesOry0iHSjNldDMGo=
|
||||
storj.io/uplink v1.7.0/go.mod h1:zqj/LFDxa6RMaSRSHOmukg3mMgesOry0iHSjNldDMGo=
|
||||
storj.io/uplink v1.7.1-0.20211012154306-65bb87992c7c h1:yhjQBOnEO1F6zTRkye2oSPgjQTa6Cf9h/rx/ssPFNGE=
|
||||
storj.io/uplink v1.7.1-0.20211012154306-65bb87992c7c/go.mod h1:FrGJM81ojBOv0qk2pZENtma+9ecQHNdfNnd3KAAwDwI=
|
||||
storj.io/uplink v1.7.1-0.20211031201307-b30e004c1ccb h1:+WaWPmWvm12StirZ/b2xgbL9lnB7UmRhvC8fckNN1ZI=
|
||||
storj.io/uplink v1.7.1-0.20211031201307-b30e004c1ccb/go.mod h1:XvtDUEQplm3nG76Qo+uR59J7Rrwm9Q40XOwGpPSpF2g=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user