diff --git a/go.mod b/go.mod index 8b3a8f85d..46873e8e6 100644 --- a/go.mod +++ b/go.mod @@ -47,11 +47,11 @@ require ( golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e gopkg.in/segmentio/analytics-go.v3 v3.1.0 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c - storj.io/common v0.0.0-20211021003554-f155346ccc11 + storj.io/common v0.0.0-20211102144601-401a79f0706a storj.io/drpc v0.0.26 storj.io/monkit-jaeger v0.0.0-20210426161729-debb1cbcbbd7 storj.io/private v0.0.0-20211029202355-a7eae71c382a - storj.io/uplink v1.7.1-0.20211012154306-65bb87992c7c + storj.io/uplink v1.7.1-0.20211031201307-b30e004c1ccb ) require ( diff --git a/go.sum b/go.sum index dfc8411a6..774cd7d60 100644 --- a/go.sum +++ b/go.sum @@ -879,9 +879,9 @@ sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2 sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= storj.io/common v0.0.0-20200424175742-65ac59022f4f/go.mod h1:pZyXiIE7bGETIRXtfs0nICqMwp7PM8HqnDuyUeldNA0= storj.io/common v0.0.0-20210805073808-8e0feb09e92a/go.mod h1:mhZYWpTojKsACxWE66RfXNz19zbyr/uEDVWHJH8dHog= -storj.io/common v0.0.0-20211008091048-1c580b970007/go.mod h1:objobGrIWQwhmTSpSm6Y7ykd40wZjB7CezNfic5YLKg= -storj.io/common v0.0.0-20211021003554-f155346ccc11 h1:3UnnSkF6gC3G+MGP4jY40oFUgV4dKcwQ8ZGSpG63wsY= -storj.io/common v0.0.0-20211021003554-f155346ccc11/go.mod h1:aJ/LW3PhU3liWU0cjQJ4EA/fXrvz6XFMfqzP+PeReOA= +storj.io/common v0.0.0-20211019072056-34a5992b4856/go.mod h1:objobGrIWQwhmTSpSm6Y7ykd40wZjB7CezNfic5YLKg= +storj.io/common v0.0.0-20211102144601-401a79f0706a h1:6AF6LcFbZ8PtQX6omT6npU8Yhh1g7OG2emZKuLgS5+o= +storj.io/common v0.0.0-20211102144601-401a79f0706a/go.mod h1:a2Kw7Uipu929OFANfWKLHRoD0JfhgssikEvimd6hbSQ= storj.io/drpc v0.0.11/go.mod h1:TiFc2obNjL9/3isMW1Rpxjy8V9uE0B2HMeMFGiiI7Iw= storj.io/drpc v0.0.24/go.mod h1:ofQUDPQbbIymRDKE0tms48k8bLP5Y+dsI9CbXGv3gko= storj.io/drpc v0.0.26 h1:T6jJzjby7QUa/2XHR1qMxTCENpDHEw4/o+kfDfZQqQI= @@ -891,5 +891,5 @@ storj.io/monkit-jaeger v0.0.0-20210426161729-debb1cbcbbd7 h1:zi0w9zoBfvuqysSAqxJ storj.io/monkit-jaeger v0.0.0-20210426161729-debb1cbcbbd7/go.mod h1:gj4vuCeyCRjRmH8LIrgoyU9Dc9uR6H+/GcDUXmTbf80= storj.io/private v0.0.0-20211029202355-a7eae71c382a h1:cWPChMGma5Cw5rdGuKrlc+XFxjisRVAXfa5Ny9/nxzw= storj.io/private v0.0.0-20211029202355-a7eae71c382a/go.mod h1:BoSaGSvsC8C6Gy0FyjrHfsElJA623hLsNIyexs6vGno= -storj.io/uplink v1.7.1-0.20211012154306-65bb87992c7c h1:yhjQBOnEO1F6zTRkye2oSPgjQTa6Cf9h/rx/ssPFNGE= -storj.io/uplink v1.7.1-0.20211012154306-65bb87992c7c/go.mod h1:FrGJM81ojBOv0qk2pZENtma+9ecQHNdfNnd3KAAwDwI= +storj.io/uplink v1.7.1-0.20211031201307-b30e004c1ccb h1:+WaWPmWvm12StirZ/b2xgbL9lnB7UmRhvC8fckNN1ZI= +storj.io/uplink v1.7.1-0.20211031201307-b30e004c1ccb/go.mod h1:XvtDUEQplm3nG76Qo+uR59J7Rrwm9Q40XOwGpPSpF2g= diff --git a/satellite/metainfo/attribution.go b/satellite/metainfo/attribution.go index ef963f910..cad16dbfc 100644 --- a/satellite/metainfo/attribution.go +++ b/satellite/metainfo/attribution.go @@ -5,7 +5,6 @@ package metainfo import ( "context" - "strings" "sync" "go.uber.org/zap" @@ -21,6 +20,9 @@ import ( "storj.io/storj/satellite/console" ) +// MaxUserAgentLength is the maximum allowable length of the User Agent. +const MaxUserAgentLength = 500 + // ensureAttribution ensures that the bucketName has the partner information specified by keyInfo partner ID or the header user agent. // PartnerID from keyInfo is a value associated with registered user and prevails over header user agent. // @@ -54,65 +56,54 @@ func (endpoint *Endpoint) ensureAttribution(ctx context.Context, header *pb.Requ } } - err := endpoint.tryUpdateBucketAttribution(ctx, header, keyInfo.ProjectID, bucketName, partnerID, userAgent) + userAgent, err := TrimUserAgent(userAgent) + if err != nil { + return err + } + + err = endpoint.tryUpdateBucketAttribution(ctx, header, keyInfo.ProjectID, bucketName, partnerID, userAgent) if errs2.IsRPC(err, rpcstatus.NotFound) || errs2.IsRPC(err, rpcstatus.AlreadyExists) { return nil } return err } -// ResolvePartnerID returns partnerIDBytes as parsed or UUID corresponding to header.UserAgent. -// returns empty uuid when neither is defined. -func (endpoint *Endpoint) ResolvePartnerID(ctx context.Context, header *pb.RequestHeader) (uuid.UUID, error) { - if header == nil { - return uuid.UUID{}, rpcstatus.Error(rpcstatus.InvalidArgument, "header is nil") - } - - if len(header.UserAgent) == 0 { - return uuid.UUID{}, nil - } - - entries, err := useragent.ParseEntries(header.UserAgent) +// TrimUserAgent returns userAgentBytes that consist of only the product portion of the user agent, and is bounded by +// the maxUserAgentLength. +func TrimUserAgent(userAgent []byte) ([]byte, error) { + userAgentEntries, err := useragent.ParseEntries(userAgent) if err != nil { - return uuid.UUID{}, rpcstatus.Errorf(rpcstatus.InvalidArgument, "invalid user agent %q: %v", string(header.UserAgent), err) + return userAgent, Error.New("error while parsing user agent: %w", err) } - entries = removeUplinkUserAgent(entries) - - // no user agent defined - if len(entries) == 0 { - return uuid.UUID{}, nil - } - - // Use the first partner product entry as the PartnerID. - for _, entry := range entries { - if entry.Product != "" { - partner, err := endpoint.partners.ByUserAgent(ctx, entry.Product) - if err != nil || partner.UUID.IsZero() { - continue - } - - return partner.UUID, nil + // strip comments, libraries, and empty products from the user agent + newEntries := userAgentEntries[:0] + for _, e := range userAgentEntries { + switch product := e.Product; product { + case "uplink", "common", "drpc", "": + default: + e.Comment = "" + newEntries = append(newEntries, e) } } - - return uuid.UUID{}, nil -} - -func removeUplinkUserAgent(entries []useragent.Entry) []useragent.Entry { - var xs []useragent.Entry - for i := 0; i < len(entries); i++ { - // If it's "uplink" then skip it. - if strings.EqualFold(entries[i].Product, uplinkProduct) { - // also skip any associated comments - for i+1 < len(entries) && entries[i+1].Comment != "" { - i++ - } - continue - } - - xs = append(xs, entries[i]) + userAgent, err = useragent.EncodeEntries(newEntries) + if err != nil { + return userAgent, Error.New("error while encoding user agent entries: %w", err) } - return xs + + // bound the user agent length + if len(userAgent) > MaxUserAgentLength && len(newEntries) > 0 { + // try to preserve the first entry + if (len(newEntries[0].Product) + len(newEntries[0].Version)) <= MaxUserAgentLength { + userAgent, err = useragent.EncodeEntries(newEntries[:1]) + if err != nil { + return userAgent, Error.New("error while encoding first user agent entry: %w", err) + } + } else { + // first entry is too large, truncate + userAgent = userAgent[:MaxUserAgentLength] + } + } + return userAgent, nil } func (endpoint *Endpoint) tryUpdateBucketAttribution(ctx context.Context, header *pb.RequestHeader, projectID uuid.UUID, bucketName []byte, partnerID uuid.UUID, userAgent []byte) error { diff --git a/satellite/metainfo/attribution_test.go b/satellite/metainfo/attribution_test.go index 623279a57..fedc642fd 100644 --- a/satellite/metainfo/attribution_test.go +++ b/satellite/metainfo/attribution_test.go @@ -14,65 +14,53 @@ import ( "github.com/stretchr/testify/require" "storj.io/common/memory" - "storj.io/common/pb" "storj.io/common/testcontext" "storj.io/common/testrand" - "storj.io/common/uuid" "storj.io/storj/private/testplanet" "storj.io/storj/satellite/attribution" "storj.io/storj/satellite/console" + "storj.io/storj/satellite/metainfo" "storj.io/uplink" ) -func TestResolvePartnerID(t *testing.T) { - testplanet.Run(t, testplanet.Config{ - SatelliteCount: 1, StorageNodeCount: 0, UplinkCount: 0, - }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { - endpoint := planet.Satellites[0].Metainfo.Endpoint - - zenkoPartnerID, err := uuid.FromString("8cd605fa-ad00-45b6-823e-550eddc611d6") +func TestTrimUserAgent(t *testing.T) { + oversizeProduct := testrand.RandAlphaNumeric(metainfo.MaxUserAgentLength) + oversizeVersion := testrand.RandNumeric(metainfo.MaxUserAgentLength) + for _, tt := range []struct { + userAgent []byte + strippedUserAgent []byte + }{ + {userAgent: []byte("not-a-partner"), strippedUserAgent: []byte("not-a-partner")}, + {userAgent: []byte("Zenko"), strippedUserAgent: []byte("Zenko")}, + {userAgent: []byte("Zenko uplink/v1.0.0"), strippedUserAgent: []byte("Zenko")}, + {userAgent: []byte("Zenko uplink/v1.0.0 (drpc/v0.10.0 common/v0.0.0-00010101000000-000000000000)"), strippedUserAgent: []byte("Zenko")}, + {userAgent: []byte("Zenko uplink/v1.0.0 (drpc/v0.10.0) (common/v0.0.0-00010101000000-000000000000)"), strippedUserAgent: []byte("Zenko")}, + {userAgent: []byte("uplink/v1.0.0 (drpc/v0.10.0 common/v0.0.0-00010101000000-000000000000)"), strippedUserAgent: []byte("")}, + {userAgent: []byte("uplink/v1.0.0"), strippedUserAgent: []byte("")}, + {userAgent: []byte("uplink/v1.0.0 Zenko/v3"), strippedUserAgent: []byte("Zenko/v3")}, + // oversize alphanumeric as 2nd entry product should use just the first entry + {userAgent: append([]byte("Zenko/v3 "), oversizeProduct...), strippedUserAgent: []byte("Zenko/v3")}, + // all comments (small or oversize) should be completely removed + {userAgent: append([]byte("Zenko ("), append(oversizeVersion, []byte(")")...)...), strippedUserAgent: []byte("Zenko")}, + // oversize version should truncate + {userAgent: append([]byte("Zenko/v"), oversizeVersion...), strippedUserAgent: []byte("Zenko/v" + string(oversizeVersion[:len(oversizeVersion)-len("Zenko/v")]))}, + // oversize product names should truncate + {userAgent: append([]byte("Zenko"), oversizeProduct...), strippedUserAgent: []byte("Zenko" + string(oversizeProduct[:len(oversizeProduct)-len("Zenko")]))}, + } { + userAgent, err := metainfo.TrimUserAgent(tt.userAgent) require.NoError(t, err) - - // no header - _, err = endpoint.ResolvePartnerID(ctx, nil) + assert.Equal(t, tt.strippedUserAgent, userAgent) + } + for _, tt := range []struct { + userAgent []byte + strippedUserAgent []byte + }{ + {userAgent: nil, strippedUserAgent: nil}, + {userAgent: []byte(""), strippedUserAgent: []byte("")}, + } { + _, err := metainfo.TrimUserAgent(tt.userAgent) require.Error(t, err) - - partnerID, err := endpoint.ResolvePartnerID(ctx, &pb.RequestHeader{ - UserAgent: []byte("not-a-partner"), - }) - require.NoError(t, err) - require.Equal(t, uuid.UUID{}, partnerID) - - partnerID, err = endpoint.ResolvePartnerID(ctx, &pb.RequestHeader{ - UserAgent: []byte("Zenko"), - }) - require.NoError(t, err) - require.Equal(t, zenkoPartnerID, partnerID) - - partnerID, err = endpoint.ResolvePartnerID(ctx, &pb.RequestHeader{ - UserAgent: []byte("Zenko uplink/v1.0.0"), - }) - require.NoError(t, err) - require.Equal(t, zenkoPartnerID, partnerID) - - partnerID, err = endpoint.ResolvePartnerID(ctx, &pb.RequestHeader{ - UserAgent: []byte("Zenko uplink/v1.0.0 (drpc/v0.10.0 common/v0.0.0-00010101000000-000000000000)"), - }) - require.NoError(t, err) - require.Equal(t, zenkoPartnerID, partnerID) - - partnerID, err = endpoint.ResolvePartnerID(ctx, &pb.RequestHeader{ - UserAgent: []byte("Zenko uplink/v1.0.0 (drpc/v0.10.0) (common/v0.0.0-00010101000000-000000000000)"), - }) - require.NoError(t, err) - require.Equal(t, zenkoPartnerID, partnerID) - - partnerID, err = endpoint.ResolvePartnerID(ctx, &pb.RequestHeader{ - UserAgent: []byte("uplink/v1.0.0 (drpc/v0.10.0 common/v0.0.0-00010101000000-000000000000)"), - }) - require.NoError(t, err) - require.Equal(t, uuid.UUID{}, partnerID) - }) + } } func TestBucketAttribution(t *testing.T) { @@ -90,6 +78,7 @@ func TestBucketAttribution(t *testing.T) { {signupPartner: []byte("Minio"), userAgent: nil, expectedAttribution: []byte("Minio")}, {signupPartner: []byte("Minio"), userAgent: []byte("Minio"), expectedAttribution: []byte("Minio")}, {signupPartner: []byte("Minio"), userAgent: []byte("Zenko"), expectedAttribution: []byte("Minio")}, + {signupPartner: nil, userAgent: []byte("rclone/1.0 uplink/v1.6.1-0.20211005203254-bb2eda8c28d3"), expectedAttribution: []byte("rclone/1.0")}, {signupPartner: nil, userAgent: []byte("Zenko"), expectedAttribution: []byte("Zenko")}, } { errTag := fmt.Sprintf("%d. %+v", i, tt) diff --git a/testsuite/go.mod b/testsuite/go.mod index 34a3ab632..53b93efc5 100644 --- a/testsuite/go.mod +++ b/testsuite/go.mod @@ -10,10 +10,10 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 go.uber.org/zap v1.17.0 - storj.io/common v0.0.0-20211021003554-f155346ccc11 + storj.io/common v0.0.0-20211102144601-401a79f0706a storj.io/gateway-mt v1.14.4-0.20211015103214-01eddbc864fb storj.io/private v0.0.0-20211029202355-a7eae71c382a - storj.io/storj v0.12.1-0.20210916114455-b2d724962c24 + storj.io/storj v0.12.1-0.20211102170500-1de8a695e84a ) require ( @@ -231,5 +231,5 @@ require ( storj.io/gateway v1.3.1-0.20211004141903-f55ba9105164 // indirect storj.io/minio v0.0.0-20211007171754-df6c27823c8a // indirect storj.io/monkit-jaeger v0.0.0-20210426161729-debb1cbcbbd7 // indirect - storj.io/uplink v1.7.1-0.20211012154306-65bb87992c7c // indirect + storj.io/uplink v1.7.1-0.20211031201307-b30e004c1ccb // indirect ) diff --git a/testsuite/go.sum b/testsuite/go.sum index 857055406..3a75ba769 100644 --- a/testsuite/go.sum +++ b/testsuite/go.sum @@ -1395,9 +1395,9 @@ storj.io/common v0.0.0-20200424175742-65ac59022f4f/go.mod h1:pZyXiIE7bGETIRXtfs0 storj.io/common v0.0.0-20210805073808-8e0feb09e92a/go.mod h1:mhZYWpTojKsACxWE66RfXNz19zbyr/uEDVWHJH8dHog= storj.io/common v0.0.0-20210916151047-6aaeb34bb916/go.mod h1:objobGrIWQwhmTSpSm6Y7ykd40wZjB7CezNfic5YLKg= storj.io/common v0.0.0-20211006105453-d3fff091f9d2/go.mod h1:objobGrIWQwhmTSpSm6Y7ykd40wZjB7CezNfic5YLKg= -storj.io/common v0.0.0-20211008091048-1c580b970007/go.mod h1:objobGrIWQwhmTSpSm6Y7ykd40wZjB7CezNfic5YLKg= -storj.io/common v0.0.0-20211021003554-f155346ccc11 h1:3UnnSkF6gC3G+MGP4jY40oFUgV4dKcwQ8ZGSpG63wsY= -storj.io/common v0.0.0-20211021003554-f155346ccc11/go.mod h1:aJ/LW3PhU3liWU0cjQJ4EA/fXrvz6XFMfqzP+PeReOA= +storj.io/common v0.0.0-20211019072056-34a5992b4856/go.mod h1:objobGrIWQwhmTSpSm6Y7ykd40wZjB7CezNfic5YLKg= +storj.io/common v0.0.0-20211102144601-401a79f0706a h1:6AF6LcFbZ8PtQX6omT6npU8Yhh1g7OG2emZKuLgS5+o= +storj.io/common v0.0.0-20211102144601-401a79f0706a/go.mod h1:a2Kw7Uipu929OFANfWKLHRoD0JfhgssikEvimd6hbSQ= storj.io/dotworld v0.0.0-20210324183515-0d11aeccd840/go.mod h1:KU9YvEgRrMMiWLvH8pzn1UkoCoxggKIPvQxmNdx7aXQ= storj.io/drpc v0.0.11/go.mod h1:TiFc2obNjL9/3isMW1Rpxjy8V9uE0B2HMeMFGiiI7Iw= storj.io/drpc v0.0.24/go.mod h1:ofQUDPQbbIymRDKE0tms48k8bLP5Y+dsI9CbXGv3gko= @@ -1418,5 +1418,5 @@ storj.io/private v0.0.0-20211029202355-a7eae71c382a h1:cWPChMGma5Cw5rdGuKrlc+XFx storj.io/private v0.0.0-20211029202355-a7eae71c382a/go.mod h1:BoSaGSvsC8C6Gy0FyjrHfsElJA623hLsNIyexs6vGno= storj.io/uplink v1.6.0/go.mod h1:zqj/LFDxa6RMaSRSHOmukg3mMgesOry0iHSjNldDMGo= storj.io/uplink v1.7.0/go.mod h1:zqj/LFDxa6RMaSRSHOmukg3mMgesOry0iHSjNldDMGo= -storj.io/uplink v1.7.1-0.20211012154306-65bb87992c7c h1:yhjQBOnEO1F6zTRkye2oSPgjQTa6Cf9h/rx/ssPFNGE= -storj.io/uplink v1.7.1-0.20211012154306-65bb87992c7c/go.mod h1:FrGJM81ojBOv0qk2pZENtma+9ecQHNdfNnd3KAAwDwI= +storj.io/uplink v1.7.1-0.20211031201307-b30e004c1ccb h1:+WaWPmWvm12StirZ/b2xgbL9lnB7UmRhvC8fckNN1ZI= +storj.io/uplink v1.7.1-0.20211031201307-b30e004c1ccb/go.mod h1:XvtDUEQplm3nG76Qo+uR59J7Rrwm9Q40XOwGpPSpF2g=