5272fd8497
We are doing full bucket name validation for many requests but we should do this only while creating bucket. Other requests will be covered only by basic name length validation. Less strict validation for other requests will make bucket usable in case of invalid bucket names in DB (we have such cases from the past). https://github.com/storj/storj/issues/6044 Change-Id: I3a41050e3637787f788705ef15b5dc4df4d01fc6
523 lines
16 KiB
Go
523 lines
16 KiB
Go
// Copyright (C) 2019 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package metainfo
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/subtle"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/jtolio/eventkit"
|
|
"github.com/zeebo/errs"
|
|
"go.uber.org/zap"
|
|
"golang.org/x/time/rate"
|
|
|
|
"storj.io/common/encryption"
|
|
"storj.io/common/errs2"
|
|
"storj.io/common/macaroon"
|
|
"storj.io/common/memory"
|
|
"storj.io/common/pb"
|
|
"storj.io/common/rpc/rpcstatus"
|
|
"storj.io/common/storj"
|
|
"storj.io/common/uuid"
|
|
"storj.io/storj/satellite/accounting"
|
|
"storj.io/storj/satellite/buckets"
|
|
"storj.io/storj/satellite/console"
|
|
"storj.io/storj/satellite/console/consoleauth"
|
|
"storj.io/storj/satellite/metabase"
|
|
)
|
|
|
|
const encryptedKeySize = 48
|
|
|
|
var (
|
|
ipRegexp = regexp.MustCompile(`^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$`)
|
|
)
|
|
|
|
var ek = eventkit.Package()
|
|
|
|
func getAPIKey(ctx context.Context, header *pb.RequestHeader) (key *macaroon.APIKey, err error) {
|
|
defer mon.Task()(&ctx)(&err)
|
|
if header != nil {
|
|
return macaroon.ParseRawAPIKey(header.ApiKey)
|
|
}
|
|
|
|
keyData, ok := consoleauth.GetAPIKey(ctx)
|
|
if !ok {
|
|
return nil, errs.New("missing credentials")
|
|
}
|
|
|
|
return macaroon.ParseAPIKey(string(keyData))
|
|
}
|
|
|
|
// validateAuth validates things like API key, user permissions and rate limit and always returns valid rpc error.
|
|
func (endpoint *Endpoint) validateAuth(ctx context.Context, header *pb.RequestHeader, action macaroon.Action) (_ *console.APIKeyInfo, err error) {
|
|
defer mon.Task()(&ctx)(&err)
|
|
|
|
key, keyInfo, err := endpoint.validateBasic(ctx, header)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
err = key.Check(ctx, keyInfo.Secret, action, endpoint.revocations)
|
|
if err != nil {
|
|
endpoint.log.Debug("unauthorized request", zap.Error(err))
|
|
return nil, rpcstatus.Error(rpcstatus.PermissionDenied, "Unauthorized API credentials")
|
|
}
|
|
|
|
return keyInfo, nil
|
|
}
|
|
|
|
type verifyPermission struct {
|
|
action macaroon.Action
|
|
actionPermitted *bool
|
|
optional bool
|
|
}
|
|
|
|
// validateAuthN validates things like API keys, rate limit and user permissions
|
|
// for each permission from permissions. It returns an error for the first
|
|
// required (not optional) permission that the check fails for. There must be at
|
|
// least one required (not optional) permission. In case all permissions are
|
|
// optional, it will return an error. It always returns valid RPC errors.
|
|
func (endpoint *Endpoint) validateAuthN(ctx context.Context, header *pb.RequestHeader, permissions ...verifyPermission) (_ *console.APIKeyInfo, err error) {
|
|
defer mon.Task()(&ctx)(&err)
|
|
|
|
allOptional := true
|
|
|
|
for _, p := range permissions {
|
|
if !p.optional {
|
|
allOptional = false
|
|
break
|
|
}
|
|
}
|
|
|
|
if allOptional {
|
|
return nil, rpcstatus.Error(rpcstatus.Internal, "All permissions are optional")
|
|
}
|
|
|
|
key, keyInfo, err := endpoint.validateBasic(ctx, header)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
for _, p := range permissions {
|
|
err = key.Check(ctx, keyInfo.Secret, p.action, endpoint.revocations)
|
|
if p.actionPermitted != nil {
|
|
*p.actionPermitted = err == nil
|
|
}
|
|
if err != nil && !p.optional {
|
|
endpoint.log.Debug("unauthorized request", zap.Error(err))
|
|
return nil, rpcstatus.Error(rpcstatus.PermissionDenied, "Unauthorized API credentials")
|
|
}
|
|
}
|
|
|
|
return keyInfo, nil
|
|
}
|
|
|
|
// validateAuthAny validates things like API keys, rate limit and user permissions.
|
|
// At least one of the action from actions must be permitted to return successfully.
|
|
// It always returns valid RPC errors.
|
|
func (endpoint *Endpoint) validateAuthAny(ctx context.Context, header *pb.RequestHeader, actions ...macaroon.Action) (_ *console.APIKeyInfo, err error) {
|
|
defer mon.Task()(&ctx)(&err)
|
|
|
|
key, keyInfo, err := endpoint.validateBasic(ctx, header)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if len(actions) == 0 {
|
|
return nil, rpcstatus.Error(rpcstatus.Internal, "No action to validate")
|
|
}
|
|
|
|
var combinedErrs error
|
|
for _, action := range actions {
|
|
err = key.Check(ctx, keyInfo.Secret, action, endpoint.revocations)
|
|
if err == nil {
|
|
return keyInfo, nil
|
|
}
|
|
combinedErrs = errs.Combine(combinedErrs, err)
|
|
}
|
|
|
|
endpoint.log.Debug("unauthorized request", zap.Error(combinedErrs))
|
|
return nil, rpcstatus.Error(rpcstatus.PermissionDenied, "Unauthorized API credentials")
|
|
}
|
|
|
|
func (endpoint *Endpoint) validateBasic(ctx context.Context, header *pb.RequestHeader) (_ *macaroon.APIKey, _ *console.APIKeyInfo, err error) {
|
|
defer mon.Task()(&ctx)(&err)
|
|
|
|
key, err := getAPIKey(ctx, header)
|
|
if err != nil {
|
|
endpoint.log.Debug("invalid request", zap.Error(err))
|
|
return nil, nil, rpcstatus.Error(rpcstatus.InvalidArgument, "Invalid API credentials")
|
|
}
|
|
|
|
keyInfo, err := endpoint.apiKeys.GetByHead(ctx, key.Head())
|
|
if err != nil {
|
|
endpoint.log.Debug("unauthorized request", zap.Error(err))
|
|
return nil, nil, rpcstatus.Error(rpcstatus.PermissionDenied, "Unauthorized API credentials")
|
|
}
|
|
|
|
userAgent := ""
|
|
if keyInfo.UserAgent != nil {
|
|
userAgent = string(keyInfo.UserAgent)
|
|
}
|
|
ek.Event("auth",
|
|
eventkit.String("user-agent", userAgent),
|
|
eventkit.String("project", keyInfo.ProjectID.String()),
|
|
eventkit.String("partner", string(keyInfo.UserAgent)),
|
|
)
|
|
|
|
if err = endpoint.checkRate(ctx, keyInfo.ProjectID); err != nil {
|
|
endpoint.log.Debug("rate check failed", zap.Error(err))
|
|
return nil, nil, err
|
|
}
|
|
|
|
return key, keyInfo, nil
|
|
}
|
|
|
|
func (endpoint *Endpoint) validateRevoke(ctx context.Context, header *pb.RequestHeader, macToRevoke *macaroon.Macaroon) (_ *console.APIKeyInfo, err error) {
|
|
defer mon.Task()(&ctx)(&err)
|
|
key, keyInfo, err := endpoint.validateBasic(ctx, header)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// The macaroon to revoke must be valid with the same secret as the key.
|
|
if !macToRevoke.Validate(keyInfo.Secret) {
|
|
return nil, rpcstatus.Error(rpcstatus.InvalidArgument, "Macaroon to revoke invalid")
|
|
}
|
|
|
|
keyTail := key.Tail()
|
|
tails := macToRevoke.Tails(keyInfo.Secret)
|
|
|
|
// A macaroon cannot revoke itself. So we only check len(tails-1), skipping
|
|
// the final tail. To be valid, the final tail of the auth key must be
|
|
// contained within the checked tails of the macaroon we want to revoke.
|
|
for i := 0; i < len(tails)-1; i++ {
|
|
if subtle.ConstantTimeCompare(tails[i], keyTail) == 1 {
|
|
return keyInfo, nil
|
|
}
|
|
}
|
|
return nil, rpcstatus.Error(rpcstatus.PermissionDenied, "Unauthorized attempt to revoke macaroon")
|
|
}
|
|
|
|
func (endpoint *Endpoint) checkRate(ctx context.Context, projectID uuid.UUID) (err error) {
|
|
defer mon.Task()(&ctx)(&err)
|
|
if !endpoint.config.RateLimiter.Enabled {
|
|
return nil
|
|
}
|
|
limiter, err := endpoint.limiterCache.Get(ctx, projectID.String(), func() (*rate.Limiter, error) {
|
|
rateLimit := rate.Limit(endpoint.config.RateLimiter.Rate)
|
|
burstLimit := int(endpoint.config.RateLimiter.Rate)
|
|
|
|
limits, err := endpoint.projectLimits.GetLimits(ctx, projectID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if limits.RateLimit != nil {
|
|
rateLimit = rate.Limit(*limits.RateLimit)
|
|
burstLimit = *limits.RateLimit
|
|
}
|
|
// use the explicitly set burst value if it's defined
|
|
if limits.BurstLimit != nil {
|
|
burstLimit = *limits.BurstLimit
|
|
}
|
|
|
|
return rate.NewLimiter(rateLimit, burstLimit), nil
|
|
})
|
|
if err != nil {
|
|
return rpcstatus.Error(rpcstatus.Unavailable, err.Error())
|
|
}
|
|
|
|
if !limiter.Allow() {
|
|
endpoint.log.Warn("too many requests for project",
|
|
zap.Stringer("projectID", projectID),
|
|
zap.Float64("rate limit", float64(limiter.Limit())),
|
|
zap.Float64("burst limit", float64(limiter.Burst())))
|
|
|
|
mon.Event("metainfo_rate_limit_exceeded") //mon:locked
|
|
|
|
return rpcstatus.Error(rpcstatus.ResourceExhausted, "Too Many Requests")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (endpoint *Endpoint) validateBucketNameLength(bucket []byte) (err error) {
|
|
if len(bucket) == 0 {
|
|
return Error.Wrap(buckets.ErrNoBucket.New(""))
|
|
}
|
|
|
|
if len(bucket) < 3 || len(bucket) > 63 {
|
|
return Error.New("bucket name must be at least 3 and no more than 63 characters long")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (endpoint *Endpoint) validateBucketName(bucket []byte) error {
|
|
if err := endpoint.validateBucketNameLength(bucket); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Regexp not used because benchmark shows it will be slower for valid bucket names
|
|
// https://gist.github.com/mniewrzal/49de3af95f36e63e88fac24f565e444c
|
|
labels := bytes.Split(bucket, []byte("."))
|
|
for _, label := range labels {
|
|
err := validateBucketLabel(label)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if ipRegexp.MatchString(string(bucket)) {
|
|
return Error.New("bucket name cannot be formatted as an IP address")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func validateBucketLabel(label []byte) error {
|
|
if len(label) == 0 {
|
|
return Error.New("bucket label cannot be empty")
|
|
}
|
|
|
|
if !isLowerLetter(label[0]) && !isDigit(label[0]) {
|
|
return Error.New("bucket label must start with a lowercase letter or number")
|
|
}
|
|
|
|
if !isLowerLetter(label[len(label)-1]) && !isDigit(label[len(label)-1]) {
|
|
return Error.New("bucket label must end with a lowercase letter or number")
|
|
}
|
|
|
|
for i := 1; i < len(label)-1; i++ {
|
|
if !isLowerLetter(label[i]) && !isDigit(label[i]) && (label[i] != '-') && (label[i] != '.') {
|
|
return Error.New("bucket name must contain only lowercase letters, numbers or hyphens")
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func isLowerLetter(r byte) bool {
|
|
return r >= 'a' && r <= 'z'
|
|
}
|
|
|
|
func isDigit(r byte) bool {
|
|
return r >= '0' && r <= '9'
|
|
}
|
|
|
|
func (endpoint *Endpoint) validateRemoteSegment(ctx context.Context, commitRequest metabase.CommitSegment, originalLimits []*pb.OrderLimit) (err error) {
|
|
defer mon.Task()(&ctx)(&err)
|
|
|
|
if len(originalLimits) == 0 {
|
|
return Error.New("no order limits")
|
|
}
|
|
if len(originalLimits) != int(commitRequest.Redundancy.TotalShares) {
|
|
return Error.New("invalid no order limit for piece")
|
|
}
|
|
|
|
maxAllowed, err := encryption.CalcEncryptedSize(endpoint.config.MaxSegmentSize.Int64(), storj.EncryptionParameters{
|
|
CipherSuite: storj.EncAESGCM,
|
|
BlockSize: 128, // intentionally low block size to allow maximum possible encryption overhead
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if int64(commitRequest.EncryptedSize) > maxAllowed || commitRequest.EncryptedSize < 0 {
|
|
return Error.New("encrypted segment size %v is out of range, maximum allowed is %v", commitRequest.EncryptedSize, maxAllowed)
|
|
}
|
|
|
|
// TODO more validation for plain size and plain offset
|
|
if commitRequest.PlainSize > commitRequest.EncryptedSize {
|
|
return Error.New("plain segment size %v is out of range, maximum allowed is %v", commitRequest.PlainSize, commitRequest.EncryptedSize)
|
|
}
|
|
|
|
pieceNums := make(map[uint16]struct{})
|
|
nodeIds := make(map[storj.NodeID]struct{})
|
|
deriver := commitRequest.RootPieceID.Deriver()
|
|
for _, piece := range commitRequest.Pieces {
|
|
if int(piece.Number) >= len(originalLimits) {
|
|
return Error.New("invalid piece number")
|
|
}
|
|
|
|
limit := originalLimits[piece.Number]
|
|
if limit == nil {
|
|
return Error.New("empty order limit for piece")
|
|
}
|
|
|
|
err := endpoint.orders.VerifyOrderLimitSignature(ctx, limit)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// expect that too much time has not passed between order limit creation and now
|
|
if time.Since(limit.OrderCreation) > endpoint.config.MaxCommitInterval {
|
|
return Error.New("Segment not committed before max commit interval of %f minutes.", endpoint.config.MaxCommitInterval.Minutes())
|
|
}
|
|
|
|
derivedPieceID := deriver.Derive(piece.StorageNode, int32(piece.Number))
|
|
if limit.PieceId.IsZero() || limit.PieceId != derivedPieceID {
|
|
return Error.New("invalid order limit piece id")
|
|
}
|
|
if piece.StorageNode != limit.StorageNodeId {
|
|
return Error.New("piece NodeID != order limit NodeID")
|
|
}
|
|
|
|
if _, ok := pieceNums[piece.Number]; ok {
|
|
return Error.New("piece num %d is duplicated", piece.Number)
|
|
}
|
|
|
|
if _, ok := nodeIds[piece.StorageNode]; ok {
|
|
return Error.New("node id %s for piece num %d is duplicated", piece.StorageNode.String(), piece.Number)
|
|
}
|
|
|
|
pieceNums[piece.Number] = struct{}{}
|
|
nodeIds[piece.StorageNode] = struct{}{}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (endpoint *Endpoint) checkUploadLimits(ctx context.Context, projectID uuid.UUID) error {
|
|
return endpoint.checkUploadLimitsForNewObject(ctx, projectID, 1, 1)
|
|
}
|
|
|
|
func (endpoint *Endpoint) checkUploadLimitsForNewObject(
|
|
ctx context.Context, projectID uuid.UUID, newObjectSize int64, newObjectSegmentCount int64,
|
|
) error {
|
|
if limit, err := endpoint.projectUsage.ExceedsUploadLimits(ctx, projectID, newObjectSize, newObjectSegmentCount); err != nil {
|
|
if errs2.IsCanceled(err) {
|
|
return rpcstatus.Wrap(rpcstatus.Canceled, err)
|
|
}
|
|
|
|
endpoint.log.Error(
|
|
"Retrieving project upload limit failed; limit won't be enforced",
|
|
zap.Stringer("Project ID", projectID),
|
|
zap.Error(err),
|
|
)
|
|
} else {
|
|
if limit.ExceedsSegments {
|
|
endpoint.log.Warn("Segment limit exceeded",
|
|
zap.String("Limit", strconv.Itoa(int(limit.SegmentsLimit))),
|
|
zap.Stringer("Project ID", projectID),
|
|
)
|
|
return rpcstatus.Error(rpcstatus.ResourceExhausted, "Exceeded Segments Limit")
|
|
}
|
|
|
|
if limit.ExceedsStorage {
|
|
endpoint.log.Warn("Storage limit exceeded",
|
|
zap.String("Limit", strconv.Itoa(limit.StorageLimit.Int())),
|
|
zap.Stringer("Project ID", projectID),
|
|
)
|
|
return rpcstatus.Error(rpcstatus.ResourceExhausted, "Exceeded Storage Limit")
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (endpoint *Endpoint) addSegmentToUploadLimits(ctx context.Context, projectID uuid.UUID, segmentSize int64) error {
|
|
return endpoint.addToUploadLimits(ctx, projectID, segmentSize, 1)
|
|
}
|
|
|
|
func (endpoint *Endpoint) addToUploadLimits(ctx context.Context, projectID uuid.UUID, size int64, segmentCount int64) error {
|
|
if err := endpoint.projectUsage.AddProjectStorageUsage(ctx, projectID, size); err != nil {
|
|
if errs2.IsCanceled(err) {
|
|
return rpcstatus.Wrap(rpcstatus.Canceled, err)
|
|
}
|
|
|
|
// log it and continue. it's most likely our own fault that we couldn't
|
|
// track it, and the only thing that will be affected is our per-project
|
|
// bandwidth and storage limits.
|
|
endpoint.log.Error("Could not track new project's storage usage",
|
|
zap.Stringer("Project ID", projectID),
|
|
zap.Error(err),
|
|
)
|
|
}
|
|
|
|
err := endpoint.projectUsage.UpdateProjectSegmentUsage(ctx, projectID, segmentCount)
|
|
if err != nil {
|
|
if errs2.IsCanceled(err) {
|
|
return rpcstatus.Wrap(rpcstatus.Canceled, err)
|
|
}
|
|
|
|
// log it and continue. it's most likely our own fault that we couldn't
|
|
// track it, and the only thing that will be affected is our per-project
|
|
// segment limits.
|
|
endpoint.log.Error(
|
|
"Could not track the new project's segment usage when committing",
|
|
zap.Stringer("Project ID", projectID),
|
|
zap.Error(err),
|
|
)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (endpoint *Endpoint) addStorageUsageUpToLimit(ctx context.Context, projectID uuid.UUID, storage int64, segments int64) (err error) {
|
|
err = endpoint.projectUsage.AddProjectUsageUpToLimit(ctx, projectID, storage, segments)
|
|
|
|
if err != nil {
|
|
if accounting.ErrProjectLimitExceeded.Has(err) {
|
|
endpoint.log.Warn("Upload limit exceeded",
|
|
zap.Stringer("Project ID", projectID),
|
|
zap.Error(err),
|
|
)
|
|
return rpcstatus.Error(rpcstatus.ResourceExhausted, err.Error())
|
|
}
|
|
|
|
if errs2.IsCanceled(err) {
|
|
return rpcstatus.Wrap(rpcstatus.Canceled, err)
|
|
}
|
|
|
|
endpoint.log.Error(
|
|
"Updating project upload limits failed; limits won't be enforced",
|
|
zap.Stringer("Project ID", projectID),
|
|
zap.Error(err),
|
|
)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// checkEncryptedMetadata checks encrypted metadata and it's encrypted key sizes. Metadata encrypted key nonce
|
|
// is serialized to storj.Nonce automatically.
|
|
func (endpoint *Endpoint) checkEncryptedMetadataSize(encryptedMetadata, encryptedKey []byte) error {
|
|
metadataSize := memory.Size(len(encryptedMetadata))
|
|
if metadataSize > endpoint.config.MaxMetadataSize {
|
|
return rpcstatus.Errorf(rpcstatus.InvalidArgument, "Encrypted metadata is too large, got %v, maximum allowed is %v", metadataSize, endpoint.config.MaxMetadataSize)
|
|
}
|
|
|
|
// verify key only if any metadata was set
|
|
if metadataSize > 0 && len(encryptedKey) != encryptedKeySize {
|
|
return rpcstatus.Errorf(rpcstatus.InvalidArgument, "Encrypted metadata key size is invalid, got %v, expected %v", len(encryptedKey), encryptedKeySize)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (endpoint *Endpoint) checkObjectUploadRate(ctx context.Context, projectID uuid.UUID, bucketName []byte, objectKey []byte) error {
|
|
if !endpoint.config.UploadLimiter.Enabled {
|
|
return nil
|
|
}
|
|
|
|
limited := true
|
|
// if object location is in cache it means that we won't allow to upload yet here,
|
|
// if it's not or internally key expired we are good to go
|
|
key := strings.Join([]string{string(projectID[:]), string(bucketName), string(objectKey)}, "/")
|
|
_, _ = endpoint.singleObjectLimitCache.Get(ctx, key, func() (struct{}, error) {
|
|
limited = false
|
|
return struct{}{}, nil
|
|
})
|
|
if limited {
|
|
return rpcstatus.Error(rpcstatus.ResourceExhausted, "Too Many Requests")
|
|
}
|
|
|
|
return nil
|
|
}
|