From a038fc1dc4356bbab8cea2edfce8c7af89832df2 Mon Sep 17 00:00:00 2001 From: Mya Date: Fri, 12 Nov 2021 14:48:29 -0600 Subject: [PATCH] satellite/admin: add endpoints for managing a buckets geofence Change-Id: I407243b8b9c769d968e70478c45230d9fdecaeb2 --- satellite/admin/README.md | 35 ++++++ satellite/admin/bucket.go | 137 ++++++++++++++++++++++ satellite/admin/bucket_test.go | 125 ++++++++++++++++++++ satellite/admin/bucket_testplanet_test.go | 113 ++++++++++++++++++ satellite/admin/server.go | 3 + 5 files changed, 413 insertions(+) create mode 100644 satellite/admin/bucket.go create mode 100644 satellite/admin/bucket_test.go create mode 100644 satellite/admin/bucket_testplanet_test.go diff --git a/satellite/admin/README.md b/satellite/admin/README.md index 77e950362..41f5c2c83 100644 --- a/satellite/admin/README.md +++ b/satellite/admin/README.md @@ -30,6 +30,11 @@ Requires setting `Authorization` header for requests. * [POST /api/projects/{project-id}/limit?bandwidth={value}](#post-apiprojectsproject-idlimitbandwidthvalue) * [POST /api/projects/{project-id}/limit?rate={value}](#post-apiprojectsproject-idlimitratevalue) * [POST /api/projects/{project-id}/limit?buckets={value}](#post-apiprojectsproject-idlimitbucketsvalue) + * [Bucket Management](#bucket-management) + * [Geofencing](#geofencing) + * [POST /api/projects/{project-id}/buckets/{bucket-name}/geofence?region={value}](#post-apiprojectsproject-idbucketsbucket-namegeofenceregionvalue) + * [DELETE /api/projects/{project-id}/buckets/{bucket-name}/geofence](#delete-apiprojectsproject-idbucketsbucket-namegeofence) + * [GET /api/projects/{project-id}/buckets/{bucket-name}/geofence](#get-apiprojectsproject-idbucketsbucket-namegeofence) * [APIKey Management](#apikey-management) * [DELETE /api/apikeys/{apikey}](#delete-apiapikeysapikey) @@ -279,6 +284,36 @@ Updates rate limit for a project. Updates bucket limit for a project. +### Bucket Management + +This set of APIs provide administrative functionality over bucket functionality. + +#### Geofencing + +Manage geofencing capabilities for a given bucket. + +##### POST /api/projects/{project-id}/buckets/{bucket-name}/geofence?region={value} + +Enables the geofencing configuration for the specified bucket. The bucket MUST be empty in order for this to work. Valid +values for the `region` parameter are: + +- `EU` - restrict placement to data nodes that reside in the [European Union][] +- `EEA` - restrict placement to data nodes that reside in the [European Economic Area][] +- `US` - restricts placement to data nodes in the United States +- `DE` - restricts placement to data nodes in Germany + +[European Union]: https://github.com/storj/common/blob/main/storj/location/region.go#L14 + +[European Economic Area]: https://github.com/storj/common/blob/main/storj/location/region.go#L7 + +##### DELETE /api/projects/{project-id}/buckets/{bucket-name}/geofence + +Removes the geofencing configuration for the specified bucket. The bucket MUST be empty in order for this to work. + +##### GET /api/projects/{project-id}/buckets/{bucket-name}/geofence + +Pulls the current geofence configuration for the specified bucket. + ### APIKey Management #### DELETE /api/apikeys/{apikey} diff --git a/satellite/admin/bucket.go b/satellite/admin/bucket.go new file mode 100644 index 000000000..88fe6b49a --- /dev/null +++ b/satellite/admin/bucket.go @@ -0,0 +1,137 @@ +// Copyright (C) 2021 Storj Labs, Inc. +// See LICENSE for copying information. + +package admin + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/gorilla/mux" + + "storj.io/common/storj" + "storj.io/common/uuid" + "storj.io/storj/satellite/buckets" +) + +func validateGeofencePathParameters(vars map[string]string) (project uuid.NullUUID, bucket []byte, err error) { + projectUUIDString, ok := vars["project"] + if !ok { + return project, bucket, fmt.Errorf("project-uuid missing") + } + + project.UUID, err = uuid.FromString(projectUUIDString) + if err != nil { + return project, bucket, fmt.Errorf("project-uuid is not a valid uuid") + } + project.Valid = true + + bucketName := vars["bucket"] + if len(bucketName) == 0 { + return project, bucket, fmt.Errorf("bucket name is missing") + } + + bucket = []byte(bucketName) + return +} + +func parsePlacementConstraint(regionCode string) (storj.PlacementConstraint, error) { + switch regionCode { + case "EU": + return storj.EU, nil + case "EEA": + return storj.EEA, nil + case "US": + return storj.US, nil + case "DE": + return storj.DE, nil + case "": + return storj.EveryCountry, fmt.Errorf("missing region parameter") + default: + return storj.EveryCountry, fmt.Errorf("unrecognized region parameter: %s", regionCode) + } +} + +func (server *Server) updateBucket(w http.ResponseWriter, r *http.Request, placement storj.PlacementConstraint) { + ctx := r.Context() + + project, bucket, err := validateGeofencePathParameters(mux.Vars(r)) + if err != nil { + sendJSONError(w, err.Error(), "", http.StatusBadRequest) + return + } + + b, err := server.buckets.GetBucket(ctx, bucket, project.UUID) + if err != nil { + if storj.ErrBucketNotFound.Has(err) { + sendJSONError(w, "bucket does not exist", "", http.StatusBadRequest) + } else { + sendJSONError(w, "unable to create geofence for bucket", err.Error(), http.StatusInternalServerError) + } + return + } + + b.Placement = placement + + b, err = server.buckets.UpdateBucket(ctx, b) + if err != nil { + switch { + case storj.ErrBucketNotFound.Has(err): + sendJSONError(w, "bucket does not exist", "", http.StatusBadRequest) + case buckets.ErrBucketNotEmpty.Has(err): + sendJSONError(w, "bucket must be empty", "", http.StatusBadRequest) + default: + sendJSONError(w, "unable to create geofence for bucket", err.Error(), http.StatusInternalServerError) + } + return + } + + data, err := json.Marshal(b) + if err != nil { + sendJSONError(w, "failed to marshal bucket", err.Error(), http.StatusInternalServerError) + } else { + sendJSONData(w, http.StatusOK, data) + } +} + +func (server *Server) createGeofenceForBucket(w http.ResponseWriter, r *http.Request) { + placement, err := parsePlacementConstraint(r.URL.Query().Get("region")) + if err != nil { + sendJSONError(w, err.Error(), "available: EU, EEA, US, DE", http.StatusBadRequest) + return + } + + server.updateBucket(w, r, placement) +} + +func (server *Server) deleteGeofenceForBucket(w http.ResponseWriter, r *http.Request) { + server.updateBucket(w, r, storj.EveryCountry) +} + +func (server *Server) checkGeofenceForBucket(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + project, bucket, err := validateGeofencePathParameters(mux.Vars(r)) + if err != nil { + sendJSONError(w, err.Error(), "", http.StatusBadRequest) + return + } + + b, err := server.buckets.GetBucket(ctx, bucket, project.UUID) + if err != nil { + if storj.ErrBucketNotFound.Has(err) { + sendJSONError(w, "bucket does not exist", "", http.StatusBadRequest) + } else { + sendJSONError(w, "unable to check bucket", err.Error(), http.StatusInternalServerError) + } + return + } + + data, err := json.Marshal(b) + if err != nil { + sendJSONError(w, "failed to marshal bucket", err.Error(), http.StatusInternalServerError) + } else { + sendJSONData(w, http.StatusOK, data) + } +} diff --git a/satellite/admin/bucket_test.go b/satellite/admin/bucket_test.go new file mode 100644 index 000000000..0ca92cf19 --- /dev/null +++ b/satellite/admin/bucket_test.go @@ -0,0 +1,125 @@ +// Copyright (C) 2021 Storj Labs, Inc. +// See LICENSE for copying information. + +package admin + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "storj.io/common/storj" + "storj.io/common/uuid" +) + +func TestValidateRequestParameters(t *testing.T) { + + uid, err := uuid.New() + require.NoError(t, err, "failed to generate uuid") + + testCases := []struct { + name string + params map[string]string + // expectations + project uuid.NullUUID + bucket []byte + err string + }{ + {"missing project", map[string]string{}, uuid.NullUUID{}, nil, "project-uuid missing"}, + { + name: "invalid project", + params: map[string]string{ + "project": "invalidUUID", + }, + project: uuid.NullUUID{}, + bucket: nil, + err: "project-uuid is not a valid uuid", + }, + { + name: "missing bucket", + params: map[string]string{ + "project": uid.String(), + }, + project: uuid.NullUUID{ + UUID: uid, + Valid: true, + }, + bucket: nil, + err: "bucket name is missing", + }, + { + name: "empty bucket", + params: map[string]string{ + "project": uid.String(), + "bucket": "", + }, + project: uuid.NullUUID{ + UUID: uid, + Valid: true, + }, + bucket: nil, + err: "bucket name is missing", + }, + { + name: "valid parameters", + params: map[string]string{ + "project": uid.String(), + "bucket": "test-bucket", + }, + project: uuid.NullUUID{ + UUID: uid, + Valid: true, + }, + bucket: []byte("test-bucket"), + err: "", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + project, bucket, err := validateGeofencePathParameters(testCase.params) + + require.Equal(t, testCase.project, project) + require.Equal(t, testCase.bucket, bucket) + + if len(testCase.err) > 0 { + require.Error(t, err) + require.Equal(t, testCase.err, err.Error()) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestParsePlacementConstraint(t *testing.T) { + testCases := []struct { + name string + region string + placement storj.PlacementConstraint + err string + }{ + {"invalid", "invalid", storj.EveryCountry, "unrecognized region parameter: invalid"}, + {"empty", "", storj.EveryCountry, "missing region parameter"}, + {"US", "US", storj.US, ""}, + {"EU", "EU", storj.EU, ""}, + {"EEA", "EEA", storj.EEA, ""}, + {"DE", "DE", storj.DE, ""}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + placement, err := parsePlacementConstraint(testCase.region) + + require.Equal(t, testCase.placement, placement) + + if len(testCase.err) > 0 { + require.Error(t, err) + require.Equal(t, testCase.err, err.Error()) + } else { + require.NoError(t, err) + } + }) + + } +} diff --git a/satellite/admin/bucket_testplanet_test.go b/satellite/admin/bucket_testplanet_test.go new file mode 100644 index 000000000..19efa876c --- /dev/null +++ b/satellite/admin/bucket_testplanet_test.go @@ -0,0 +1,113 @@ +// Copyright (C) 2021 Storj Labs, Inc. +// See LICENSE for copying information. + +package admin_test + +import ( + "encoding/json" + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "storj.io/common/storj" + "storj.io/common/testcontext" + "storj.io/common/uuid" + "storj.io/storj/private/testplanet" + "storj.io/storj/satellite" +) + +func TestAdminBucketGeofenceAPI(t *testing.T) { + testplanet.Run(t, testplanet.Config{ + SatelliteCount: 1, + StorageNodeCount: 0, + UplinkCount: 1, + Reconfigure: testplanet.Reconfigure{ + Satellite: func(_ *zap.Logger, _ int, config *satellite.Config) { + config.Admin.Address = "127.0.0.1:0" + }, + }, + }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { + uplink := planet.Uplinks[0] + sat := planet.Satellites[0] + address := sat.Admin.Admin.Listener.Addr() + project, err := sat.DB.Console().Projects().Get(ctx, uplink.Projects[0].ID) + require.NoError(t, err) + + err = uplink.CreateBucket(ctx, sat, "filled") + require.NoError(t, err) + + _, err = sat.DB.Buckets().UpdateBucket(ctx, storj.Bucket{ + Name: "filled", + ProjectID: project.ID, + Placement: storj.EEA, + }) + require.NoError(t, err) + + err = uplink.Upload(ctx, sat, "filled", "README.md", []byte("hello world")) + require.NoError(t, err) + + err = uplink.CreateBucket(ctx, sat, "empty") + require.NoError(t, err) + + testCases := []struct { + name string + project uuid.UUID + bucket []byte + // expectations + status int + body string + }{ + { + name: "bucket does not exist", + project: project.ID, + bucket: []byte("non-existent"), + status: http.StatusBadRequest, + body: `{"error":"bucket does not exist","detail":""}`, + }, + { + name: "bucket is not empty", + project: project.ID, + bucket: []byte("filled"), + status: http.StatusBadRequest, + body: `{"error":"bucket must be empty","detail":""}`, + }, + { + name: "validated", + project: project.ID, + bucket: []byte("empty"), + status: http.StatusOK, + body: "", + }, + } + + for _, testCase := range testCases { + baseURL := fmt.Sprintf("http://%s/api/projects/%s/buckets/%s/geofence", address, testCase.project, string(testCase.bucket)) + t.Log(baseURL) + + t.Run(testCase.name, func(t *testing.T) { + assertReq(ctx, t, baseURL+"?region=EU", "POST", "", testCase.status, testCase.body, sat.Config.Console.AuthToken) + + if testCase.status == http.StatusOK { + b, err := sat.DB.Buckets().GetBucket(ctx, testCase.bucket, testCase.project) + require.NoError(t, err) + + expected, err := json.Marshal(storj.Bucket{ + ID: b.ID, + Name: b.Name, + ProjectID: testCase.project, + Created: b.Created, + Placement: storj.EU, + }) + require.NoError(t, err, "failed to json encode expected bucket") + + assertGet(ctx, t, baseURL, string(expected), sat.Config.Console.AuthToken) + } + + assertReq(ctx, t, baseURL, "DELETE", "", testCase.status, testCase.body, sat.Config.Console.AuthToken) + }) + } + }) +} diff --git a/satellite/admin/server.go b/satellite/admin/server.go index a07b15d07..deed61610 100644 --- a/satellite/admin/server.go +++ b/satellite/admin/server.go @@ -101,6 +101,9 @@ func NewServer(log *zap.Logger, listener net.Listener, db DB, buckets *buckets.S api.HandleFunc("/projects/{project}/apikeys", server.listAPIKeys).Methods("GET") api.HandleFunc("/projects/{project}/apikeys", server.addAPIKey).Methods("POST") api.HandleFunc("/projects/{project}/apikeys/{name}", server.deleteAPIKeyByName).Methods("DELETE") + api.HandleFunc("/projects/{project}/buckets/{bucket}/geofence", server.createGeofenceForBucket).Methods("POST") + api.HandleFunc("/projects/{project}/buckets/{bucket}/geofence", server.deleteGeofenceForBucket).Methods("DELETE") + api.HandleFunc("/projects/{project}/buckets/{bucket}/geofence", server.checkGeofenceForBucket).Methods("GET") api.HandleFunc("/apikeys/{apikey}", server.deleteAPIKey).Methods("DELETE") // This handler must be the last one because it uses the root as prefix,