From 416fa80e8570a8089d6364fa9a7ae9dc6e217e98 Mon Sep 17 00:00:00 2001 From: Andrew Harding Date: Thu, 18 Jul 2019 06:26:09 -0600 Subject: [PATCH] Link Sharing Service (#2431) Link sharing service. See `docs/design/link-sharing-service.md` for the design and `cmd/linksharing/README.md` for operational instructions. --- cmd/linksharing/.gitignore | 1 + cmd/linksharing/README.md | 57 +++++++ cmd/linksharing/main.go | 141 ++++++++++++++++ cmd/uplink/cmd/share.go | 13 ++ pkg/httpserver/server.go | 132 +++++++++++++++ pkg/httpserver/server_test.go | 180 ++++++++++++++++++++ pkg/linksharing/handler.go | 225 +++++++++++++++++++++++++ pkg/linksharing/handler_test.go | 284 ++++++++++++++++++++++++++++++++ 8 files changed, 1033 insertions(+) create mode 100644 cmd/linksharing/.gitignore create mode 100644 cmd/linksharing/README.md create mode 100644 cmd/linksharing/main.go create mode 100644 pkg/httpserver/server.go create mode 100644 pkg/httpserver/server_test.go create mode 100644 pkg/linksharing/handler.go create mode 100644 pkg/linksharing/handler_test.go diff --git a/cmd/linksharing/.gitignore b/cmd/linksharing/.gitignore new file mode 100644 index 000000000..1564dc00f --- /dev/null +++ b/cmd/linksharing/.gitignore @@ -0,0 +1 @@ +linksharing diff --git a/cmd/linksharing/README.md b/cmd/linksharing/README.md new file mode 100644 index 000000000..40e81ac73 --- /dev/null +++ b/cmd/linksharing/README.md @@ -0,0 +1,57 @@ +# Link Sharing Service + +## Building + +``` +$ go install storj.io/storj/cmd/linksharing +``` + +## Configuring + +### Development + +Default development configuration has the link sharing service hosted on +`localhost:8080` serving plain HTTP. + +``` +$ linksharing setup --defaults dev +``` + +### Production + +To configure the link sharing service for production, run the `setup` command +using the `release` defaults. You must also provide the public URL for +the sharing service, which is used to construct URLs returned to +clients. Since there is currently no server affinity for requests, the URL +can point to a pool of servers: + +``` +$ linksharing setup --defaults release --public-url +``` + +Default release configuration has the link sharing service hosted on `:8443` +serving HTTPS using a server certificate (`server.crt.pem`) and +key (`server.key.pem`) residing in the working directory where the linksharing +service is run. + +You can modify the configuration file or use the `--cert-file` and `--key-file` +flags to configure an alternate location for the server keypair. + +In order to run the link sharing service in release mode serving HTTP, you must +clear the certificate and key file configurables: + +``` +$ linksharing setup --defaults release --public-url --cert-file="" --key-file="" --address=":8080" +``` + +**WARNING** HTTP is only recommended if you are doing TLS termination on the +same machine running the link sharing service as the link sharing service +serves unencrypted user data. + +## Running + +After configuration is complete, running the link sharing is as simple as: + +``` +$ linksharing run +``` diff --git a/cmd/linksharing/main.go b/cmd/linksharing/main.go new file mode 100644 index 000000000..7bd2e70b8 --- /dev/null +++ b/cmd/linksharing/main.go @@ -0,0 +1,141 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package main + +import ( + "crypto/tls" + "fmt" + "os" + "path/filepath" + + "github.com/spf13/cobra" + "github.com/zeebo/errs" + "go.uber.org/zap" + + "storj.io/storj/internal/fpath" + "storj.io/storj/lib/uplink" + "storj.io/storj/pkg/cfgstruct" + "storj.io/storj/pkg/httpserver" + "storj.io/storj/pkg/linksharing" + "storj.io/storj/pkg/process" +) + +// LinkSharing defines link sharing configuration +type LinkSharing struct { + Address string `user:"true" help:"public address to listen on" devDefault:"localhost:8080" releaseDefault:":8443"` + CertFile string `user:"true" help:"server certificate file" devDefault:"" releaseDefault:"server.crt.pem"` + KeyFile string `user:"true" help:"server key file" devDefault:"" releaseDefault:"server.key.pem"` + PublicURL string `user:"true" help:"public url for the server" devDefault:"http://localhost:8080" releaseDefault:""` +} + +var ( + rootCmd = &cobra.Command{ + Use: "link sharing service", + Short: "Link Sharing Service", + } + runCmd = &cobra.Command{ + Use: "run", + Short: "Run the link sharing service", + RunE: cmdRun, + } + setupCmd = &cobra.Command{ + Use: "setup", + Short: "Create config files", + RunE: cmdSetup, + Annotations: map[string]string{"type": "setup"}, + } + + runCfg LinkSharing + setupCfg LinkSharing + + confDir string +) + +func init() { + defaultConfDir := fpath.ApplicationDir("storj", "linksharing") + cfgstruct.SetupFlag(zap.L(), rootCmd, &confDir, "config-dir", defaultConfDir, "main directory for link sharing configuration") + defaults := cfgstruct.DefaultsFlag(rootCmd) + rootCmd.AddCommand(runCmd) + rootCmd.AddCommand(setupCmd) + process.Bind(runCmd, &runCfg, defaults, cfgstruct.ConfDir(confDir)) + process.Bind(setupCmd, &setupCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.SetupMode()) +} + +func cmdRun(cmd *cobra.Command, args []string) (err error) { + ctx := process.Ctx(cmd) + log := zap.L() + + uplink, err := uplink.NewUplink(ctx, nil) + if err != nil { + return err + } + + tlsConfig, err := configureTLS(runCfg.CertFile, runCfg.KeyFile) + if err != nil { + return err + } + + handler, err := linksharing.NewHandler(linksharing.HandlerConfig{ + Log: log, + Uplink: uplink, + URLBase: runCfg.PublicURL, + }) + if err != nil { + return err + } + + server, err := httpserver.New(log, httpserver.Config{ + Name: "Link Sharing", + Address: runCfg.Address, + Handler: handler, + TLSConfig: tlsConfig, + ShutdownTimeout: -1, + }) + + return server.Run(ctx) +} + +func cmdSetup(cmd *cobra.Command, args []string) (err error) { + setupDir, err := filepath.Abs(confDir) + if err != nil { + return err + } + + valid, _ := fpath.IsValidSetupDir(setupDir) + if !valid { + return fmt.Errorf("link sharing configuration already exists (%v)", setupDir) + } + + err = os.MkdirAll(setupDir, 0700) + if err != nil { + return err + } + + return process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), nil) +} + +func configureTLS(certFile, keyFile string) (*tls.Config, error) { + switch { + case certFile != "" && keyFile != "": + case certFile == "" && keyFile == "": + return nil, nil + case certFile != "" && keyFile == "": + return nil, errs.New("key file must be provided with cert file") + case certFile == "" && keyFile != "": + return nil, errs.New("cert file must be provided with key file") + } + + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, errs.New("unable to load server keypair: %v", err) + } + + return &tls.Config{ + Certificates: []tls.Certificate{cert}, + }, nil +} + +func main() { + process.Exec(rootCmd) +} diff --git a/cmd/uplink/cmd/share.go b/cmd/uplink/cmd/share.go index 46d6b99b8..844b64f4f 100644 --- a/cmd/uplink/cmd/share.go +++ b/cmd/uplink/cmd/share.go @@ -12,6 +12,7 @@ import ( "github.com/zeebo/errs" "storj.io/storj/internal/fpath" + "storj.io/storj/lib/uplink" libuplink "storj.io/storj/lib/uplink" "storj.io/storj/pkg/macaroon" "storj.io/storj/pkg/process" @@ -151,7 +152,19 @@ func shareMain(cmd *cobra.Command, args []string) (err error) { return err } + scope := &uplink.Scope{ + SatelliteAddr: cfg.Client.SatelliteAddr, + APIKey: key, + EncryptionAccess: access, + } + + scopeData, err := scope.Serialize() + if err != nil { + return err + } + fmt.Println("api key:", key.Serialize()) fmt.Println("enc ctx:", accessData) + fmt.Println("scope :", scopeData) return nil } diff --git a/pkg/httpserver/server.go b/pkg/httpserver/server.go new file mode 100644 index 000000000..b9e481a73 --- /dev/null +++ b/pkg/httpserver/server.go @@ -0,0 +1,132 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package httpserver + +import ( + "context" + "crypto/tls" + "net" + "net/http" + "time" + + "github.com/zeebo/errs" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" + + "storj.io/storj/internal/errs2" +) + +const ( + // DefaultShutdownTimeout is the default ShutdownTimeout (see Config) + DefaultShutdownTimeout = time.Second * 10 +) + +// Config holds the HTTP server configuration +type Config struct { + // Name is the name of the server. It is only used for logging. It can + // be empty. + Name string + + // Address is the address to bind the server to. It must be set. + Address string + + // Handler is the HTTP handler to be served. It must be set. + Handler http.Handler + + // TLSConfig is the TLS configuration for the server. It is optional. + TLSConfig *tls.Config + + // ShutdownTimeout controls how long to wait for requests to finish before + // returning from Run() after the context is canceled. It defaults to + // 10 seconds if unset. If set to a negative value, the server will be + // closed immediately. + ShutdownTimeout time.Duration +} + +// Server is the HTTP server +type Server struct { + log *zap.Logger + name string + listener net.Listener + server *http.Server + shutdownTimeout time.Duration +} + +// New creates a new URL Service Server. +func New(log *zap.Logger, config Config) (*Server, error) { + switch { + case config.Address == "": + return nil, errs.New("server address is required") + case config.Handler == nil: + return nil, errs.New("server handler is required") + } + + listener, err := net.Listen("tcp", config.Address) + if err != nil { + return nil, errs.New("unable to listen on %s: %v", config.Address, err) + } + + server := &http.Server{ + Handler: config.Handler, + TLSConfig: config.TLSConfig, + ErrorLog: zap.NewStdLog(log), + } + + if config.ShutdownTimeout == 0 { + config.ShutdownTimeout = DefaultShutdownTimeout + } + + if config.Name != "" { + log = log.With(zap.String("server", config.Name)) + } + + return &Server{ + log: log, + name: config.Name, + listener: listener, + server: server, + shutdownTimeout: config.ShutdownTimeout, + }, nil +} + +// Run runs the server until it's either closed or it errors. +func (server *Server) Run(ctx context.Context) (err error) { + ctx, cancel := context.WithCancel(ctx) + var group errgroup.Group + + group.Go(func() error { + <-ctx.Done() + server.log.Info("Server shutting down") + return shutdownWithTimeout(server.server, server.shutdownTimeout) + }) + group.Go(func() (err error) { + defer cancel() + server.log.With(zap.String("addr", server.Addr())).Sugar().Info("Server started") + if server.server.TLSConfig == nil { + err = server.server.Serve(server.listener) + } else { + err = server.server.ServeTLS(server.listener, "", "") + } + if err == http.ErrServerClosed { + return nil + } + server.log.With(zap.Error(err)).Error("Server closed unexpectedly") + return err + }) + return group.Wait() +} + +// Addr returns the public address. +func (server *Server) Addr() string { + return server.listener.Addr().String() +} + +func shutdownWithTimeout(server *http.Server, timeout time.Duration) error { + if timeout < 0 { + return server.Close() + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return errs2.IgnoreCanceled(server.Shutdown(ctx)) +} diff --git a/pkg/httpserver/server_test.go b/pkg/httpserver/server_test.go new file mode 100644 index 000000000..9d827d6bf --- /dev/null +++ b/pkg/httpserver/server_test.go @@ -0,0 +1,180 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package httpserver + +import ( + "context" + "crypto" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + "math/big" + "net" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" + + "storj.io/storj/internal/testcontext" + "storj.io/storj/pkg/pkcrypto" +) + +var ( + testKey = mustSignerFromPEM(`-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgT8yIof+3qG3wQzXf +eAOcuTgWmgqXRnHVwKJl2g1pCb2hRANCAARWxVAPyT1BRs2hqiDuHlPXr1kVDXuw +7/a1USmgsVWiZ0W3JopcTbTMhvMZk+2MKqtWcc3gHF4vRDnHTeQl4lsx +-----END PRIVATE KEY----- +`) + testCert = mustCreateLocalhostCert() +) + +func TestServer(t *testing.T) { + address := "localhost:0" + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "OK") + }) + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{ + { + Certificate: [][]byte{testCert.Raw}, + PrivateKey: testKey, + }, + }, + } + + testCases := []serverTestCase{ + { + Name: "missing address", + Handler: handler, + NewErr: "server address is required", + }, + { + Name: "bad address", + Address: "this is no good", + Handler: handler, + NewErr: "unable to listen on this is no good: listen tcp: address this is no good: missing port in address", + }, + { + Name: "missing handler", + Address: address, + NewErr: "server handler is required", + }, + { + Name: "success via HTTP", + Address: address, + Handler: handler, + }, + { + Name: "success via HTTPS", + Address: address, + Handler: handler, + TLSConfig: tlsConfig, + }, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run(testCase.Name, func(t *testing.T) { + ctx := testcontext.NewWithTimeout(t, time.Minute) + defer ctx.Cleanup() + + s, ok := testCase.NewServer(t) + if !ok { + return + } + + runCtx, cancel := context.WithCancel(ctx) + ctx.Go(func() error { + return s.Run(runCtx) + }) + + testCase.DoGet(t, s) + cancel() + }) + } +} + +type serverTestCase struct { + Name string + Address string + Handler http.Handler + TLSConfig *tls.Config + NewErr string +} + +func (testCase *serverTestCase) NewServer(tb testing.TB) (*Server, bool) { + s, err := New(zaptest.NewLogger(tb), Config{ + Name: "test", + Address: testCase.Address, + Handler: testCase.Handler, + TLSConfig: testCase.TLSConfig, + }) + if testCase.NewErr != "" { + require.EqualError(tb, err, testCase.NewErr) + return nil, false + } + require.NoError(tb, err) + return s, true +} + +func (testCase *serverTestCase) DoGet(tb testing.TB, s *Server) { + scheme := "http" + client := &http.Client{} + if testCase.TLSConfig != nil { + scheme = "https" + client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPoolFromCert(testCert), + }, + } + } + + resp, err := client.Get(fmt.Sprintf("%s://%s", scheme, s.Addr())) + require.NoError(tb, err) + defer func() { _ = resp.Body.Close() }() + + assert.Equal(tb, resp.StatusCode, http.StatusOK) + + body, err := ioutil.ReadAll(resp.Body) + assert.NoError(tb, err) + assert.Equal(tb, "OK", string(body)) +} + +func mustSignerFromPEM(keyBytes string) crypto.Signer { + key, err := pkcrypto.PrivateKeyFromPEM([]byte(keyBytes)) + if err != nil { + panic(err) + } + return key.(crypto.Signer) +} + +func mustCreateLocalhostCert() *x509.Certificate { + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(0), + NotAfter: time.Now().Add(time.Hour), + DNSNames: []string{"localhost"}, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}, + } + certDER, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, testKey.Public(), testKey) + if err != nil { + panic(err) + } + cert, err := x509.ParseCertificate(certDER) + if err != nil { + panic(err) + } + return cert +} + +func certPoolFromCert(cert *x509.Certificate) *x509.CertPool { + pool := x509.NewCertPool() + pool.AddCert(cert) + return pool +} diff --git a/pkg/linksharing/handler.go b/pkg/linksharing/handler.go new file mode 100644 index 000000000..38c198673 --- /dev/null +++ b/pkg/linksharing/handler.go @@ -0,0 +1,225 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package linksharing + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + + "github.com/zeebo/errs" + "go.uber.org/zap" + "gopkg.in/spacemonkeygo/monkit.v2" + + "storj.io/storj/lib/uplink" + "storj.io/storj/pkg/ranger" + "storj.io/storj/pkg/storj" +) + +var ( + mon = monkit.Package() +) + +// HandlerConfig specifies the handler configuration +type HandlerConfig struct { + // Log is a logger used for logging + Log *zap.Logger + + // Uplink is the uplink used to talk to the storage network + Uplink *uplink.Uplink + + // URLBase is the base URL of the link sharing handler. It is used + // to construct URLs returned to clients. It should be a fully formed URL. + URLBase string +} + +// Handler implements the link sharing HTTP handler +type Handler struct { + log *zap.Logger + uplink *uplink.Uplink + urlBase *url.URL +} + +// NewHandler creates a new link sharing HTTP handler +func NewHandler(config HandlerConfig) (*Handler, error) { + if config.Log == nil { + config.Log = zap.L() + } + + if config.Uplink == nil { + return nil, errs.New("uplink is required") + } + + urlBase, err := parseURLBase(config.URLBase) + if err != nil { + return nil, err + } + + return &Handler{ + log: config.Log, + uplink: config.Uplink, + urlBase: urlBase, + }, nil +} + +// ServeHTTP handles link sharing requests +func (handler *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // serveHTTP handles the request in full. the error that is returned can + // be ignored since it was only added to facilitate monitoring. + _ = handler.serveHTTP(w, r) +} + +func (handler *Handler) serveHTTP(w http.ResponseWriter, r *http.Request) (err error) { + ctx := r.Context() + defer mon.Task()(&ctx)(&err) + + locationOnly := false + + switch r.Method { + case http.MethodHead: + locationOnly = true + case http.MethodGet: + default: + err = errs.New("method not allowed") + http.Error(w, err.Error(), http.StatusMethodNotAllowed) + return err + } + + scope, bucket, unencPath, err := parseRequestPath(r.URL.Path) + if err != nil { + err = fmt.Errorf("invalid request: %v", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return err + } + + p, err := handler.uplink.OpenProject(ctx, scope.SatelliteAddr, scope.APIKey) + if err != nil { + handler.handleUplinkErr(w, "open project", err) + return err + } + defer func() { + if err := p.Close(); err != nil { + handler.log.With(zap.Error(err)).Warn("unable to close project") + } + }() + + b, err := p.OpenBucket(ctx, bucket, scope.EncryptionAccess) + if err != nil { + handler.handleUplinkErr(w, "open bucket", err) + return err + } + defer func() { + if err := b.Close(); err != nil { + handler.log.With(zap.Error(err)).Warn("unable to close bucket") + } + }() + + o, err := b.OpenObject(ctx, unencPath) + if err != nil { + handler.handleUplinkErr(w, "open object", err) + return err + } + defer func() { + if err := o.Close(); err != nil { + handler.log.With(zap.Error(err)).Warn("unable to close object") + } + }() + + if locationOnly { + location := makeLocation(handler.urlBase, r.URL.Path) + http.Redirect(w, r, location, http.StatusFound) + return nil + } + + ranger.ServeContent(ctx, w, r, unencPath, o.Meta.Modified, newObjectRanger(o)) + return nil +} + +func (handler *Handler) handleUplinkErr(w http.ResponseWriter, action string, err error) { + switch { + case storj.ErrBucketNotFound.Has(err): + http.Error(w, "bucket not found", http.StatusNotFound) + case storj.ErrObjectNotFound.Has(err): + http.Error(w, "object not found", http.StatusNotFound) + default: + handler.log.Error("unable to handle request", zap.String("action", action), zap.Error(err)) + http.Error(w, "unable to handle request", http.StatusInternalServerError) + } +} + +func parseRequestPath(p string) (*uplink.Scope, string, string, error) { + // Drop the leading slash, if necessary + p = strings.TrimPrefix(p, "/") + + // Split the request path + segments := strings.SplitN(p, "/", 3) + switch len(segments) { + case 1: + if segments[0] == "" { + return nil, "", "", errs.New("missing scope") + } + return nil, "", "", errs.New("missing bucket") + case 2: + return nil, "", "", errs.New("missing bucket path") + } + scopeb58 := segments[0] + bucket := segments[1] + unencPath := segments[2] + + scope, err := uplink.ParseScope(scopeb58) + if err != nil { + return nil, "", "", err + } + return scope, bucket, unencPath, nil +} + +type objectRanger struct { + o *uplink.Object +} + +func newObjectRanger(o *uplink.Object) ranger.Ranger { + return &objectRanger{ + o: o, + } +} + +func (ranger *objectRanger) Size() int64 { + return ranger.o.Meta.Size +} + +func (ranger *objectRanger) Range(ctx context.Context, offset, length int64) (_ io.ReadCloser, err error) { + defer mon.Task()(&ctx)(&err) + return ranger.o.DownloadRange(ctx, offset, length) +} + +func parseURLBase(s string) (*url.URL, error) { + u, err := url.Parse(s) + if err != nil { + return nil, errs.Wrap(err) + } + + switch { + case u.Scheme != "http" && u.Scheme != "https": + return nil, errs.New("URL base must be http:// or https://") + case u.Host == "": + return nil, errs.New("URL base must contain host") + case u.User != nil: + return nil, errs.New("URL base must not contain user info") + case u.RawQuery != "": + return nil, errs.New("URL base must not contain query values") + case u.Fragment != "": + return nil, errs.New("URL base must not contain a fragment") + } + return u, nil +} + +func makeLocation(base *url.URL, reqPath string) string { + location := *base + location.Path = path.Join(location.Path, reqPath) + return location.String() +} diff --git a/pkg/linksharing/handler_test.go b/pkg/linksharing/handler_test.go new file mode 100644 index 000000000..90c1a97fb --- /dev/null +++ b/pkg/linksharing/handler_test.go @@ -0,0 +1,284 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package linksharing + +import ( + "context" + "net/http" + "net/http/httptest" + "path" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" + + "storj.io/storj/internal/testcontext" + "storj.io/storj/internal/testplanet" + "storj.io/storj/lib/uplink" + "storj.io/storj/pkg/storj" +) + +func TestNewHandler(t *testing.T) { + ctx := testcontext.New(t) + defer ctx.Cleanup() + + uplink := newUplink(ctx, t) + defer ctx.Check(uplink.Close) + + testCases := []struct { + name string + config HandlerConfig + err string + }{ + { + name: "missing uplink", + config: HandlerConfig{ + URLBase: "http://localhost", + }, + err: "uplink is required", + }, + { + name: "URL base must be http or https", + config: HandlerConfig{ + Uplink: uplink, + URLBase: "gopher://chunks", + }, + err: "URL base must be http:// or https://", + }, + { + name: "URL base must contain host", + config: HandlerConfig{ + Uplink: uplink, + URLBase: "http://", + }, + err: "URL base must contain host", + }, + { + name: "URL base can have a port", + config: HandlerConfig{ + Uplink: uplink, + URLBase: "http://host:99", + }, + }, + { + name: "URL base can have a path", + config: HandlerConfig{ + Uplink: uplink, + URLBase: "http://host/gopher", + }, + }, + { + name: "URL base must not contain user info", + config: HandlerConfig{ + Uplink: uplink, + URLBase: "http://joe@host", + }, + err: "URL base must not contain user info", + }, + { + name: "URL base must not contain query values", + config: HandlerConfig{ + Uplink: uplink, + URLBase: "http://host/?gopher=chunks", + }, + err: "URL base must not contain query values", + }, + { + name: "URL base must not contain a fragment", + config: HandlerConfig{ + Uplink: uplink, + URLBase: "http://host/#gopher-chunks", + }, + err: "URL base must not contain a fragment", + }, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run(testCase.name, func(t *testing.T) { + + handler, err := NewHandler(testCase.config) + if testCase.err != "" { + require.EqualError(t, err, testCase.err) + return + } + require.NoError(t, err) + require.NotNil(t, handler) + }) + } +} + +func TestHandlerRequests(t *testing.T) { + testplanet.Run(t, testplanet.Config{ + SatelliteCount: 2, + StorageNodeCount: 1, + UplinkCount: 1, + }, testHandlerRequests) +} + +func testHandlerRequests(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { + err := planet.Uplinks[0].Upload(ctx, planet.Satellites[0], "testbucket", "test/foo", []byte("FOO")) + require.NoError(t, err) + + apiKey, err := uplink.ParseAPIKey(planet.Uplinks[0].APIKey[planet.Satellites[0].ID()]) + require.NoError(t, err) + + scope, err := (&uplink.Scope{ + SatelliteAddr: planet.Satellites[0].Addr(), + APIKey: apiKey, + EncryptionAccess: uplink.NewEncryptionAccessWithDefaultKey(storj.Key{}), + }).Serialize() + require.NoError(t, err) + + testCases := []struct { + name string + method string + path string + status int + header http.Header + body string + }{ + { + name: "invalid method", + method: "PUT", + status: http.StatusMethodNotAllowed, + body: "method not allowed\n", + }, + { + name: "GET missing scope", + method: "GET", + status: http.StatusBadRequest, + body: "invalid request: missing scope\n", + }, + { + name: "GET malformed scope", + method: "GET", + path: path.Join("BADSCOPE", "testbucket", "test/foo"), + status: http.StatusBadRequest, + body: "invalid request: invalid scope format\n", + }, + { + name: "GET missing bucket", + method: "GET", + path: scope, + status: http.StatusBadRequest, + body: "invalid request: missing bucket\n", + }, + { + name: "GET bucket not found", + method: "GET", + path: path.Join(scope, "someotherbucket", "test/foo"), + status: http.StatusNotFound, + body: "bucket not found\n", + }, + { + name: "GET missing bucket path", + method: "GET", + path: path.Join(scope, "testbucket"), + status: http.StatusBadRequest, + body: "invalid request: missing bucket path\n", + }, + { + name: "GET object not found", + method: "GET", + path: path.Join(scope, "testbucket", "test/bar"), + status: http.StatusNotFound, + body: "object not found\n", + }, + { + name: "GET success", + method: "GET", + path: path.Join(scope, "testbucket", "test/foo"), + status: http.StatusOK, + body: "FOO", + }, + { + name: "HEAD missing scope", + method: "HEAD", + status: http.StatusBadRequest, + body: "invalid request: missing scope\n", + }, + { + name: "HEAD malformed scope", + method: "HEAD", + path: path.Join("BADSCOPE", "testbucket", "test/foo"), + status: http.StatusBadRequest, + body: "invalid request: invalid scope format\n", + }, + { + name: "HEAD missing bucket", + method: "HEAD", + path: scope, + status: http.StatusBadRequest, + body: "invalid request: missing bucket\n", + }, + { + name: "HEAD bucket not found", + method: "HEAD", + path: path.Join(scope, "someotherbucket", "test/foo"), + status: http.StatusNotFound, + body: "bucket not found\n", + }, + { + name: "HEAD missing bucket path", + method: "HEAD", + path: path.Join(scope, "testbucket"), + status: http.StatusBadRequest, + body: "invalid request: missing bucket path\n", + }, + { + name: "HEAD object not found", + method: "HEAD", + path: path.Join(scope, "testbucket", "test/bar"), + status: http.StatusNotFound, + body: "object not found\n", + }, + { + name: "HEAD success", + method: "HEAD", + path: path.Join(scope, "testbucket", "test/foo"), + status: http.StatusFound, + header: http.Header{ + "Location": []string{"http://localhost/" + path.Join(scope, "testbucket", "test/foo")}, + }, + body: "", + }, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run(testCase.name, func(t *testing.T) { + uplink := newUplink(ctx, t) + defer ctx.Check(uplink.Close) + + handler, err := NewHandler(HandlerConfig{ + Log: zaptest.NewLogger(t), + Uplink: uplink, + URLBase: "http://localhost", + }) + require.NoError(t, err) + + url := "http://localhost/" + testCase.path + w := httptest.NewRecorder() + r, err := http.NewRequest(testCase.method, url, nil) + require.NoError(t, err) + handler.ServeHTTP(w, r) + + assert.Equal(t, testCase.status, w.Code, "status code does not match") + for h, v := range testCase.header { + assert.Equal(t, v, w.Header()[h], "%q header does not match", h) + } + assert.Equal(t, testCase.body, w.Body.String(), "body does not match") + }) + } +} + +func newUplink(ctx context.Context, tb testing.TB) *uplink.Uplink { + cfg := new(uplink.Config) + cfg.Volatile.TLS.SkipPeerCAWhitelist = true + up, err := uplink.NewUplink(ctx, cfg) + require.NoError(tb, err) + return up +}