Link Sharing Service (#2431)
Link sharing service. See `docs/design/link-sharing-service.md` for the design and `cmd/linksharing/README.md` for operational instructions.
This commit is contained in:
Andrew Harding
GitHub
parent
af7ffb8072
commit
416fa80e85
1
cmd/linksharing/.gitignore
vendored
Normal file
1
cmd/linksharing/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
linksharing
|
||||
57
cmd/linksharing/README.md
Normal file
57
cmd/linksharing/README.md
Normal file
@ -0,0 +1,57 @@
|
||||
# Link Sharing Service
|
||||
|
||||
## Building
|
||||
|
||||
```
|
||||
$ go install storj.io/storj/cmd/linksharing
|
||||
```
|
||||
|
||||
## Configuring
|
||||
|
||||
### Development
|
||||
|
||||
Default development configuration has the link sharing service hosted on
|
||||
`localhost:8080` serving plain HTTP.
|
||||
|
||||
```
|
||||
$ linksharing setup --defaults dev
|
||||
```
|
||||
|
||||
### Production
|
||||
|
||||
To configure the link sharing service for production, run the `setup` command
|
||||
using the `release` defaults. You must also provide the public URL for
|
||||
the sharing service, which is used to construct URLs returned to
|
||||
clients. Since there is currently no server affinity for requests, the URL
|
||||
can point to a pool of servers:
|
||||
|
||||
```
|
||||
$ linksharing setup --defaults release --public-url <PUBLIC URL>
|
||||
```
|
||||
|
||||
Default release configuration has the link sharing service hosted on `:8443`
|
||||
serving HTTPS using a server certificate (`server.crt.pem`) and
|
||||
key (`server.key.pem`) residing in the working directory where the linksharing
|
||||
service is run.
|
||||
|
||||
You can modify the configuration file or use the `--cert-file` and `--key-file`
|
||||
flags to configure an alternate location for the server keypair.
|
||||
|
||||
In order to run the link sharing service in release mode serving HTTP, you must
|
||||
clear the certificate and key file configurables:
|
||||
|
||||
```
|
||||
$ linksharing setup --defaults release --public-url <PUBLIC URL> --cert-file="" --key-file="" --address=":8080"
|
||||
```
|
||||
|
||||
**WARNING** HTTP is only recommended if you are doing TLS termination on the
|
||||
same machine running the link sharing service as the link sharing service
|
||||
serves unencrypted user data.
|
||||
|
||||
## Running
|
||||
|
||||
After configuration is complete, running the link sharing is as simple as:
|
||||
|
||||
```
|
||||
$ linksharing run
|
||||
```
|
||||
141
cmd/linksharing/main.go
Normal file
141
cmd/linksharing/main.go
Normal file
@ -0,0 +1,141 @@
|
||||
// Copyright (C) 2019 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/zeebo/errs"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"storj.io/storj/internal/fpath"
|
||||
"storj.io/storj/lib/uplink"
|
||||
"storj.io/storj/pkg/cfgstruct"
|
||||
"storj.io/storj/pkg/httpserver"
|
||||
"storj.io/storj/pkg/linksharing"
|
||||
"storj.io/storj/pkg/process"
|
||||
)
|
||||
|
||||
// LinkSharing defines link sharing configuration
|
||||
type LinkSharing struct {
|
||||
Address string `user:"true" help:"public address to listen on" devDefault:"localhost:8080" releaseDefault:":8443"`
|
||||
CertFile string `user:"true" help:"server certificate file" devDefault:"" releaseDefault:"server.crt.pem"`
|
||||
KeyFile string `user:"true" help:"server key file" devDefault:"" releaseDefault:"server.key.pem"`
|
||||
PublicURL string `user:"true" help:"public url for the server" devDefault:"http://localhost:8080" releaseDefault:""`
|
||||
}
|
||||
|
||||
var (
|
||||
rootCmd = &cobra.Command{
|
||||
Use: "link sharing service",
|
||||
Short: "Link Sharing Service",
|
||||
}
|
||||
runCmd = &cobra.Command{
|
||||
Use: "run",
|
||||
Short: "Run the link sharing service",
|
||||
RunE: cmdRun,
|
||||
}
|
||||
setupCmd = &cobra.Command{
|
||||
Use: "setup",
|
||||
Short: "Create config files",
|
||||
RunE: cmdSetup,
|
||||
Annotations: map[string]string{"type": "setup"},
|
||||
}
|
||||
|
||||
runCfg LinkSharing
|
||||
setupCfg LinkSharing
|
||||
|
||||
confDir string
|
||||
)
|
||||
|
||||
func init() {
|
||||
defaultConfDir := fpath.ApplicationDir("storj", "linksharing")
|
||||
cfgstruct.SetupFlag(zap.L(), rootCmd, &confDir, "config-dir", defaultConfDir, "main directory for link sharing configuration")
|
||||
defaults := cfgstruct.DefaultsFlag(rootCmd)
|
||||
rootCmd.AddCommand(runCmd)
|
||||
rootCmd.AddCommand(setupCmd)
|
||||
process.Bind(runCmd, &runCfg, defaults, cfgstruct.ConfDir(confDir))
|
||||
process.Bind(setupCmd, &setupCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.SetupMode())
|
||||
}
|
||||
|
||||
func cmdRun(cmd *cobra.Command, args []string) (err error) {
|
||||
ctx := process.Ctx(cmd)
|
||||
log := zap.L()
|
||||
|
||||
uplink, err := uplink.NewUplink(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tlsConfig, err := configureTLS(runCfg.CertFile, runCfg.KeyFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
handler, err := linksharing.NewHandler(linksharing.HandlerConfig{
|
||||
Log: log,
|
||||
Uplink: uplink,
|
||||
URLBase: runCfg.PublicURL,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
server, err := httpserver.New(log, httpserver.Config{
|
||||
Name: "Link Sharing",
|
||||
Address: runCfg.Address,
|
||||
Handler: handler,
|
||||
TLSConfig: tlsConfig,
|
||||
ShutdownTimeout: -1,
|
||||
})
|
||||
|
||||
return server.Run(ctx)
|
||||
}
|
||||
|
||||
func cmdSetup(cmd *cobra.Command, args []string) (err error) {
|
||||
setupDir, err := filepath.Abs(confDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
valid, _ := fpath.IsValidSetupDir(setupDir)
|
||||
if !valid {
|
||||
return fmt.Errorf("link sharing configuration already exists (%v)", setupDir)
|
||||
}
|
||||
|
||||
err = os.MkdirAll(setupDir, 0700)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return process.SaveConfigWithAllDefaults(cmd.Flags(), filepath.Join(setupDir, "config.yaml"), nil)
|
||||
}
|
||||
|
||||
func configureTLS(certFile, keyFile string) (*tls.Config, error) {
|
||||
switch {
|
||||
case certFile != "" && keyFile != "":
|
||||
case certFile == "" && keyFile == "":
|
||||
return nil, nil
|
||||
case certFile != "" && keyFile == "":
|
||||
return nil, errs.New("key file must be provided with cert file")
|
||||
case certFile == "" && keyFile != "":
|
||||
return nil, errs.New("cert file must be provided with key file")
|
||||
}
|
||||
|
||||
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
|
||||
if err != nil {
|
||||
return nil, errs.New("unable to load server keypair: %v", err)
|
||||
}
|
||||
|
||||
return &tls.Config{
|
||||
Certificates: []tls.Certificate{cert},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
process.Exec(rootCmd)
|
||||
}
|
||||
@ -12,6 +12,7 @@ import (
|
||||
"github.com/zeebo/errs"
|
||||
|
||||
"storj.io/storj/internal/fpath"
|
||||
"storj.io/storj/lib/uplink"
|
||||
libuplink "storj.io/storj/lib/uplink"
|
||||
"storj.io/storj/pkg/macaroon"
|
||||
"storj.io/storj/pkg/process"
|
||||
@ -151,7 +152,19 @@ func shareMain(cmd *cobra.Command, args []string) (err error) {
|
||||
return err
|
||||
}
|
||||
|
||||
scope := &uplink.Scope{
|
||||
SatelliteAddr: cfg.Client.SatelliteAddr,
|
||||
APIKey: key,
|
||||
EncryptionAccess: access,
|
||||
}
|
||||
|
||||
scopeData, err := scope.Serialize()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Println("api key:", key.Serialize())
|
||||
fmt.Println("enc ctx:", accessData)
|
||||
fmt.Println("scope :", scopeData)
|
||||
return nil
|
||||
}
|
||||
|
||||
132
pkg/httpserver/server.go
Normal file
132
pkg/httpserver/server.go
Normal file
@ -0,0 +1,132 @@
|
||||
// Copyright (C) 2019 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package httpserver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/zeebo/errs"
|
||||
"go.uber.org/zap"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"storj.io/storj/internal/errs2"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultShutdownTimeout is the default ShutdownTimeout (see Config)
|
||||
DefaultShutdownTimeout = time.Second * 10
|
||||
)
|
||||
|
||||
// Config holds the HTTP server configuration
|
||||
type Config struct {
|
||||
// Name is the name of the server. It is only used for logging. It can
|
||||
// be empty.
|
||||
Name string
|
||||
|
||||
// Address is the address to bind the server to. It must be set.
|
||||
Address string
|
||||
|
||||
// Handler is the HTTP handler to be served. It must be set.
|
||||
Handler http.Handler
|
||||
|
||||
// TLSConfig is the TLS configuration for the server. It is optional.
|
||||
TLSConfig *tls.Config
|
||||
|
||||
// ShutdownTimeout controls how long to wait for requests to finish before
|
||||
// returning from Run() after the context is canceled. It defaults to
|
||||
// 10 seconds if unset. If set to a negative value, the server will be
|
||||
// closed immediately.
|
||||
ShutdownTimeout time.Duration
|
||||
}
|
||||
|
||||
// Server is the HTTP server
|
||||
type Server struct {
|
||||
log *zap.Logger
|
||||
name string
|
||||
listener net.Listener
|
||||
server *http.Server
|
||||
shutdownTimeout time.Duration
|
||||
}
|
||||
|
||||
// New creates a new URL Service Server.
|
||||
func New(log *zap.Logger, config Config) (*Server, error) {
|
||||
switch {
|
||||
case config.Address == "":
|
||||
return nil, errs.New("server address is required")
|
||||
case config.Handler == nil:
|
||||
return nil, errs.New("server handler is required")
|
||||
}
|
||||
|
||||
listener, err := net.Listen("tcp", config.Address)
|
||||
if err != nil {
|
||||
return nil, errs.New("unable to listen on %s: %v", config.Address, err)
|
||||
}
|
||||
|
||||
server := &http.Server{
|
||||
Handler: config.Handler,
|
||||
TLSConfig: config.TLSConfig,
|
||||
ErrorLog: zap.NewStdLog(log),
|
||||
}
|
||||
|
||||
if config.ShutdownTimeout == 0 {
|
||||
config.ShutdownTimeout = DefaultShutdownTimeout
|
||||
}
|
||||
|
||||
if config.Name != "" {
|
||||
log = log.With(zap.String("server", config.Name))
|
||||
}
|
||||
|
||||
return &Server{
|
||||
log: log,
|
||||
name: config.Name,
|
||||
listener: listener,
|
||||
server: server,
|
||||
shutdownTimeout: config.ShutdownTimeout,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Run runs the server until it's either closed or it errors.
|
||||
func (server *Server) Run(ctx context.Context) (err error) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
var group errgroup.Group
|
||||
|
||||
group.Go(func() error {
|
||||
<-ctx.Done()
|
||||
server.log.Info("Server shutting down")
|
||||
return shutdownWithTimeout(server.server, server.shutdownTimeout)
|
||||
})
|
||||
group.Go(func() (err error) {
|
||||
defer cancel()
|
||||
server.log.With(zap.String("addr", server.Addr())).Sugar().Info("Server started")
|
||||
if server.server.TLSConfig == nil {
|
||||
err = server.server.Serve(server.listener)
|
||||
} else {
|
||||
err = server.server.ServeTLS(server.listener, "", "")
|
||||
}
|
||||
if err == http.ErrServerClosed {
|
||||
return nil
|
||||
}
|
||||
server.log.With(zap.Error(err)).Error("Server closed unexpectedly")
|
||||
return err
|
||||
})
|
||||
return group.Wait()
|
||||
}
|
||||
|
||||
// Addr returns the public address.
|
||||
func (server *Server) Addr() string {
|
||||
return server.listener.Addr().String()
|
||||
}
|
||||
|
||||
func shutdownWithTimeout(server *http.Server, timeout time.Duration) error {
|
||||
if timeout < 0 {
|
||||
return server.Close()
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
return errs2.IgnoreCanceled(server.Shutdown(ctx))
|
||||
}
|
||||
180
pkg/httpserver/server_test.go
Normal file
180
pkg/httpserver/server_test.go
Normal file
@ -0,0 +1,180 @@
|
||||
// Copyright (C) 2019 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package httpserver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/big"
|
||||
"net"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap/zaptest"
|
||||
|
||||
"storj.io/storj/internal/testcontext"
|
||||
"storj.io/storj/pkg/pkcrypto"
|
||||
)
|
||||
|
||||
var (
|
||||
testKey = mustSignerFromPEM(`-----BEGIN PRIVATE KEY-----
|
||||
MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgT8yIof+3qG3wQzXf
|
||||
eAOcuTgWmgqXRnHVwKJl2g1pCb2hRANCAARWxVAPyT1BRs2hqiDuHlPXr1kVDXuw
|
||||
7/a1USmgsVWiZ0W3JopcTbTMhvMZk+2MKqtWcc3gHF4vRDnHTeQl4lsx
|
||||
-----END PRIVATE KEY-----
|
||||
`)
|
||||
testCert = mustCreateLocalhostCert()
|
||||
)
|
||||
|
||||
func TestServer(t *testing.T) {
|
||||
address := "localhost:0"
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprint(w, "OK")
|
||||
})
|
||||
tlsConfig := &tls.Config{
|
||||
Certificates: []tls.Certificate{
|
||||
{
|
||||
Certificate: [][]byte{testCert.Raw},
|
||||
PrivateKey: testKey,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
testCases := []serverTestCase{
|
||||
{
|
||||
Name: "missing address",
|
||||
Handler: handler,
|
||||
NewErr: "server address is required",
|
||||
},
|
||||
{
|
||||
Name: "bad address",
|
||||
Address: "this is no good",
|
||||
Handler: handler,
|
||||
NewErr: "unable to listen on this is no good: listen tcp: address this is no good: missing port in address",
|
||||
},
|
||||
{
|
||||
Name: "missing handler",
|
||||
Address: address,
|
||||
NewErr: "server handler is required",
|
||||
},
|
||||
{
|
||||
Name: "success via HTTP",
|
||||
Address: address,
|
||||
Handler: handler,
|
||||
},
|
||||
{
|
||||
Name: "success via HTTPS",
|
||||
Address: address,
|
||||
Handler: handler,
|
||||
TLSConfig: tlsConfig,
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
testCase := testCase
|
||||
t.Run(testCase.Name, func(t *testing.T) {
|
||||
ctx := testcontext.NewWithTimeout(t, time.Minute)
|
||||
defer ctx.Cleanup()
|
||||
|
||||
s, ok := testCase.NewServer(t)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
ctx.Go(func() error {
|
||||
return s.Run(runCtx)
|
||||
})
|
||||
|
||||
testCase.DoGet(t, s)
|
||||
cancel()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type serverTestCase struct {
|
||||
Name string
|
||||
Address string
|
||||
Handler http.Handler
|
||||
TLSConfig *tls.Config
|
||||
NewErr string
|
||||
}
|
||||
|
||||
func (testCase *serverTestCase) NewServer(tb testing.TB) (*Server, bool) {
|
||||
s, err := New(zaptest.NewLogger(tb), Config{
|
||||
Name: "test",
|
||||
Address: testCase.Address,
|
||||
Handler: testCase.Handler,
|
||||
TLSConfig: testCase.TLSConfig,
|
||||
})
|
||||
if testCase.NewErr != "" {
|
||||
require.EqualError(tb, err, testCase.NewErr)
|
||||
return nil, false
|
||||
}
|
||||
require.NoError(tb, err)
|
||||
return s, true
|
||||
}
|
||||
|
||||
func (testCase *serverTestCase) DoGet(tb testing.TB, s *Server) {
|
||||
scheme := "http"
|
||||
client := &http.Client{}
|
||||
if testCase.TLSConfig != nil {
|
||||
scheme = "https"
|
||||
client.Transport = &http.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
RootCAs: certPoolFromCert(testCert),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := client.Get(fmt.Sprintf("%s://%s", scheme, s.Addr()))
|
||||
require.NoError(tb, err)
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
assert.Equal(tb, resp.StatusCode, http.StatusOK)
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
assert.NoError(tb, err)
|
||||
assert.Equal(tb, "OK", string(body))
|
||||
}
|
||||
|
||||
func mustSignerFromPEM(keyBytes string) crypto.Signer {
|
||||
key, err := pkcrypto.PrivateKeyFromPEM([]byte(keyBytes))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return key.(crypto.Signer)
|
||||
}
|
||||
|
||||
func mustCreateLocalhostCert() *x509.Certificate {
|
||||
tmpl := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(0),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
DNSNames: []string{"localhost"},
|
||||
IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)},
|
||||
}
|
||||
certDER, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, testKey.Public(), testKey)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
cert, err := x509.ParseCertificate(certDER)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return cert
|
||||
}
|
||||
|
||||
func certPoolFromCert(cert *x509.Certificate) *x509.CertPool {
|
||||
pool := x509.NewCertPool()
|
||||
pool.AddCert(cert)
|
||||
return pool
|
||||
}
|
||||
225
pkg/linksharing/handler.go
Normal file
225
pkg/linksharing/handler.go
Normal file
@ -0,0 +1,225 @@
|
||||
// Copyright (C) 2019 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package linksharing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/zeebo/errs"
|
||||
"go.uber.org/zap"
|
||||
"gopkg.in/spacemonkeygo/monkit.v2"
|
||||
|
||||
"storj.io/storj/lib/uplink"
|
||||
"storj.io/storj/pkg/ranger"
|
||||
"storj.io/storj/pkg/storj"
|
||||
)
|
||||
|
||||
var (
|
||||
mon = monkit.Package()
|
||||
)
|
||||
|
||||
// HandlerConfig specifies the handler configuration
|
||||
type HandlerConfig struct {
|
||||
// Log is a logger used for logging
|
||||
Log *zap.Logger
|
||||
|
||||
// Uplink is the uplink used to talk to the storage network
|
||||
Uplink *uplink.Uplink
|
||||
|
||||
// URLBase is the base URL of the link sharing handler. It is used
|
||||
// to construct URLs returned to clients. It should be a fully formed URL.
|
||||
URLBase string
|
||||
}
|
||||
|
||||
// Handler implements the link sharing HTTP handler
|
||||
type Handler struct {
|
||||
log *zap.Logger
|
||||
uplink *uplink.Uplink
|
||||
urlBase *url.URL
|
||||
}
|
||||
|
||||
// NewHandler creates a new link sharing HTTP handler
|
||||
func NewHandler(config HandlerConfig) (*Handler, error) {
|
||||
if config.Log == nil {
|
||||
config.Log = zap.L()
|
||||
}
|
||||
|
||||
if config.Uplink == nil {
|
||||
return nil, errs.New("uplink is required")
|
||||
}
|
||||
|
||||
urlBase, err := parseURLBase(config.URLBase)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Handler{
|
||||
log: config.Log,
|
||||
uplink: config.Uplink,
|
||||
urlBase: urlBase,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ServeHTTP handles link sharing requests
|
||||
func (handler *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// serveHTTP handles the request in full. the error that is returned can
|
||||
// be ignored since it was only added to facilitate monitoring.
|
||||
_ = handler.serveHTTP(w, r)
|
||||
}
|
||||
|
||||
func (handler *Handler) serveHTTP(w http.ResponseWriter, r *http.Request) (err error) {
|
||||
ctx := r.Context()
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
|
||||
locationOnly := false
|
||||
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
locationOnly = true
|
||||
case http.MethodGet:
|
||||
default:
|
||||
err = errs.New("method not allowed")
|
||||
http.Error(w, err.Error(), http.StatusMethodNotAllowed)
|
||||
return err
|
||||
}
|
||||
|
||||
scope, bucket, unencPath, err := parseRequestPath(r.URL.Path)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("invalid request: %v", err)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return err
|
||||
}
|
||||
|
||||
p, err := handler.uplink.OpenProject(ctx, scope.SatelliteAddr, scope.APIKey)
|
||||
if err != nil {
|
||||
handler.handleUplinkErr(w, "open project", err)
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err := p.Close(); err != nil {
|
||||
handler.log.With(zap.Error(err)).Warn("unable to close project")
|
||||
}
|
||||
}()
|
||||
|
||||
b, err := p.OpenBucket(ctx, bucket, scope.EncryptionAccess)
|
||||
if err != nil {
|
||||
handler.handleUplinkErr(w, "open bucket", err)
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err := b.Close(); err != nil {
|
||||
handler.log.With(zap.Error(err)).Warn("unable to close bucket")
|
||||
}
|
||||
}()
|
||||
|
||||
o, err := b.OpenObject(ctx, unencPath)
|
||||
if err != nil {
|
||||
handler.handleUplinkErr(w, "open object", err)
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err := o.Close(); err != nil {
|
||||
handler.log.With(zap.Error(err)).Warn("unable to close object")
|
||||
}
|
||||
}()
|
||||
|
||||
if locationOnly {
|
||||
location := makeLocation(handler.urlBase, r.URL.Path)
|
||||
http.Redirect(w, r, location, http.StatusFound)
|
||||
return nil
|
||||
}
|
||||
|
||||
ranger.ServeContent(ctx, w, r, unencPath, o.Meta.Modified, newObjectRanger(o))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (handler *Handler) handleUplinkErr(w http.ResponseWriter, action string, err error) {
|
||||
switch {
|
||||
case storj.ErrBucketNotFound.Has(err):
|
||||
http.Error(w, "bucket not found", http.StatusNotFound)
|
||||
case storj.ErrObjectNotFound.Has(err):
|
||||
http.Error(w, "object not found", http.StatusNotFound)
|
||||
default:
|
||||
handler.log.Error("unable to handle request", zap.String("action", action), zap.Error(err))
|
||||
http.Error(w, "unable to handle request", http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
|
||||
func parseRequestPath(p string) (*uplink.Scope, string, string, error) {
|
||||
// Drop the leading slash, if necessary
|
||||
p = strings.TrimPrefix(p, "/")
|
||||
|
||||
// Split the request path
|
||||
segments := strings.SplitN(p, "/", 3)
|
||||
switch len(segments) {
|
||||
case 1:
|
||||
if segments[0] == "" {
|
||||
return nil, "", "", errs.New("missing scope")
|
||||
}
|
||||
return nil, "", "", errs.New("missing bucket")
|
||||
case 2:
|
||||
return nil, "", "", errs.New("missing bucket path")
|
||||
}
|
||||
scopeb58 := segments[0]
|
||||
bucket := segments[1]
|
||||
unencPath := segments[2]
|
||||
|
||||
scope, err := uplink.ParseScope(scopeb58)
|
||||
if err != nil {
|
||||
return nil, "", "", err
|
||||
}
|
||||
return scope, bucket, unencPath, nil
|
||||
}
|
||||
|
||||
type objectRanger struct {
|
||||
o *uplink.Object
|
||||
}
|
||||
|
||||
func newObjectRanger(o *uplink.Object) ranger.Ranger {
|
||||
return &objectRanger{
|
||||
o: o,
|
||||
}
|
||||
}
|
||||
|
||||
func (ranger *objectRanger) Size() int64 {
|
||||
return ranger.o.Meta.Size
|
||||
}
|
||||
|
||||
func (ranger *objectRanger) Range(ctx context.Context, offset, length int64) (_ io.ReadCloser, err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
return ranger.o.DownloadRange(ctx, offset, length)
|
||||
}
|
||||
|
||||
func parseURLBase(s string) (*url.URL, error) {
|
||||
u, err := url.Parse(s)
|
||||
if err != nil {
|
||||
return nil, errs.Wrap(err)
|
||||
}
|
||||
|
||||
switch {
|
||||
case u.Scheme != "http" && u.Scheme != "https":
|
||||
return nil, errs.New("URL base must be http:// or https://")
|
||||
case u.Host == "":
|
||||
return nil, errs.New("URL base must contain host")
|
||||
case u.User != nil:
|
||||
return nil, errs.New("URL base must not contain user info")
|
||||
case u.RawQuery != "":
|
||||
return nil, errs.New("URL base must not contain query values")
|
||||
case u.Fragment != "":
|
||||
return nil, errs.New("URL base must not contain a fragment")
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
func makeLocation(base *url.URL, reqPath string) string {
|
||||
location := *base
|
||||
location.Path = path.Join(location.Path, reqPath)
|
||||
return location.String()
|
||||
}
|
||||
284
pkg/linksharing/handler_test.go
Normal file
284
pkg/linksharing/handler_test.go
Normal file
@ -0,0 +1,284 @@
|
||||
// Copyright (C) 2019 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package linksharing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap/zaptest"
|
||||
|
||||
"storj.io/storj/internal/testcontext"
|
||||
"storj.io/storj/internal/testplanet"
|
||||
"storj.io/storj/lib/uplink"
|
||||
"storj.io/storj/pkg/storj"
|
||||
)
|
||||
|
||||
func TestNewHandler(t *testing.T) {
|
||||
ctx := testcontext.New(t)
|
||||
defer ctx.Cleanup()
|
||||
|
||||
uplink := newUplink(ctx, t)
|
||||
defer ctx.Check(uplink.Close)
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
config HandlerConfig
|
||||
err string
|
||||
}{
|
||||
{
|
||||
name: "missing uplink",
|
||||
config: HandlerConfig{
|
||||
URLBase: "http://localhost",
|
||||
},
|
||||
err: "uplink is required",
|
||||
},
|
||||
{
|
||||
name: "URL base must be http or https",
|
||||
config: HandlerConfig{
|
||||
Uplink: uplink,
|
||||
URLBase: "gopher://chunks",
|
||||
},
|
||||
err: "URL base must be http:// or https://",
|
||||
},
|
||||
{
|
||||
name: "URL base must contain host",
|
||||
config: HandlerConfig{
|
||||
Uplink: uplink,
|
||||
URLBase: "http://",
|
||||
},
|
||||
err: "URL base must contain host",
|
||||
},
|
||||
{
|
||||
name: "URL base can have a port",
|
||||
config: HandlerConfig{
|
||||
Uplink: uplink,
|
||||
URLBase: "http://host:99",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "URL base can have a path",
|
||||
config: HandlerConfig{
|
||||
Uplink: uplink,
|
||||
URLBase: "http://host/gopher",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "URL base must not contain user info",
|
||||
config: HandlerConfig{
|
||||
Uplink: uplink,
|
||||
URLBase: "http://joe@host",
|
||||
},
|
||||
err: "URL base must not contain user info",
|
||||
},
|
||||
{
|
||||
name: "URL base must not contain query values",
|
||||
config: HandlerConfig{
|
||||
Uplink: uplink,
|
||||
URLBase: "http://host/?gopher=chunks",
|
||||
},
|
||||
err: "URL base must not contain query values",
|
||||
},
|
||||
{
|
||||
name: "URL base must not contain a fragment",
|
||||
config: HandlerConfig{
|
||||
Uplink: uplink,
|
||||
URLBase: "http://host/#gopher-chunks",
|
||||
},
|
||||
err: "URL base must not contain a fragment",
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
testCase := testCase
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
|
||||
handler, err := NewHandler(testCase.config)
|
||||
if testCase.err != "" {
|
||||
require.EqualError(t, err, testCase.err)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, handler)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerRequests(t *testing.T) {
|
||||
testplanet.Run(t, testplanet.Config{
|
||||
SatelliteCount: 2,
|
||||
StorageNodeCount: 1,
|
||||
UplinkCount: 1,
|
||||
}, testHandlerRequests)
|
||||
}
|
||||
|
||||
func testHandlerRequests(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
||||
err := planet.Uplinks[0].Upload(ctx, planet.Satellites[0], "testbucket", "test/foo", []byte("FOO"))
|
||||
require.NoError(t, err)
|
||||
|
||||
apiKey, err := uplink.ParseAPIKey(planet.Uplinks[0].APIKey[planet.Satellites[0].ID()])
|
||||
require.NoError(t, err)
|
||||
|
||||
scope, err := (&uplink.Scope{
|
||||
SatelliteAddr: planet.Satellites[0].Addr(),
|
||||
APIKey: apiKey,
|
||||
EncryptionAccess: uplink.NewEncryptionAccessWithDefaultKey(storj.Key{}),
|
||||
}).Serialize()
|
||||
require.NoError(t, err)
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
method string
|
||||
path string
|
||||
status int
|
||||
header http.Header
|
||||
body string
|
||||
}{
|
||||
{
|
||||
name: "invalid method",
|
||||
method: "PUT",
|
||||
status: http.StatusMethodNotAllowed,
|
||||
body: "method not allowed\n",
|
||||
},
|
||||
{
|
||||
name: "GET missing scope",
|
||||
method: "GET",
|
||||
status: http.StatusBadRequest,
|
||||
body: "invalid request: missing scope\n",
|
||||
},
|
||||
{
|
||||
name: "GET malformed scope",
|
||||
method: "GET",
|
||||
path: path.Join("BADSCOPE", "testbucket", "test/foo"),
|
||||
status: http.StatusBadRequest,
|
||||
body: "invalid request: invalid scope format\n",
|
||||
},
|
||||
{
|
||||
name: "GET missing bucket",
|
||||
method: "GET",
|
||||
path: scope,
|
||||
status: http.StatusBadRequest,
|
||||
body: "invalid request: missing bucket\n",
|
||||
},
|
||||
{
|
||||
name: "GET bucket not found",
|
||||
method: "GET",
|
||||
path: path.Join(scope, "someotherbucket", "test/foo"),
|
||||
status: http.StatusNotFound,
|
||||
body: "bucket not found\n",
|
||||
},
|
||||
{
|
||||
name: "GET missing bucket path",
|
||||
method: "GET",
|
||||
path: path.Join(scope, "testbucket"),
|
||||
status: http.StatusBadRequest,
|
||||
body: "invalid request: missing bucket path\n",
|
||||
},
|
||||
{
|
||||
name: "GET object not found",
|
||||
method: "GET",
|
||||
path: path.Join(scope, "testbucket", "test/bar"),
|
||||
status: http.StatusNotFound,
|
||||
body: "object not found\n",
|
||||
},
|
||||
{
|
||||
name: "GET success",
|
||||
method: "GET",
|
||||
path: path.Join(scope, "testbucket", "test/foo"),
|
||||
status: http.StatusOK,
|
||||
body: "FOO",
|
||||
},
|
||||
{
|
||||
name: "HEAD missing scope",
|
||||
method: "HEAD",
|
||||
status: http.StatusBadRequest,
|
||||
body: "invalid request: missing scope\n",
|
||||
},
|
||||
{
|
||||
name: "HEAD malformed scope",
|
||||
method: "HEAD",
|
||||
path: path.Join("BADSCOPE", "testbucket", "test/foo"),
|
||||
status: http.StatusBadRequest,
|
||||
body: "invalid request: invalid scope format\n",
|
||||
},
|
||||
{
|
||||
name: "HEAD missing bucket",
|
||||
method: "HEAD",
|
||||
path: scope,
|
||||
status: http.StatusBadRequest,
|
||||
body: "invalid request: missing bucket\n",
|
||||
},
|
||||
{
|
||||
name: "HEAD bucket not found",
|
||||
method: "HEAD",
|
||||
path: path.Join(scope, "someotherbucket", "test/foo"),
|
||||
status: http.StatusNotFound,
|
||||
body: "bucket not found\n",
|
||||
},
|
||||
{
|
||||
name: "HEAD missing bucket path",
|
||||
method: "HEAD",
|
||||
path: path.Join(scope, "testbucket"),
|
||||
status: http.StatusBadRequest,
|
||||
body: "invalid request: missing bucket path\n",
|
||||
},
|
||||
{
|
||||
name: "HEAD object not found",
|
||||
method: "HEAD",
|
||||
path: path.Join(scope, "testbucket", "test/bar"),
|
||||
status: http.StatusNotFound,
|
||||
body: "object not found\n",
|
||||
},
|
||||
{
|
||||
name: "HEAD success",
|
||||
method: "HEAD",
|
||||
path: path.Join(scope, "testbucket", "test/foo"),
|
||||
status: http.StatusFound,
|
||||
header: http.Header{
|
||||
"Location": []string{"http://localhost/" + path.Join(scope, "testbucket", "test/foo")},
|
||||
},
|
||||
body: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
testCase := testCase
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
uplink := newUplink(ctx, t)
|
||||
defer ctx.Check(uplink.Close)
|
||||
|
||||
handler, err := NewHandler(HandlerConfig{
|
||||
Log: zaptest.NewLogger(t),
|
||||
Uplink: uplink,
|
||||
URLBase: "http://localhost",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
url := "http://localhost/" + testCase.path
|
||||
w := httptest.NewRecorder()
|
||||
r, err := http.NewRequest(testCase.method, url, nil)
|
||||
require.NoError(t, err)
|
||||
handler.ServeHTTP(w, r)
|
||||
|
||||
assert.Equal(t, testCase.status, w.Code, "status code does not match")
|
||||
for h, v := range testCase.header {
|
||||
assert.Equal(t, v, w.Header()[h], "%q header does not match", h)
|
||||
}
|
||||
assert.Equal(t, testCase.body, w.Body.String(), "body does not match")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func newUplink(ctx context.Context, tb testing.TB) *uplink.Uplink {
|
||||
cfg := new(uplink.Config)
|
||||
cfg.Volatile.TLS.SkipPeerCAWhitelist = true
|
||||
up, err := uplink.NewUplink(ctx, cfg)
|
||||
require.NoError(tb, err)
|
||||
return up
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user