From 4f96a85642c19805663ce074df53a92c40d455ab Mon Sep 17 00:00:00 2001 From: Erik van Velzen Date: Mon, 23 Aug 2021 13:43:22 +0200 Subject: [PATCH] cmd/uplink/share: register access via DRPC Convert registering access with the edge services from the HTTP protocol to DRPC protocol Change-Id: Iba88dd0758c26f613cf501be9a20ead07d122d0b --- cmd/uplink/cmd/access.go | 88 ++++++++------------ cmd/uplink/cmd/access_test.go | 147 ++++++++++++++++++++++++++-------- cmd/uplink/cmd/share.go | 9 ++- testsuite/go.mod | 4 +- testsuite/go.sum | 19 +++-- testsuite/ui/uitest/edge.go | 122 ++++++++++++++++++++++------ 6 files changed, 259 insertions(+), 130 deletions(-) diff --git a/cmd/uplink/cmd/access.go b/cmd/uplink/cmd/access.go index 7659c38de..c850a2e23 100644 --- a/cmd/uplink/cmd/access.go +++ b/cmd/uplink/cmd/access.go @@ -4,14 +4,12 @@ package cmd import ( - "bytes" "context" "encoding/base64" "encoding/json" "fmt" - "io/ioutil" - "net/http" - "time" + "os" + "strings" "github.com/btcsuite/btcutil/base58" "github.com/spf13/cobra" @@ -22,12 +20,12 @@ import ( "storj.io/private/cfgstruct" "storj.io/private/process" "storj.io/uplink" + "storj.io/uplink/edge" ) -const defaultAccessRegisterTimeout = 15 * time.Second - type registerConfig struct { AuthService string `help:"the address to the service you wish to register your access with" default:"" basic-help:"true"` + CACert string `help:"path to a file in PEM format with certificate(s) or certificate chain(s) to validate the auth service against" default:""` Public bool `help:"if the access should be public" default:"false" basic-help:"true"` Format string `help:"format of credentials, use 'env' or 'aws' for using in scripts" default:""` AWSProfile string `help:"if using --format=aws, output the --profile tag using this profile" default:""` @@ -64,7 +62,7 @@ func init() { registerCmd := &cobra.Command{ Use: "register [ACCESS]", - Short: "Register your access for use with a hosted gateway.", + Short: "Register your access for use with a hosted S3 compatible gateway and linksharing.", RunE: accessRegister, Args: cobra.MaximumNArgs(1), } @@ -211,12 +209,12 @@ func accessRegister(cmd *cobra.Command, args []string) (err error) { return errs.New("no access specified: %w", err) } - accessKey, secretKey, endpoint, err := RegisterAccess(ctx, access, registerCfg.AuthService, registerCfg.Public, defaultAccessRegisterTimeout) + credentials, err := RegisterAccess(ctx, access, registerCfg.AuthService, registerCfg.Public, registerCfg.CACert) if err != nil { return err } - return DisplayGatewayCredentials(accessKey, secretKey, endpoint, registerCfg.Format, registerCfg.AWSProfile) + return DisplayGatewayCredentials(credentials, registerCfg.Format, registerCfg.AWSProfile) } func getAccessFromArgZeroOrConfig(config AccessConfig, args []string) (access *uplink.Access, err error) { @@ -234,14 +232,16 @@ func getAccessFromArgZeroOrConfig(config AccessConfig, args []string) (access *u } // DisplayGatewayCredentials formats and writes credentials to stdout. -func DisplayGatewayCredentials(accessKey, secretKey, endpoint, format, awsProfile string) (err error) { +func DisplayGatewayCredentials(credentials *edge.Credentials, format, awsProfile string) (err error) { switch format { case "env": // export / set compatible format // note that AWS_ENDPOINT configuration is not natively utilized by the AWS CLI _, err = fmt.Printf("AWS_ACCESS_KEY_ID=%s\n"+ "AWS_SECRET_ACCESS_KEY=%s\n"+ "AWS_ENDPOINT=%s\n", - accessKey, secretKey, endpoint) + credentials.AccessKeyID, + credentials.SecretKey, + credentials.Endpoint) if err != nil { return err } @@ -258,7 +258,9 @@ func DisplayGatewayCredentials(accessKey, secretKey, endpoint, format, awsProfil _, err = fmt.Printf("aws configure %s set aws_access_key_id %s\n"+ "aws configure %s set aws_secret_access_key %s\n"+ "aws configure %s set s3.endpoint_url %s\n", - profile, accessKey, profile, secretKey, profile, endpoint) + profile, credentials.AccessKeyID, + profile, credentials.SecretKey, + profile, credentials.Endpoint) if err != nil { return err } @@ -267,7 +269,7 @@ func DisplayGatewayCredentials(accessKey, secretKey, endpoint, format, awsProfil "Access Key ID: %s\n"+ "Secret Key : %s\n"+ "Endpoint : %s\n", - accessKey, secretKey, endpoint) + credentials.AccessKeyID, credentials.SecretKey, credentials.Endpoint) if err != nil { return err } @@ -276,55 +278,29 @@ func DisplayGatewayCredentials(accessKey, secretKey, endpoint, format, awsProfil } // RegisterAccess registers an access grant with a Gateway Authorization Service. -func RegisterAccess(ctx context.Context, access *uplink.Access, authService string, public bool, timeout time.Duration) (accessKey, secretKey, endpoint string, err error) { +func RegisterAccess(ctx context.Context, access *uplink.Access, authService string, public bool, certificateFile string) (credentials *edge.Credentials, err error) { if authService == "" { - return "", "", "", errs.New("no auth service address provided") - } - accesssSerialized, err := access.Serialize() - if err != nil { - return "", "", "", errs.Wrap(err) - } - postData, err := json.Marshal(map[string]interface{}{ - "access_grant": accesssSerialized, - "public": public, - }) - if err != nil { - return accessKey, "", "", errs.Wrap(err) + return nil, errs.New("no auth service address provided") } - client := &http.Client{ - Timeout: timeout, + // preserve compatibility with previous https service + authService = strings.TrimPrefix(authService, "https://") + authService = strings.TrimSuffix(authService, "/") + if !strings.Contains(authService, ":") { + authService += ":7777" } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("%s/v1/access", authService), bytes.NewReader(postData)) - if err != nil { - return "", "", "", err - } - req.Header.Set("Content-Type", "application/json") - - resp, err := client.Do(req) - if err != nil { - return "", "", "", err - } - defer func() { err = errs.Combine(err, resp.Body.Close()) }() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", "", "", err + var certificatePEM []byte + if certificateFile != "" { + certificatePEM, err = os.ReadFile(certificateFile) + if err != nil { + return nil, errs.New("can't read certificate file: %w", err) + } } - respBody := make(map[string]string) - if err := json.Unmarshal(body, &respBody); err != nil { - return "", "", "", errs.New("unexpected response from auth service: %s", string(body)) + edgeConfig := edge.Config{ + AuthServiceAddress: authService, + CertificatePEM: certificatePEM, } - - accessKey, ok := respBody["access_key_id"] - if !ok { - return "", "", "", errs.New("access_key_id missing in response") - } - secretKey, ok = respBody["secret_key"] - if !ok { - return "", "", "", errs.New("secret_key missing in response") - } - return accessKey, secretKey, respBody["endpoint"], nil + return edgeConfig.RegisterAccess(ctx, access, &edge.RegisterAccessOptions{Public: public}) } diff --git a/cmd/uplink/cmd/access_test.go b/cmd/uplink/cmd/access_test.go index bab97e7a5..03582681a 100644 --- a/cmd/uplink/cmd/access_test.go +++ b/cmd/uplink/cmd/access_test.go @@ -4,61 +4,63 @@ package cmd_test import ( - "fmt" - "net/http" - "net/http/httptest" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "os" "os/exec" + "strconv" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "storj.io/common/pb" "storj.io/common/testcontext" + "storj.io/drpc/drpcmux" + "storj.io/drpc/drpcserver" "storj.io/storj/cmd/uplink/cmd" "storj.io/uplink" + "storj.io/uplink/edge" ) const testAccess = "12edqrJX1V243n5fWtUrwpMQXL8gKdY2wbyqRPSG3rsA1tzmZiQjtCyF896egifN2C2qdY6g5S1t6e8iDhMUon9Pb7HdecBFheAcvmN8652mqu8hRx5zcTUaRTWfFCKS2S6DHmTeqPUHJLEp6cJGXNHcdqegcKfeahVZGP4rTagHvFGEraXjYRJ3knAcWDGW6BxACqogEWez6r274JiUBfs4yRSbRNRqUEURd28CwDXMSHLRKKA7TEDKEdQ" func TestRegisterAccess(t *testing.T) { - ctx := testcontext.New(t) + ctx := testcontext.NewWithTimeout(t, 5*time.Second) + defer ctx.Cleanup() + + server := DRPCServerMock{} + + cancelCtx, authCancel := context.WithCancel(ctx) + defer authCancel() + port, certificatePEM := startMockAuthService(cancelCtx, ctx, t, &server) + caFile := ctx.File("cert.pem") + err := os.WriteFile(caFile, certificatePEM, os.FileMode(0600)) + require.NoError(t, err) + + url := "https://localhost:" + strconv.Itoa(port) - // mock the auth service - ts := httptest.NewServer( - http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, `{"access_key_id":"1", "secret_key":"2", "endpoint":"3"}`) - })) - defer ts.Close() // make sure we get back things access, err := uplink.ParseAccess(testAccess) require.NoError(t, err) - accessKey, secretKey, endpoint, err := cmd.RegisterAccess(ctx, access, ts.URL, true, 15*time.Second) + credentials, err := cmd.RegisterAccess(ctx, access, url, true, caFile) require.NoError(t, err) - assert.Equal(t, "1", accessKey) - assert.Equal(t, "2", secretKey) - assert.Equal(t, "3", endpoint) -} - -func TestRegisterAccessTimeout(t *testing.T) { - ctx := testcontext.New(t) - - // mock the auth service - ch := make(chan struct{}) - ts := httptest.NewServer( - http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - <-ch - })) - defer ts.Close() - // make sure we get back things - access, err := uplink.ParseAccess(testAccess) - require.NoError(t, err) - accessKey, secretKey, endpoint, err := cmd.RegisterAccess(ctx, access, ts.URL, true, 10*time.Millisecond) - require.Error(t, err) - assert.Equal(t, "", accessKey) - assert.Equal(t, "", secretKey) - assert.Equal(t, "", endpoint) - close(ch) + assert.Equal(t, + &edge.Credentials{ + AccessKeyID: "l5pucy3dmvzxgs3fpfewix27l5pq", + SecretKey: "l5pvgzldojsxis3fpfpv6x27l5pv6x27l5pv6x27l5pv6", + Endpoint: "https://gateway.example", + }, + credentials) } func TestAccessImport(t *testing.T) { @@ -73,3 +75,78 @@ func TestAccessImport(t *testing.T) { t.Log(string(output)) require.NoError(t, err) } + +type DRPCServerMock struct { + pb.DRPCEdgeAuthServer +} + +func (g *DRPCServerMock) RegisterAccess(context.Context, *pb.EdgeRegisterAccessRequest) (*pb.EdgeRegisterAccessResponse, error) { + return &pb.EdgeRegisterAccessResponse{ + AccessKeyId: "l5pucy3dmvzxgs3fpfewix27l5pq", + SecretKey: "l5pvgzldojsxis3fpfpv6x27l5pv6x27l5pv6x27l5pv6", + Endpoint: "https://gateway.example", + }, nil +} + +func startMockAuthService(cancelCtx context.Context, testCtx *testcontext.Context, t *testing.T, srv pb.DRPCEdgeAuthServer) (port int, certificatePEM []byte) { + certificatePEM, privateKeyPEM := createSelfSignedCertificate(t, "localhost") + + certificate, err := tls.X509KeyPair(certificatePEM, privateKeyPEM) + require.NoError(t, err) + + serverTLSConfig := &tls.Config{ + Certificates: []tls.Certificate{certificate}, + } + + drpcListener, err := tls.Listen("tcp", "127.0.0.1:0", serverTLSConfig) + require.NoError(t, err) + + port = drpcListener.Addr().(*net.TCPAddr).Port + + mux := drpcmux.New() + err = pb.DRPCRegisterEdgeAuth(mux, srv) + require.NoError(t, err) + + server := drpcserver.New(mux) + testCtx.Go(func() error { + return server.Serve(cancelCtx, drpcListener) + }) + + return port, certificatePEM +} + +func createSelfSignedCertificate(t *testing.T, hostname string) (certificatePEM []byte, privateKeyPEM []byte) { + notAfter := time.Now().Add(1 * time.Minute) + + // first create a server certificate + template := x509.Certificate{ + Subject: pkix.Name{ + CommonName: hostname, + }, + DNSNames: []string{hostname}, + SerialNumber: big.NewInt(1337), + BasicConstraintsValid: false, + IsCA: true, + NotAfter: notAfter, + } + + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + certificateDERBytes, err := x509.CreateCertificate( + rand.Reader, + &template, + &template, + &privateKey.PublicKey, + privateKey, + ) + require.NoError(t, err) + + certificatePEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificateDERBytes}) + + privateKeyBytes, err := x509.MarshalPKCS8PrivateKey(privateKey) + require.NoError(t, err) + privateKeyPEM = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateKeyBytes}) + + return certificatePEM, privateKeyPEM +} diff --git a/cmd/uplink/cmd/share.go b/cmd/uplink/cmd/share.go index 4ee9ba474..0ab0af09c 100644 --- a/cmd/uplink/cmd/share.go +++ b/cmd/uplink/cmd/share.go @@ -39,6 +39,7 @@ var shareCfg struct { URL bool `default:"false" help:"if true, returns a url for the shared path. implies --register and --public" basic-help:"true"` DNS string `default:"" help:"specify your custom hostname. if set, returns dns settings for web hosting. implies --register and --public" basic-help:"true"` AuthService string `default:"https://auth.us1.storjshare.io" help:"url for shared auth service" basic-help:"true"` + CACert string `help:"path to a file in PEM format with certificate(s) or certificate chain(s) to validate the auth service against" default:""` Public bool `default:"false" help:"if true, the access will be public. --dns and --url override this" basic-help:"true"` // Share requires information about the current access @@ -82,11 +83,11 @@ func shareMain(cmd *cobra.Command, args []string) (err error) { } if shareCfg.Register || shareCfg.URL || shareCfg.DNS != "" { - accessKey, secretKey, endpoint, err := RegisterAccess(ctx, newAccess, shareCfg.AuthService, isPublic, defaultAccessRegisterTimeout) + credentials, err := RegisterAccess(ctx, newAccess, shareCfg.AuthService, isPublic, shareCfg.CACert) if err != nil { return err } - err = DisplayGatewayCredentials(accessKey, secretKey, endpoint, "", "") + err = DisplayGatewayCredentials(credentials, "", "") if err != nil { return err } @@ -97,12 +98,12 @@ func shareMain(cmd *cobra.Command, args []string) (err error) { if len(shareCfg.AllowedPathPrefix) == 1 && !permission.AllowUpload && !permission.AllowDelete { if shareCfg.URL { - if err = createURL(accessKey, sharePrefixes); err != nil { + if err = createURL(credentials.AccessKeyID, sharePrefixes); err != nil { return err } } if shareCfg.DNS != "" { - if err = createDNS(accessKey); err != nil { + if err = createDNS(credentials.AccessKeyID); err != nil { return err } } diff --git a/testsuite/go.mod b/testsuite/go.mod index 532685572..1cc7ff119 100644 --- a/testsuite/go.mod +++ b/testsuite/go.mod @@ -11,7 +11,7 @@ require ( github.com/stretchr/testify v1.7.0 go.uber.org/zap v1.17.0 storj.io/common v0.0.0-20211217122906-6be0b96ce7e0 - storj.io/gateway-mt v1.14.4-0.20211015103214-01eddbc864fb + storj.io/gateway-mt v1.18.1-0.20211210081136-cada9a567d31 storj.io/private v0.0.0-20211209191323-6595d4aa0cfe storj.io/storj v0.12.1-0.20211102170500-1de8a695e84a ) @@ -229,7 +229,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect storj.io/drpc v0.0.26 // indirect - storj.io/gateway v1.3.1-0.20211004141903-f55ba9105164 // indirect + storj.io/gateway v1.4.1 // indirect storj.io/minio v0.0.0-20211007171754-df6c27823c8a // indirect storj.io/monkit-jaeger v0.0.0-20210426161729-debb1cbcbbd7 // indirect storj.io/uplink v1.7.2-0.20220110164813-03e4f59f7af2 // indirect diff --git a/testsuite/go.sum b/testsuite/go.sum index 571a11a95..4d0aa5b01 100644 --- a/testsuite/go.sum +++ b/testsuite/go.sum @@ -312,6 +312,7 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/pprof v0.0.0-20211108044417-e9b028704de0 h1:rsq1yB2xiFLDYYaYdlGBsSkwVzsCo500wMhxvW5A/bk= github.com/google/pprof v0.0.0-20211108044417-e9b028704de0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= @@ -543,8 +544,9 @@ github.com/lucas-clemente/quic-go v0.23.0 h1:5vFnKtZ6nHDFsc/F3uuiF4T3y/AXaQdxjUq github.com/lucas-clemente/quic-go v0.23.0/go.mod h1:paZuzjXCE5mj6sikVLMvqXk8lJV2AsqtJ6bDhjEfxx0= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/magefile/mage v1.10.0 h1:3HiXzCUY12kh9bIuyXShaVe529fJfyqoVM42o/uom2g= github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/magefile/mage v1.11.0 h1:C/55Ywp9BpgVVclD3lRnSYCwXTYxmSppIgLeDYlNuls= +github.com/magefile/mage v1.11.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= @@ -1042,6 +1044,7 @@ golang.org/x/crypto v0.0.0-20210415154028-4f45737414dc/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa h1:idItI2DDfCokpg0N51B2VtiLdJ4vAuXC9fnCb2gACo4= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1192,6 +1195,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211020064051-0ec99a608a1b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211109065445-02f5c0300f6e h1:i6Vklmyu+fZMFYpum+sR4ZWABGW7MyIxfJZXYvcnbns= golang.org/x/sys v0.0.0-20211109065445-02f5c0300f6e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1397,7 +1401,7 @@ sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3 storj.io/common v0.0.0-20200424175742-65ac59022f4f/go.mod h1:pZyXiIE7bGETIRXtfs0nICqMwp7PM8HqnDuyUeldNA0= storj.io/common v0.0.0-20210805073808-8e0feb09e92a/go.mod h1:mhZYWpTojKsACxWE66RfXNz19zbyr/uEDVWHJH8dHog= storj.io/common v0.0.0-20210916151047-6aaeb34bb916/go.mod h1:objobGrIWQwhmTSpSm6Y7ykd40wZjB7CezNfic5YLKg= -storj.io/common v0.0.0-20211006105453-d3fff091f9d2/go.mod h1:objobGrIWQwhmTSpSm6Y7ykd40wZjB7CezNfic5YLKg= +storj.io/common v0.0.0-20211102144601-401a79f0706a/go.mod h1:a2Kw7Uipu929OFANfWKLHRoD0JfhgssikEvimd6hbSQ= storj.io/common v0.0.0-20211217122906-6be0b96ce7e0 h1:xMUDQXSEzmhvmtC9jOqZEJ8jltjr3Hop4MxQlfUJ19Q= storj.io/common v0.0.0-20211217122906-6be0b96ce7e0/go.mod h1:Lufva6X8BVz5dEDNabpbjre4WVZr0x9sv3S90uVz7wk= storj.io/dotworld v0.0.0-20210324183515-0d11aeccd840/go.mod h1:KU9YvEgRrMMiWLvH8pzn1UkoCoxggKIPvQxmNdx7aXQ= @@ -1405,11 +1409,10 @@ storj.io/drpc v0.0.11/go.mod h1:TiFc2obNjL9/3isMW1Rpxjy8V9uE0B2HMeMFGiiI7Iw= storj.io/drpc v0.0.24/go.mod h1:ofQUDPQbbIymRDKE0tms48k8bLP5Y+dsI9CbXGv3gko= storj.io/drpc v0.0.26 h1:T6jJzjby7QUa/2XHR1qMxTCENpDHEw4/o+kfDfZQqQI= storj.io/drpc v0.0.26/go.mod h1:ofQUDPQbbIymRDKE0tms48k8bLP5Y+dsI9CbXGv3gko= -storj.io/gateway v1.3.1-0.20211004141903-f55ba9105164 h1:LO1aYnMKaJVZlaCX3uX3cBi9K+zDuP9ih2JwJuV6f7A= -storj.io/gateway v1.3.1-0.20211004141903-f55ba9105164/go.mod h1:fzVsPF3N/fICpn5dSoq6gmdp+4/B/4grhr7lUies77U= -storj.io/gateway-mt v1.14.4-0.20211015103214-01eddbc864fb h1:aHB/Tpd9kKvbEYby8ayXNA26ON/y6vsMxNYL2mF90GY= -storj.io/gateway-mt v1.14.4-0.20211015103214-01eddbc864fb/go.mod h1:7j06kxFOZ3KA9/j21+J5aLoDH3Eb6RHJ8DeuloXeYJY= -storj.io/minio v0.0.0-20211001113105-a621e66615dd/go.mod h1:mx9Vc+B1monPUqwM6nrDn+c3Wte8dBddVhTRNIoD1sY= +storj.io/gateway v1.4.1 h1:pRS0Q2Api+TZZdFVk84vdwcngrs0p+7LWFrfbrO1Q30= +storj.io/gateway v1.4.1/go.mod h1:36cfB5Zmgg3s0Y3+/igIx3Yr2OSptKJFVxmrsyymthE= +storj.io/gateway-mt v1.18.1-0.20211210081136-cada9a567d31 h1:Z1ndiuQzotm9rXKYUZ4VBLjDcGcJoJ0E7ZDOY7/9Q4k= +storj.io/gateway-mt v1.18.1-0.20211210081136-cada9a567d31/go.mod h1:S05u5dFF3rpYUqkdZlOVfphDx9WYcK4ymNoBzmWa3og= storj.io/minio v0.0.0-20211007171754-df6c27823c8a h1:Fs2BTYG6qGPaV2KyHqWU4kkfnThzs7UFkWd1l9kqrPQ= storj.io/minio v0.0.0-20211007171754-df6c27823c8a/go.mod h1:mx9Vc+B1monPUqwM6nrDn+c3Wte8dBddVhTRNIoD1sY= storj.io/monkit-jaeger v0.0.0-20210225162224-66fb37637bf6/go.mod h1:gj4vuCeyCRjRmH8LIrgoyU9Dc9uR6H+/GcDUXmTbf80= @@ -1418,7 +1421,7 @@ storj.io/monkit-jaeger v0.0.0-20210426161729-debb1cbcbbd7/go.mod h1:gj4vuCeyCRjR storj.io/private v0.0.0-20210810102517-434aeab3f17d/go.mod h1:wvudoWSyOyB2daZNaMykjjqsSUad/ttFlUwgelg9+qg= storj.io/private v0.0.0-20211209191323-6595d4aa0cfe h1:gPf2s3d247JWd/Iqzw7g8mvpdlqdBpVTsBhe6oPMkKU= storj.io/private v0.0.0-20211209191323-6595d4aa0cfe/go.mod h1:BoSaGSvsC8C6Gy0FyjrHfsElJA623hLsNIyexs6vGno= -storj.io/uplink v1.6.0/go.mod h1:zqj/LFDxa6RMaSRSHOmukg3mMgesOry0iHSjNldDMGo= storj.io/uplink v1.7.0/go.mod h1:zqj/LFDxa6RMaSRSHOmukg3mMgesOry0iHSjNldDMGo= +storj.io/uplink v1.7.1-0.20211103104100-a785482780d8/go.mod h1:pKqsMpNMIAz//2TXzUGOR6tpu3iyabvXV4VWINj4jaY= storj.io/uplink v1.7.2-0.20220110164813-03e4f59f7af2 h1:+dvNcQNxdjffZTFsPEegunnQybAjINL+AoUq1aqMCR8= storj.io/uplink v1.7.2-0.20220110164813-03e4f59f7af2/go.mod h1:jDVW7WrKk6jpVT8MXZpaXU7g7iy8WBSrFifiiixIHRI= diff --git a/testsuite/ui/uitest/edge.go b/testsuite/ui/uitest/edge.go index 6a5e2bc50..60ce056d0 100644 --- a/testsuite/ui/uitest/edge.go +++ b/testsuite/ui/uitest/edge.go @@ -5,11 +5,17 @@ package uitest import ( "context" + "crypto/ecdsa" + "crypto/elliptic" "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" "encoding/binary" + "encoding/pem" "fmt" + "io/ioutil" + "math/big" "net" - "net/url" "os" "strconv" "testing" @@ -22,7 +28,7 @@ import ( "storj.io/common/sync2" "storj.io/common/testcontext" - "storj.io/gateway-mt/auth" + "storj.io/gateway-mt/pkg/auth" "storj.io/gateway-mt/pkg/authclient" "storj.io/gateway-mt/pkg/server" "storj.io/gateway-mt/pkg/trustedip" @@ -55,13 +61,15 @@ type EdgeTest func(t *testing.T, ctx *testcontext.Context, planet *EdgePlanet, b func Edge(t *testing.T, test EdgeTest) { edgehost := os.Getenv("STORJ_TEST_EDGE_HOST") if edgehost == "" { - edgehost = "127.0.0.1" + edgehost = "localhost" } // TODO: make address not hardcoded the address selection here may // conflict with some automatically bound address. - authSvcAddr := net.JoinHostPort(edgehost, strconv.Itoa(randomRange(20000, 40000))) - authSvcAddrTLS := net.JoinHostPort(edgehost, strconv.Itoa(randomRange(20000, 40000))) + startPort := randomRange(20000, 40000) + authSvcAddr := net.JoinHostPort(edgehost, strconv.Itoa(startPort)) + authSvcAddrTLS := net.JoinHostPort(edgehost, strconv.Itoa(startPort+1)) + authSvcDrpcAddrTLS := net.JoinHostPort(edgehost, strconv.Itoa(startPort+2)) testplanet.Run(t, testplanet.Config{ SatelliteCount: 1, StorageNodeCount: 4, UplinkCount: 1, @@ -79,49 +87,54 @@ func Edge(t *testing.T, test EdgeTest) { gwConfig := server.Config{} cfgstruct.Bind(&pflag.FlagSet{}, &gwConfig, cfgstruct.UseTestDefaults()) gwConfig.Server.Address = "127.0.0.1:0" - gwConfig.AuthURL = "http://" + authSvcAddr + gwConfig.Auth.BaseURL = "http://" + authSvcAddr + gwConfig.Auth.Token = "super-secret" + gwConfig.Auth.Timeout = 5 * time.Minute gwConfig.InsecureLogAll = true - authURL, err := url.Parse("http://" + authSvcAddr) - require.NoError(t, err) - authClient, err := authclient.New(authURL, "super-secret", 5*time.Minute) - require.NoError(t, err) + authClient := authclient.New(gwConfig.Auth) - gateway, err := server.New(gwConfig, planet.Log().Named("gateway"), nil, trustedip.NewListTrustAll(), []string{}, authClient) + gateway, err := server.New(gwConfig, planet.Log().Named("gateway"), nil, trustedip.NewListTrustAll(), []string{}, authClient, []string{}) require.NoError(t, err) defer ctx.Check(gateway.Close) + certFile, keyFile, _, _ := createSelfSignedCertificateFile(t, edgehost) + authConfig := auth.Config{ - Endpoint: "http://" + gateway.Address(), - AuthToken: "super-secret", - KVBackend: "memory://", - ListenAddr: authSvcAddr, - ListenAddrTLS: authSvcAddrTLS, + Endpoint: "http://" + gateway.Address(), + AuthToken: "super-secret", + KVBackend: "memory://", + ListenAddr: authSvcAddr, + ListenAddrTLS: authSvcAddrTLS, + DRPCListenAddr: ":0", + DRPCListenAddrTLS: authSvcDrpcAddrTLS, + CertFile: certFile.Name(), + KeyFile: keyFile.Name(), } for _, sat := range planet.Satellites { authConfig.AllowedSatellites = append(authConfig.AllowedSatellites, sat.NodeURL().String()) } - auth, err := auth.New(ctx, planet.Log().Named("auth"), authConfig, ctx.Dir("authservice")) + authPeer, err := auth.New(ctx, planet.Log().Named("auth"), authConfig, ctx.Dir("authservice")) require.NoError(t, err) - defer ctx.Check(auth.Close) - ctx.Go(func() error { return auth.Run(ctx) }) - require.NoError(t, waitForAddress(ctx, authSvcAddr, 3*time.Second)) + defer ctx.Check(authPeer.Close) + ctx.Go(func() error { return authPeer.Run(ctx) }) + require.NoError(t, waitForAddress(ctx, authSvcAddrTLS, 3*time.Second)) + require.NoError(t, waitForAddress(ctx, authSvcDrpcAddrTLS, 3*time.Second)) ctx.Go(gateway.Run) require.NoError(t, waitForAddress(ctx, gateway.Address(), 3*time.Second)) - // todo: use the unused endpoint below - accessKey, secretKey, _, err := cmd.RegisterAccess(ctx, access, "http://"+authSvcAddr, false, 15*time.Second) + edgeCredentials, err := cmd.RegisterAccess(ctx, access, authSvcDrpcAddrTLS, false, certFile.Name()) require.NoError(t, err) edge := &EdgePlanet{} edge.Planet = planet - edge.Gateway.AccessKey = accessKey - edge.Gateway.SecretKey = secretKey - edge.Gateway.Addr = gateway.Address() + edge.Gateway.AccessKey = edgeCredentials.AccessKeyID + edge.Gateway.SecretKey = edgeCredentials.SecretKey + edge.Gateway.Addr = edgeCredentials.Endpoint edge.Auth.Addr = authSvcAddr Browser(t, ctx, planet, func(browser *rod.Browser) { @@ -130,6 +143,65 @@ func Edge(t *testing.T, test EdgeTest) { }) } +func createSelfSignedCertificateFile(t *testing.T, hostname string) (certFile *os.File, keyFile *os.File, certificatePEM []byte, privateKeyPEM []byte) { + certificatePEM, privateKeyPEM = createSelfSignedCertificate(t, hostname) + + certFile, err := ioutil.TempFile(os.TempDir(), "*-cert.pem") + require.NoError(t, err) + _, err = certFile.Write(certificatePEM) + require.NoError(t, err) + + keyFile, err = ioutil.TempFile(os.TempDir(), "*-key.pem") + require.NoError(t, err) + _, err = keyFile.Write(privateKeyPEM) + require.NoError(t, err) + + return certFile, keyFile, certificatePEM, privateKeyPEM +} + +func createSelfSignedCertificate(t *testing.T, hostname string) (certificatePEM []byte, privateKeyPEM []byte) { + notAfter := time.Now().Add(1 * time.Minute) + + var ips []net.IP + ip := net.ParseIP(hostname) + if ip != nil { + ips = []net.IP{ip} + } + + // first create a server certificate + template := x509.Certificate{ + Subject: pkix.Name{ + CommonName: hostname, + }, + DNSNames: []string{hostname}, + IPAddresses: ips, + SerialNumber: big.NewInt(1337), + BasicConstraintsValid: false, + IsCA: true, + NotAfter: notAfter, + } + + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + certificateDERBytes, err := x509.CreateCertificate( + rand.Reader, + &template, + &template, + &privateKey.PublicKey, + privateKey, + ) + require.NoError(t, err) + + certificatePEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificateDERBytes}) + + privateKeyBytes, err := x509.MarshalPKCS8PrivateKey(privateKey) + require.NoError(t, err) + privateKeyPEM = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateKeyBytes}) + + return certificatePEM, privateKeyPEM +} + func randomRange(low, high int) int { // this generates biased crypt random numbers // but it uses crypt/rand to avoid potentially