storj/storagenode/trust/http_source.go

// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.

package trust

import (
	"bufio"
	"context"
	"io"
	"net"
	"net/http"
	"net/url"
	"strings"

	"github.com/zeebo/errs"
)

var (
	// ErrHTTPSource is an error class for HTTP source errors.
	ErrHTTPSource = errs.Class("HTTP source")
)

// HTTPSource represents a trust source at a http:// or https:// URL.
type HTTPSource struct {
	url *url.URL
}

// NewHTTPSource constructs a new HTTPSource from a URL. The URL must be
// an http:// or https:// URL. The fragment cannot be set.
func NewHTTPSource(httpURL string) (*HTTPSource, error) {
	u, err := url.Parse(httpURL)
	if err != nil {
		return nil, ErrHTTPSource.New("%q: not a URL: %w", httpURL, err)
	}
	if u.Scheme != "http" && u.Scheme != "https" {
		return nil, ErrHTTPSource.New("%q: scheme is not supported", httpURL)
	}
	if u.Host == "" {
		return nil, ErrHTTPSource.New(`%q: host is missing`, httpURL)
	}
	if u.Fragment != "" {
		return nil, ErrHTTPSource.New("%q: fragment is not allowed", httpURL)
	}
	return &HTTPSource{url: u}, nil
}

// String implements the Source interface and returns the URL.
func (source *HTTPSource) String() string {
	return source.url.String()
}

// Static implements the Source interface. It returns false for this source.
func (source *HTTPSource) Static() bool { return false }

// FetchEntries implements the Source interface and returns entries parsed from
// the list retrieved over HTTP(S). The entries returned are only authoritative
// if the entry URL has a host that matches or is a subdomain of the source URL.
func (source *HTTPSource) FetchEntries(ctx context.Context) (_ []Entry, err error) {
	defer mon.Task()(&ctx)(&err)

	req, err := http.NewRequestWithContext(ctx, "GET", source.url.String(), nil)
	if err != nil {
		return nil, ErrHTTPSource.Wrap(err)
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, ErrHTTPSource.Wrap(err)
	}
	defer func() {
		// Errors closing the response body can be ignored since they don't
		// impact the correctness of the function.
		_ = resp.Body.Close()
	}()
	if resp.StatusCode != http.StatusOK {
		return nil, ErrHTTPSource.New("%q: unexpected status code %d: %q", source.url, resp.StatusCode, tryReadLine(resp.Body))
	}

	urls, err := ParseSatelliteURLList(ctx, resp.Body)
	if err != nil {
		return nil, ErrHTTPSource.New("cannot parse list at %q: %w", source.url, err)
	}

	var entries []Entry
	for _, url := range urls {
		authoritative := URLMatchesHTTPSourceHost(url.Host, source.url.Hostname())

		entries = append(entries, Entry{
			SatelliteURL:  url,
			Authoritative: authoritative,
		})
	}
	return entries, nil
}

// URLMatchesHTTPSourceHost takes the Satellite URL host and the host of the
// HTTPSource URL and determines if the SatelliteURL matches or is in the
// same domain as the HTTPSource URL.
func URLMatchesHTTPSourceHost(urlHost, sourceHost string) bool {
	urlIP := net.ParseIP(urlHost)
	sourceIP := net.ParseIP(sourceHost)

	// If one is an IP and the other isn't, then this isn't a match.
	// TODO: should we resolve the non-IP host and see if it then matches?
	if (urlIP != nil) != (sourceIP != nil) {
		return false
	}

	// Both are IP addresses. Check for equality.
	if urlIP != nil && sourceIP != nil {
		return urlIP.Equal(sourceIP)
	}

	// Both are domain names. Check if the URL host matches or is a subdomain of
	// the source host.
	urlHost = normalizeHost(urlHost)
	sourceHost = normalizeHost(sourceHost)
	if urlHost == sourceHost {
		return true
	}
	return strings.HasSuffix(urlHost, "."+sourceHost)
}

func normalizeHost(host string) string {
	return strings.ToLower(strings.Trim(host, "."))
}

func tryReadLine(r io.Reader) string {
	scanner := bufio.NewScanner(r)
	scanner.Scan()
	return scanner.Text()
}