133 lines
3.7 KiB
Go
133 lines
3.7 KiB
Go
// Copyright (C) 2019 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package trust
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
|
|
"github.com/zeebo/errs"
|
|
)
|
|
|
|
var (
|
|
// ErrHTTPSource is an error class for HTTP source errors.
|
|
ErrHTTPSource = errs.Class("HTTP source")
|
|
)
|
|
|
|
// HTTPSource represents a trust source at a http:// or https:// URL.
|
|
type HTTPSource struct {
|
|
url *url.URL
|
|
}
|
|
|
|
// NewHTTPSource constructs a new HTTPSource from a URL. The URL must be
|
|
// an http:// or https:// URL. The fragment cannot be set.
|
|
func NewHTTPSource(httpURL string) (*HTTPSource, error) {
|
|
u, err := url.Parse(httpURL)
|
|
if err != nil {
|
|
return nil, ErrHTTPSource.New("%q: not a URL: %w", httpURL, err)
|
|
}
|
|
if u.Scheme != "http" && u.Scheme != "https" {
|
|
return nil, ErrHTTPSource.New("%q: scheme is not supported", httpURL)
|
|
}
|
|
if u.Host == "" {
|
|
return nil, ErrHTTPSource.New(`%q: host is missing`, httpURL)
|
|
}
|
|
if u.Fragment != "" {
|
|
return nil, ErrHTTPSource.New("%q: fragment is not allowed", httpURL)
|
|
}
|
|
return &HTTPSource{url: u}, nil
|
|
}
|
|
|
|
// String implements the Source interface and returns the URL.
|
|
func (source *HTTPSource) String() string {
|
|
return source.url.String()
|
|
}
|
|
|
|
// Static implements the Source interface. It returns false for this source.
|
|
func (source *HTTPSource) Static() bool { return false }
|
|
|
|
// FetchEntries implements the Source interface and returns entries parsed from
|
|
// the list retrieved over HTTP(S). The entries returned are only authoritative
|
|
// if the entry URL has a host that matches or is a subdomain of the source URL.
|
|
func (source *HTTPSource) FetchEntries(ctx context.Context) (_ []Entry, err error) {
|
|
defer mon.Task()(&ctx)(&err)
|
|
|
|
req, err := http.NewRequestWithContext(ctx, "GET", source.url.String(), nil)
|
|
if err != nil {
|
|
return nil, ErrHTTPSource.Wrap(err)
|
|
}
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return nil, ErrHTTPSource.Wrap(err)
|
|
}
|
|
defer func() {
|
|
// Errors closing the response body can be ignored since they don't
|
|
// impact the correctness of the function.
|
|
_ = resp.Body.Close()
|
|
}()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, ErrHTTPSource.New("%q: unexpected status code %d: %q", source.url, resp.StatusCode, tryReadLine(resp.Body))
|
|
}
|
|
|
|
urls, err := ParseSatelliteURLList(ctx, resp.Body)
|
|
if err != nil {
|
|
return nil, ErrHTTPSource.New("cannot parse list at %q: %w", source.url, err)
|
|
}
|
|
|
|
var entries []Entry
|
|
for _, url := range urls {
|
|
authoritative := URLMatchesHTTPSourceHost(url.Host, source.url.Hostname())
|
|
|
|
entries = append(entries, Entry{
|
|
SatelliteURL: url,
|
|
Authoritative: authoritative,
|
|
})
|
|
}
|
|
return entries, nil
|
|
}
|
|
|
|
// URLMatchesHTTPSourceHost takes the Satellite URL host and the host of the
|
|
// HTTPSource URL and determines if the SatelliteURL matches or is in the
|
|
// same domain as the HTTPSource URL.
|
|
func URLMatchesHTTPSourceHost(urlHost, sourceHost string) bool {
|
|
urlIP := net.ParseIP(urlHost)
|
|
sourceIP := net.ParseIP(sourceHost)
|
|
|
|
// If one is an IP and the other isn't, then this isn't a match.
|
|
// TODO: should we resolve the non-IP host and see if it then matches?
|
|
if (urlIP != nil) != (sourceIP != nil) {
|
|
return false
|
|
}
|
|
|
|
// Both are IP addresses. Check for equality.
|
|
if urlIP != nil && sourceIP != nil {
|
|
return urlIP.Equal(sourceIP)
|
|
}
|
|
|
|
// Both are domain names. Check if the URL host matches or is a subdomain of
|
|
// the source host.
|
|
urlHost = normalizeHost(urlHost)
|
|
sourceHost = normalizeHost(sourceHost)
|
|
if urlHost == sourceHost {
|
|
return true
|
|
}
|
|
return strings.HasSuffix(urlHost, "."+sourceHost)
|
|
}
|
|
|
|
func normalizeHost(host string) string {
|
|
return strings.ToLower(strings.Trim(host, "."))
|
|
}
|
|
|
|
func tryReadLine(r io.Reader) string {
|
|
scanner := bufio.NewScanner(r)
|
|
scanner.Scan()
|
|
return scanner.Text()
|
|
}
|