gogobee/internal/safehttp/safehttp.go

// Package safehttp provides an http.Client hardened against SSRF and
// memory-DoS via hostile upstreams. Every outbound fetch the bot makes
// against feed-supplied URLs (RSS articles, image hosts) should go through
// one of these clients so a malicious feed can't steer the bot at loopback,
// link-local, RFC1918, or cloud metadata IPs, and can't OOM the process by
// streaming an unbounded body.
package safehttp

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// ErrBlockedHost is returned when a URL resolves to a non-public IP.
var ErrBlockedHost = errors.New("safehttp: blocked non-public host")

// AllowPrivate, when true, disables the loopback/RFC1918 dial guard. It
// exists for tests that spin up httptest.NewServer on 127.0.0.1 — never
// set this in production.
var AllowPrivate bool

// safeDialContext refuses connections to non-public IPs. It runs after
// DNS resolution, so a hostile DNS rebinding that returns 127.0.0.1
// still gets blocked at dial time.
func safeDialContext(ctx context.Context, network, addr string) (net.Conn, error) {
	host, port, err := net.SplitHostPort(addr)
	if err != nil {
		return nil, err
	}
	ips, err := (&net.Resolver{}).LookupIP(ctx, "ip", host)
	if err != nil {
		return nil, err
	}
	var allowed net.IP
	for _, ip := range ips {
		if AllowPrivate || isPublicIP(ip) {
			allowed = ip
			break
		}
	}
	if allowed == nil {
		return nil, fmt.Errorf("%w: %s", ErrBlockedHost, host)
	}
	d := &net.Dialer{Timeout: 5 * time.Second, KeepAlive: 30 * time.Second}
	return d.DialContext(ctx, network, net.JoinHostPort(allowed.String(), port))
}

// isPublicIP reports whether ip is a globally routable unicast address.
// Rejects loopback, link-local, multicast, RFC1918, CGNAT, and the
// AWS/GCP/Azure metadata IPs 169.254.169.254 / fd00:ec2::254 (these
// already fall under link-local but spell it out for clarity).
func isPublicIP(ip net.IP) bool {
	if ip == nil || ip.IsUnspecified() || ip.IsLoopback() ||
		ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
		ip.IsMulticast() || ip.IsPrivate() {
		return false
	}
	// 100.64.0.0/10 (CGNAT) is not covered by IsPrivate on older Go.
	if v4 := ip.To4(); v4 != nil {
		if v4[0] == 100 && v4[1] >= 64 && v4[1] <= 127 {
			return false
		}
		// 0.0.0.0/8 and friends.
		if v4[0] == 0 {
			return false
		}
	}
	return true
}

// ValidateURL returns nil if the URL is http(s) and parseable. It does
// not resolve DNS — the dial step does that — but it does reject bare
// schemes (file://, gopher://, etc.) before we even open a connection.
func ValidateURL(raw string) error {
	u, err := url.Parse(strings.TrimSpace(raw))
	if err != nil {
		return err
	}
	if u.Scheme != "http" && u.Scheme != "https" {
		return fmt.Errorf("safehttp: unsupported scheme %q", u.Scheme)
	}
	if u.Host == "" {
		return errors.New("safehttp: empty host")
	}
	return nil
}

// NewClient returns an http.Client whose transport blocks non-public
// destinations at dial time, caps redirects at 5, and re-validates each
// redirect target's scheme. timeout is the per-request overall budget.
func NewClient(timeout time.Duration) *http.Client {
	tr := &http.Transport{
		DialContext:           safeDialContext,
		ForceAttemptHTTP2:     true,
		MaxIdleConns:          32,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   5 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
		ResponseHeaderTimeout: 10 * time.Second,
	}
	return &http.Client{
		Transport: tr,
		Timeout:   timeout,
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			if len(via) >= 5 {
				return errors.New("safehttp: stopped after 5 redirects")
			}
			if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
				return fmt.Errorf("safehttp: unsupported redirect scheme %q", req.URL.Scheme)
			}
			return nil
		},
	}
}

// LimitedBody wraps r in a reader that reports EOF once max bytes have been
// read. Use to cap how much of a response body downstream parsers (goquery,
// image.Decode) will ever see — a hostile origin streaming an endless body
// otherwise OOMs the process.
//
// Hitting the cap is a truncation, not an error: parsers get a short-but-valid
// body and decide for themselves whether they found what they needed. Returning
// an error here instead would fail the whole parse on any oversized page, even
// when the interesting bytes (an HTML <head>, an image header) sit well inside
// the cap.
func LimitedBody(r io.Reader, max int64) io.Reader {
	return &limitedReader{R: r, N: max}
}

type limitedReader struct {
	R io.Reader
	N int64
}

func (l *limitedReader) Read(p []byte) (int, error) {
	if l.N <= 0 {
		return 0, io.EOF
	}
	if int64(len(p)) > l.N {
		p = p[:l.N]
	}
	n, err := l.R.Read(p)
	l.N -= int64(n)
	return n, err
}