httpreader: random access over an HTTP resource

internal/httpreader/content_range.rl

@@ -0,0 +1,79 @@
+package httpreader
+
+import (
+	"fmt"
+	"errors"
+	"strconv"
+)
+// The "go generate" command assumes ragel 7, which is shipped in Fedora.
+
+//go:generate sh -e ragel.sh
+
+// ContentRange is a parsed "bytes" content range.
+//
+// Unpopulated sections of the header are set to -1; refer to RFC7233 for
+// more information.
+type ContentRange struct {
+	First, Last, Length int64
+}
+
+// Reset sets all fields to a known value (-1).
+func (r *ContentRange) Reset() {
+	r.First = -1
+	r.Last = -1
+	r.Length = -1
+}
+
+// Parse populates the receiver with the "bytes" content range from the
+// supplied header value or reports an error.
+func (r *ContentRange) Parse(data string) error {
+	r.Reset()
+	// Action setup:
+	var err error
+	sc := 0
+	// State machine setup:
+	cs, p, pe, eof := 0, 0, len(data), len(data)
+%%{
+	machine content_range;
+	# Set_start is the start of a number to parse later.
+	action set_start { sc = fpc; }
+	# Set_length parses the number starting at the position stashed by set_start
+	# and assigns it to ret.Length.
+	action set_length {
+		r.Length, err = strconv.ParseInt(data[sc:fpc], 10, 64)
+		if err != nil {
+			return fmt.Errorf("odd integer: %q: %w", data[sc:fpc], err)
+		}
+	}
+	# Set_first is the same as set_length except it assigns to ret.First.
+	action set_first {
+		r.First, err = strconv.ParseInt(data[sc:fpc], 10, 64)
+		if err != nil {
+			return fmt.Errorf("odd integer: %q: %w", data[sc:fpc], err)
+		}
+	}
+	# Set_last is the same as set_length except it assigns to ret.Last.
+	action set_last {
+		r.Last, err = strconv.ParseInt(data[sc:fpc], 10, 64)
+		if err != nil {
+			return fmt.Errorf("odd integer: %q: %w", data[sc:fpc], err)
+		}
+	}
+
+	complete_length = digit+ >set_start %set_length;
+	unsatisfied_range = '*/' complete_length;
+	pos = digit+;
+	byte_range = pos >set_start %set_first '-' pos >set_start %set_last;
+	byte_range_resp = byte_range '/' ( complete_length | '*' );
+	main := 'bytes ' ( byte_range_resp | unsatisfied_range );
+
+	write data;
+	write init;
+	write exec;
+}%%
+	if p != pe {
+		// Didn't consume the header.
+		return errors.New("malformed header")
+	}
+	return nil
+}

internal/httpreader/content_range_test.go

@@ -0,0 +1,52 @@
+package httpreader
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestContentRange(t *testing.T) {
+	t.Parallel()
+	tt := []struct {
+		In   string
+		Want ContentRange
+		Err  bool
+	}{
+		{
+			In:   `bytes */64`,
+			Want: ContentRange{First: -1, Last: -1, Length: 64},
+		},
+		{
+			In:   `nonsense`,
+			Want: ContentRange{First: -1, Last: -1, Length: -1},
+			Err:  true,
+		},
+		{
+			In:   `bytes 0-63/64`,
+			Want: ContentRange{First: 0, Last: 63, Length: 64},
+		},
+		{
+			In:   `bytes 0-63/*`,
+			Want: ContentRange{First: 0, Last: 63, Length: -1},
+		},
+	}
+
+	for _, tc := range tt {
+		t.Run("", func(t *testing.T) {
+			t.Logf("In: %+q", tc.In)
+			var got ContentRange
+			err := got.Parse(tc.In)
+			if err != nil {
+				t.Logf("error: %v", err)
+			}
+			if tc.Err == (err == nil) {
+				t.Fail()
+			}
+			t.Logf("got: %d/%d/%d", got.First, got.Last, got.Length)
+			if !cmp.Equal(got, tc.Want) {
+				t.Error(cmp.Diff(got, tc.Want))
+			}
+		})
+	}
+}

internal/httpreader/doc.go

@@ -0,0 +1,18 @@
+// Package httpreader implements [io.ReaderAt] over an [http.Client] for a
+// resource that implements HTTP Range requests ([RFC7233]). Various tricks are
+// implemented to maximize compatibility.
+//
+// # Tricks
+//
+//   - Only use GET requests, to allow for locked-down signed requests.
+//   - Request last byte to negate weird CDN caching.
+//   - Try multiple ways to get the resource size.
+//
+// # Handled weirdness
+//
+//   - Server not handling negative ranges correctly.
+//   - Server not reporting content length when making Range requests.
+//   - "200 OK" for a range starting at 0.
+//
+// [RFC7233]: https://datatracker.ietf.org/doc/html/rfc7233
+package httpreader

internal/httpreader/metrics.go

@@ -0,0 +1,43 @@
+package httpreader
+
+import (
+	"context"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+var meter = otel.Meter(`github.com/quay/claircore/internal/httpreader`)
+
+var (
+	searchCount      metric.Int64Histogram
+	searchOriginKey  = attribute.Key("search.origin")
+	searchSuccessKey = attribute.Key("search.success")
+)
+
+func init() {
+	var err error
+	searchCount, err = meter.Int64Histogram(
+		"search",
+		metric.WithDescription("Number of requests made to binary search for the end of a resource"),
+		metric.WithUnit("{request}"),
+	)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func searchOrigin(v string) attribute.KeyValue {
+	return searchOriginKey.String(v)
+}
+
+func searchSuccess(v bool) attribute.KeyValue {
+	return searchSuccessKey.Bool(v)
+}
+
+func recordSearchCount(ctx context.Context, origin string, reqp *int, okp *bool) {
+	searchCount.Record(ctx,
+		int64(*reqp),
+		metric.WithAttributes(searchOrigin(origin), searchSuccess(*okp)))
+}

internal/httpreader/option.go

@@ -0,0 +1,25 @@
+package httpreader
+
+import (
+	"context"
+	"net/http"
+)
+
+// Option is used to set options in [New].
+type Option func(context.Context, *Reader) error
+
+// WithSize sets the size of the HTTP resource and skips rangefinding.
+func WithSize(sz int64) Option {
+	return func(_ context.Context, r *Reader) error {
+		r.size = sz
+		return nil
+	}
+}
+
+// WithHeaders sets additional headers for requests.
+func WithHeaders(h http.Header) Option {
+	return func(_ context.Context, r *Reader) error {
+		r.headers = h
+		return nil
+	}
+}

internal/httpreader/ragel.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -e
+ragel-go -F1 -o tmp.go content_range.rl
+trap 'rm tmp.go tmp.ri ||:' EXIT
+{
+	printf '// Code generated by ragel-go. DO NOT EDIT.\n\n'
+	sed '/^[[:space:]]\+$/d' < tmp.go
+} |
+	gofmt -s > content_range.go