diff --git a/internal/httpreader/content_range.go b/internal/httpreader/content_range.go new file mode 100644 index 000000000..73ed59f1e --- /dev/null +++ b/internal/httpreader/content_range.go @@ -0,0 +1,150 @@ +// Code generated by ragel-go. DO NOT EDIT. + +package httpreader + +import ( + "errors" + "strconv" +) + +// The "go generate" command assumes ragel 7, which is shipped in Fedora. + +//go:generate sh -e ragel.sh + +// ContentRange is a parsed "bytes" content range. +// +// Unpopulated sections of the header are set to -1; refer to RFC7233 for +// more information. +type ContentRange struct { + First, Last, Length int64 +} + +// Reset sets all fields to a known value (-1). +func (r *ContentRange) Reset() { + r.First = -1 + r.Last = -1 + r.Length = -1 +} + +// Parse populates the receiver with the "bytes" content range from the +// supplied header value or reports an error. +func (r *ContentRange) Parse(data string) error { + r.Reset() + // Action setup: + var err error + sc := 0 + // State machine setup: + cs, p, pe, eof := 0, 0, len(data), len(data) + var _content_range_trans_keys = []byte{1, 0, 6, 6, 10, 10, 9, 9, 7, 7, 8, 8, 0, 0, 2, 5, 4, 4, 5, 5, 3, 5, 5, 5, 4, 5, 2, 5, 5, 5, 1, 0, 0} + var _content_range_char_class = []int8{0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 3, 1, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 6, 1, 1, 7, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 8, 9, 1, 1, 1, 1, 10, 0} + var _content_range_index_offsets = []int8{0, 0, 1, 2, 3, 4, 5, 6, 10, 11, 12, 15, 16, 18, 22, 23, 0} + var _content_range_indices = []int8{2, 3, 4, 5, 6, 7, 8, 0, 0, 9, 10, 11, 13, 0, 12, 15, 17, 16, 19, 0, 0, 11, 21, 0} + var _content_range_index_defaults = []int8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + var _content_range_cond_targs = []int8{0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 9, 14, 10, 11, 11, 12, 12, 13, 13, 15, 14, 14, 0} + var _content_range_cond_actions = []int8{0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 1, 0, 3, 0, 0, 4, 0, 0} + var _content_range_eof_trans = []int8{1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 13, 15, 17, 19, 21, 20, 0} + var content_range_start int = 1 + var _ = content_range_start + var content_range_first_final int = 14 + var _ = content_range_first_final + var content_range_error int = 0 + var _ = content_range_error + var content_range_en_main int = 1 + var _ = content_range_en_main + { + cs = int(content_range_start) + } + { + var _trans uint = 0 + var _keys int + var _inds int + var _ic int + _resume: + { + } + if p == pe && p != eof { + goto _out + } + if p == eof { + if _content_range_eof_trans[cs] > 0 { + _trans = uint(_content_range_eof_trans[cs]) - 1 + } + } else { + _keys = int((cs << 1)) + + _inds = int(_content_range_index_offsets[cs]) + + if (data[p]) <= 121 && (data[p]) >= 32 { + _ic = int(_content_range_char_class[int((data[p]))-32]) + if _ic <= int(_content_range_trans_keys[_keys+1]) && _ic >= int(_content_range_trans_keys[_keys]) { + _trans = uint(_content_range_indices[_inds+int((_ic-int(_content_range_trans_keys[_keys])))]) + } else { + _trans = uint(_content_range_index_defaults[cs]) + } + } else { + _trans = uint(_content_range_index_defaults[cs]) + } + } + cs = int(_content_range_cond_targs[_trans]) + if _content_range_cond_actions[_trans] != 0 { + switch _content_range_cond_actions[_trans] { + case 1: + { + sc = p + } + case 4: + { + r.Length, err = strconv.ParseInt(data[sc:p], 10, 64) + if err != nil { + { + p += 1 + goto _out + } + } + } + case 2: + { + r.First, err = strconv.ParseInt(data[sc:p], 10, 64) + if err != nil { + { + p += 1 + goto _out + } + } + } + case 3: + { + r.Last, err = strconv.ParseInt(data[sc:p], 10, 64) + if err != nil { + { + p += 1 + goto _out + } + } + } + } + } + if p == eof { + if cs >= 14 { + goto _out + } + } else { + if cs != 0 { + p += 1 + goto _resume + } + } + _out: + { + } + } + if cs >= content_range_start && cs < content_range_first_final { + // Expect that err is populated + return err + } + if p != pe { + // Didn't consume the header. + return errors.New("malformed header") + } + return nil +} diff --git a/internal/httpreader/content_range.rl b/internal/httpreader/content_range.rl new file mode 100644 index 000000000..ba0f9213d --- /dev/null +++ b/internal/httpreader/content_range.rl @@ -0,0 +1,82 @@ +package httpreader + +import ( + "errors" + "strconv" +) +// The "go generate" command assumes ragel 7, which is shipped in Fedora. + +//go:generate sh -e ragel.sh + +// ContentRange is a parsed "bytes" content range. +// +// Unpopulated sections of the header are set to -1; refer to RFC7233 for +// more information. +type ContentRange struct { + First, Last, Length int64 +} + +// Reset sets all fields to a known value (-1). +func (r *ContentRange) Reset() { + r.First = -1 + r.Last = -1 + r.Length = -1 +} + +// Parse populates the receiver with the "bytes" content range from the +// supplied header value or reports an error. +func (r *ContentRange) Parse(data string) ( error) { + r.Reset() + // Action setup: + var err error + sc := 0 + // State machine setup: + cs, p, pe, eof := 0, 0, len(data), len(data) +%%{ + machine content_range; + # Set_start is the start of a number to parse later. + action set_start { sc = fpc; } + # Set_length parses the number starting at the position stashed by set_start + # and assigns it to ret.Length. + action set_length { + r.Length, err = strconv.ParseInt(data[sc:fpc], 10, 64) + if err != nil { + fbreak; + } + } + # Set_first is the same as set_length except it assigns to ret.First. + action set_first { + r.First, err = strconv.ParseInt(data[sc:fpc], 10, 64) + if err != nil { + fbreak; + } + } + # Set_last is the same as set_length except it assigns to ret.Last. + action set_last { + r.Last, err = strconv.ParseInt(data[sc:fpc], 10, 64) + if err != nil { + fbreak; + } + } + + complete_length = digit+ >set_start %set_length; + unsatisfied_range = '*/' complete_length; + pos = digit+; + byte_range = pos >set_start %set_first '-' pos >set_start %set_last; + byte_range_resp = byte_range '/' ( complete_length | '*' ); + main := 'bytes ' ( byte_range_resp | unsatisfied_range ); + + write data; + write init; + write exec; +}%% + if cs >= content_range_start && cs < content_range_first_final { + // Expect that err is populated + return err + } + if p != pe { + // Didn't consume the header. + return errors.New("malformed header") + } + return nil +} diff --git a/internal/httpreader/content_range_test.go b/internal/httpreader/content_range_test.go new file mode 100644 index 000000000..337afa8b1 --- /dev/null +++ b/internal/httpreader/content_range_test.go @@ -0,0 +1,52 @@ +package httpreader + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestContentRange(t *testing.T) { + t.Parallel() + tt := []struct { + In string + Want ContentRange + Err bool + }{ + { + In: `bytes */64`, + Want: ContentRange{First: -1, Last: -1, Length: 64}, + }, + { + In: `nonsense`, + Want: ContentRange{First: -1, Last: -1, Length: -1}, + Err: true, + }, + { + In: `bytes 0-63/64`, + Want: ContentRange{First: 0, Last: 63, Length: 64}, + }, + { + In: `bytes 0-63/*`, + Want: ContentRange{First: 0, Last: 63, Length: -1}, + }, + } + + for _, tc := range tt { + t.Run("", func(t *testing.T) { + t.Logf("In: %+q", tc.In) + var got ContentRange + err := got.Parse(tc.In) + if err != nil { + t.Logf("error: %v", err) + } + if tc.Err == (err == nil) { + t.Fail() + } + t.Logf("got: %d/%d/%d", got.First, got.Last, got.Length) + if !cmp.Equal(got, tc.Want) { + t.Error(cmp.Diff(got, tc.Want)) + } + }) + } +} diff --git a/internal/httpreader/doc.go b/internal/httpreader/doc.go new file mode 100644 index 000000000..73393a9cb --- /dev/null +++ b/internal/httpreader/doc.go @@ -0,0 +1,18 @@ +// Package httpreader implements [io.ReaderAt] over an [http.Client] for a +// resource that implements HTTP Range requests ([RFC7233]). Various tricks are +// implemented to maximize compatibility. +// +// # Tricks +// +// - Only use GET requests, to allow for locked-down signed requests. +// - Request last byte to negate weird CDN caching. +// - Try multiple ways to get the resource size. +// +// # Handled weirdness +// +// - Server not handling negative ranges correctly. +// - Server not reporting content length when making Range requests. +// - "200 OK" for a range starting at 0. +// +// [RFC7233]: https://datatracker.ietf.org/doc/html/rfc7233 +package httpreader diff --git a/internal/httpreader/metrics.go b/internal/httpreader/metrics.go new file mode 100644 index 000000000..36c315c1d --- /dev/null +++ b/internal/httpreader/metrics.go @@ -0,0 +1,43 @@ +package httpreader + +import ( + "context" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +var meter = otel.Meter(`github.com/quay/claircore/internal/httpreader`) + +var ( + searchCount metric.Int64Histogram + searchOriginKey = attribute.Key("search.origin") + searchSuccessKey = attribute.Key("search.success") +) + +func init() { + var err error + searchCount, err = meter.Int64Histogram( + "search", + metric.WithDescription("Number of requests made to binary search for the end of a resource"), + metric.WithUnit("{request}"), + ) + if err != nil { + panic(err) + } +} + +func searchOrigin(v string) attribute.KeyValue { + return searchOriginKey.String(v) +} + +func searchSuccess(v bool) attribute.KeyValue { + return searchSuccessKey.Bool(v) +} + +func recordSearchCount(ctx context.Context, origin string, reqp *int, okp *bool) { + searchCount.Record(ctx, + int64(*reqp), + metric.WithAttributes(searchOrigin(origin), searchSuccess(*okp))) +} diff --git a/internal/httpreader/option.go b/internal/httpreader/option.go new file mode 100644 index 000000000..355b185ea --- /dev/null +++ b/internal/httpreader/option.go @@ -0,0 +1,25 @@ +package httpreader + +import ( + "context" + "net/http" +) + +// Option is used to set options in [New]. +type Option func(context.Context, *Reader) error + +// WithSize sets the size of the HTTP resource and skips rangefinding. +func WithSize(sz int64) Option { + return func(_ context.Context, r *Reader) error { + r.size = sz + return nil + } +} + +// WithHeaders sets additional headers for requests. +func WithHeaders(h http.Header) Option { + return func(_ context.Context, r *Reader) error { + r.headers = h + return nil + } +} diff --git a/internal/httpreader/ragel.sh b/internal/httpreader/ragel.sh new file mode 100644 index 000000000..cbf9b4698 --- /dev/null +++ b/internal/httpreader/ragel.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -e +ragel-go -F1 -o tmp.go content_range.rl +trap 'rm tmp.go tmp.ri ||:' EXIT +{ + printf '// Code generated by ragel-go. DO NOT EDIT.\n\n' + sed '/^[[:space:]]\+$/d' < tmp.go +} | + gofmt -s > content_range.go diff --git a/internal/httpreader/reader.go b/internal/httpreader/reader.go new file mode 100644 index 000000000..b0ffe9d32 --- /dev/null +++ b/internal/httpreader/reader.go @@ -0,0 +1,442 @@ +package httpreader + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "math" + "net/http" + "net/url" + "runtime" + "strings" + "time" +) + +// Reader implements [io.ReaderAt] over an HTTP resource that implements range requests. +// +// The Reader is guaranteed to only makes GET requests. +// +// Callers can use an [io.SectionReader] if they require an [io.Reader] interface. +type Reader struct { + ctx context.Context // This is OK, I swear. + c *http.Client + done context.CancelFunc + res string + headers http.Header + + validator string + modTime time.Time + size int64 // Discovered size of the resource "res." + guessed bool // Did the rangefind method guess at the size? +} + +// New constructs a reader for the resource at "uri", using the supplied [context.Context] and [http.Client]. +// +// The Context is used for the lifetime of the Reader. +// All ReadAt calls translate to network roundtrips. +// Callers should arrange for buffering and chunking for the best performance. +// They may also want to check the discovered size and eagerly fetch the resource if it's under some threshold. +// This is not handled in this package. +// +// The returned Reader must have [Reader.Close] called or the program may panic. +func New(ctx context.Context, c *http.Client, uri string, opts ...Option) (*Reader, error) { + ctx, done := context.WithCancel(ctx) + ok := false + defer func() { + if !ok { + done() + } + }() + r := Reader{ + ctx: ctx, + done: done, + c: c, + res: uri, + size: -1, + } + for _, opt := range opts { + if err := opt(ctx, &r); err != nil { + return nil, err + } + } + if err := r.rangefind(); err != nil { + return nil, err + } + + _, file, line, _ := runtime.Caller(1) + runtime.SetFinalizer(&r, func(r *Reader) { + panic(fmt.Sprintf("%s:%d: httpreader.Reader not closed", file, line)) + }) + ok = true + return &r, nil +} + +// Close implements [io.Closer]. +// +// Callers must call Close on any Reader to release resources, or the program may panic. +func (r *Reader) Close() error { + runtime.SetFinalizer(r, nil) + r.done() + return nil +} + +var ( + _ io.ReaderAt = (*Reader)(nil) + _ io.Closer = (*Reader)(nil) +) + +// ReadAt implements [io.ReaderAt]. +func (r *Reader) ReadAt(b []byte, off int64) (int, error) { + if len(b) == 0 { + return 0, nil + } + + req, err := http.NewRequestWithContext(r.ctx, http.MethodGet, r.res, nil) + if err != nil { + return 0, err + } + r.setRange(req, off, int64(len(b))) + res, err := r.c.Do(req) + if err != nil { + return 0, err + } + defer res.Body.Close() + switch res.StatusCode { + case http.StatusPartialContent: // OK + case http.StatusOK: + if off != 0 { + return 0, fmt.Errorf("httpreader: server botch: %d (range: %s)", res.StatusCode, req.Header.Get(`range`)) + } + case http.StatusRequestedRangeNotSatisfiable: + return 0, fmt.Errorf("httpreader: server botch: %d (range: %s)", res.StatusCode, req.Header.Get(`range`)) + default: + return 0, fmt.Errorf("httpreader: unexpected status: %d", res.StatusCode) + } + n, err := io.ReadFull(res.Body, b) + atEOF := off+int64(n) == r.size + switch { + case errors.Is(err, io.EOF): + err = io.ErrNoProgress + case atEOF: + err = io.EOF + } + + return n, err +} + +// SetRange sets the Range header for "sz" bytes starting at "off". +func (r *Reader) setRange(req *http.Request, off int64, sz int64) { + switch { + case r.validator != "": + req.Header.Set(`If-Match`, r.validator) + case !r.modTime.IsZero(): + req.Header.Set(`If-Unmodified-Since`, r.modTime.Format(http.TimeFormat)) + } + // Throw in a branch to make this function useful in the search function. + var last int64 + if r.size < 0 { + last = off + sz + } else { + last = min(off+sz, r.size) + } + last-- + req.Header.Set(`Range`, fmt.Sprintf(`bytes=%d-%d`, off, last)) + for n, vs := range r.headers { + for _, v := range vs { + req.Header.Add(n, v) + } + } +} + +// Rangefind attempts to discover the size, range support, and version information for the remote resource. +func (r *Reader) rangefind() error { + // If the Option was provided, we're done. + if r.size != -1 { + return nil + } + // First, issue an extremely weird range request: request the last byte of a resource without issuing a HEAD to check if the server supports it. + // The returned request then has some caveats, explained below. + // The GET-only flow is done to be compatible with some signed URLs and to prevent middleware boxes from lying when seeing the HEAD request. + req, err := http.NewRequestWithContext(r.ctx, http.MethodGet, r.res, nil) + if err != nil { + return err + } + + req.Header.Add(`range`, `bytes=-1`) + for n, vs := range r.headers { + for _, v := range vs { + req.Header.Add(n, v) + } + } + res, err := r.c.Do(req) + if err != nil { + return err + } + if err := res.Body.Close(); err != nil { + return err + } + // RFC7232 handling, so we error out if the resource changes. + // This _really_ shouldn't happen with something content-addressed like container storage, but better safe than sorry. + // + // A server that interprets each range as a unique resource for the purpose of generating a validator would break this. + // I think that's not strictly RFC-compliant, though. + r.validator = res.Header.Get(`etag`) + if strings.HasPrefix(r.validator, `W/`) { + // Ignore weak discriminators. Can't use these, per RFC. + r.validator = "" + } + r.modTime, err = time.Parse(http.TimeFormat, res.Header.Get(`last-modified`)) + if err != nil { + // Use the reported time, unless that was absent or malformed; in that case just use "now". + r.modTime = time.Now().UTC() + } + + switch res.StatusCode { + case http.StatusPartialContent: + case http.StatusRequestedRangeNotSatisfiable: + case http.StatusOK: + // All the above are OK + default: + return fmt.Errorf("httpreader: unexpected response status: %s", res.Status) + } + + // Now, for a bunch of RFC7233 tricks: + // Does the server advertise support? + var rangeOK bool +Ranges: + for _, t := range res.Header["Accept-Ranges"] { + switch t { + case "bytes": + rangeOK = true + fallthrough + case "none": + break Ranges + } + } + // Is the Content-Range populated correctly? + if cr := res.Header.Get(`content-range`); cr != "" { + var rg ContentRange + if err := rg.Parse(cr); err == nil { + switch { + case rg.Length != -1: + r.size = rg.Length + case rg.Length == -1 && rg.Last != -1: + r.size = rg.Last + 1 + } + } + } + // If the server just responded with 200 OK _because_ of the negative range request but otherwise supports ranges, this captures the size: + if res.StatusCode == http.StatusOK { + r.size = res.ContentLength + } + rangeOK = rangeOK || res.StatusCode == http.StatusPartialContent && r.size != -1 + + switch { + case !rangeOK && r.size == -1: + // Can't issue range requests and couldn't find the size. + fallthrough + case !rangeOK && r.size != -1: + // Can't issue range requests and found the size. + return rangeUnsupported(r.res) + case rangeOK && r.size == -1: + // Can issue range requests and couldn't find the size. + // + // As a last-ditch effort, search for it. + if err := r.searchSize(); err != nil { + return err + } + case rangeOK && r.size != -1: + // Can issue range requests and found the size. + } + + return nil +} + +// RangeUnsupported returns an informative error that is [errors.Is]-wise equal +// to [errors.ErrUnsupported]. +func rangeUnsupported(s string) error { + u, err := url.Parse(s) + if err != nil { + panic(fmt.Sprintf("programmer error: bogus URL: %v", err)) + } + return &errRangeUnsupported{u: u} +} + +var _ error = (*errRangeUnsupported)(nil) + +type errRangeUnsupported struct{ u *url.URL } + +func (e *errRangeUnsupported) Error() string { + return fmt.Sprintf("httpreader: origin %q does not support range requests (resource: %q)", e.u.Host, e.u.Path) +} + +func (e *errRangeUnsupported) Is(tgt error) bool { + return tgt == errors.ErrUnsupported +} + +// SearchSize handles a server that advertises range support but refuses to specify the length. +// +// This method brute forces the length by finding bounds where the lower is satisfiable and upper is not, +// then performing range requests between them to find the last byte returned. +// This algorithm is transfer- and latency-inefficient and any server that forces us to resort to this should be ashamed. +func (r *Reader) searchSize() error { + // No idea how this could happen -- the Content-Range handling should have deduced a size even if the server refuses to put the total length in the header. + // + // Best I can tell this closer in complexity to an "exponential search" than a "binary search". + // The algorithm implemented is (I think) novel because it's looking for nonexistence instead of a known index or value. + // That means the complexity (meaning number of requests) here should be something like O(log(n)+m). + // The window grows with the requests because the power of logarithms means we really want to get to that quickly. + const window = (1 << 20) * 250 // 250 MiB + var lower, upper int64 = 0, window + proto, err := http.NewRequest(http.MethodGet, r.res, nil) + if err != nil { + return err + } + u, _ := url.Parse(r.res) + reqs := 1 + success := false + defer recordSearchCount(r.ctx, u.Host, &reqs, &success) + // Rangefind an initial window. + var rg ContentRange +Rangefind: + for ; ; reqs++ { + req := proto.Clone(r.ctx) + r.setRange(req, upper-1, 3) + res, err := r.c.Do(req) + if err != nil { + return err + } + res.Body.Close() + + switch res.StatusCode { + case http.StatusPartialContent: + case http.StatusRequestedRangeNotSatisfiable: + // These are some bounds, good to search. + break Rangefind + default: // ??? + return fmt.Errorf("httpreader: unexpected status while searching for end: %s", res.Status) + } + + // Check if the Content-Range is populated with a size for some reason, now: + cr := res.Header.Get(`content-range`) + // This shouldn't happen, but we're doing weird access patterns so just belt and suspenders (bracers): + if cr == "" { + return fmt.Errorf("httpreader: origin %q has an odd reading of RFC7233", u.Host) + } + if err := rg.Parse(cr); err != nil { + return fmt.Errorf("httpreader: origin %q header botch: %w", u.Host, err) + } + if rg.Length != -1 { + // Should never be 0: layers can't be 0-sized. + // This assumes 0 _could_ happen just in case it's used in another context. + // + // So, we just needed to make any forwards-range request to get the server to tell us the length. + r.size = rg.Length + success = true + return nil + } + // Check if we manged to request the actual end: + if (rg.Last+1)-rg.First < 3 { + r.size = rg.Last + 1 + success = true + return nil + } + lower = upper + upper += upper + } + + // Now, search using our bounds. + // This requests "searchWin" sized ranges and examines the response to see how much was returned. + // + // - If nothing is returned, the upper bound is dropped. + // - If the whole range is returned, the lower bound is raised. + // - If part of the range is returned, we've found the end. + // + // A bigger window means fewer requests, but more transfer. + // The worst case for requests should be something like: + // + // log(window/searchWin) = log(((1<<20)*250)/1024) ≅ 5.4 = 6 + // + // The searchWin value would need to be 3072 (3 KiB) to get the worst case requests under 5. + // Given that TCP receive buffers should be well north of that on machines, that's + // + // (3072*5) - (1024*6) = 9216 + // + // extra bytes of transfer every time this function hits worst-case behavior, at best. + // The above napkin math doesn't bother with protocol overhead. + // Alternatively, the window size could be shrunk, but that means more rangefinding requests in the loop above. + // Shrinking the window size to 100 MiB still results in a worst-case 5.3 requests here. + // Seeing as one additional rangefinding request negates any advantage, the current magic numbers seem OK. + const searchWin = 1024 + lowerBucket, upperBucket := lower/searchWin, upper/searchWin + nBuckets := upperBucket - lowerBucket + if (upper-lower)%searchWin != 0 { + nBuckets++ + } + ceil := int(math.Ceil(math.Log(float64(nBuckets)))) + attrs := []slog.Attr{ + slog.Int("big_O", reqs+ceil), + slog.Int("bucket_size", searchWin), + slog.Int64("buckets", nBuckets), + slog.GroupAttrs( + "range", + slog.Int64("lower", lower), + slog.Int64("upper", upper), + ), + } + var pivots []int64 + lim := reqs + (2 * ceil) + for reqs++; reqs <= lim; reqs++ { + pivot := ((upper - lower) / 2) + pivot += lower + pivots = append(pivots, pivot) + req := proto.Clone(r.ctx) + r.setRange(req, pivot, searchWin) + res, err := r.c.Do(req) + if err != nil { + return err + } + res.Body.Close() + switch res.StatusCode { + case http.StatusPartialContent: + case http.StatusRequestedRangeNotSatisfiable: + upper = pivot + continue + default: // ??? + return fmt.Errorf("httpreader: unexpected status: %s", res.Status) + } + cr := res.Header.Get(`content-range`) + if cr == "" { + return fmt.Errorf("httpreader: origin %q has an odd reading of RFC7233", u.Host) + } + if err := rg.Parse(cr); err != nil { + return fmt.Errorf("httpreader: origin %q header botch: %w", u.Host, err) + } + // Check if we manged to request the actual end: + if (rg.Last+1)-rg.First < searchWin { + r.size = rg.Last + 1 + break + } + lower = pivot + } + if reqs > lim { + return errors.New("loop") + } + slog.LogAttrs(r.ctx, slog.LevelDebug, "size search done", append( + attrs, + slog.Int("requests", reqs), + slog.Any("pivots", pivots), + )...) + + success = true + return nil +} + +// Size reports the discovered size of the HTTP resource. +// +// This function may report a guessed size less than zero. +// This means the Reader will issue range requests, but will allow arbitrary offsets. +// Callers should expect something like [io.Reader] semantics in that case. +func (r *Reader) Size() int64 { return r.size } diff --git a/internal/httpreader/reader_test.go b/internal/httpreader/reader_test.go new file mode 100644 index 000000000..4e03d63a1 --- /dev/null +++ b/internal/httpreader/reader_test.go @@ -0,0 +1,321 @@ +package httpreader + +import ( + "bytes" + "context" + "fmt" + "io" + "math/rand" + "net" + "net/http" + "net/http/httptest" + "os" + "reflect" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "testing/iotest" + "testing/quick" + "time" + + "github.com/quay/claircore/test" +) + +var tKey = &struct{}{} + +type compliantServer struct { + f *os.File + sz int64 + mod time.Time +} + +func (s *compliantServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.Header().Set(`Allow`, http.MethodGet) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set(`Content-Type`, `application/octet-stream`) + // SectionReader trick to get multiple independent cursors into the file. + content := io.NewSectionReader(s.f, 0, s.sz) + http.ServeContent(w, r, "randfile", s.mod, content) +} + +func fileserver(t testing.TB, name string) *compliantServer { + var err error + srv := new(compliantServer) + srv.f, err = os.Open(name) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := srv.f.Close(); err != nil { + t.Error(err) + } + }) + fi, err := srv.f.Stat() + if err != nil { + t.Fatal(err) + } + srv.sz = fi.Size() + srv.mod = test.Modtime(t, name) + return srv +} + +// NoLengthServer is a cut-rate implementation of a byte-range aware server. +// +// It does most of the "normal" work, but strenuously avoids noticing the resource's length. +type noLengthServer struct { + zero *os.File + size int64 + prev int64 +} + +func newNoLength(t *testing.T, size int64) *noLengthServer { + zero, err := os.Open("/dev/zero") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := zero.Close(); err != nil { + t.Log(err) + } + }) + return &noLengthServer{ + size: size, + zero: zero, + } +} + +func (s *noLengthServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + t := ctx.Value(tKey).(*testing.T) + if r.Method != http.MethodGet { + panic("unexpected method: " + r.Method) + } + br := r.Header.Get("range") + w.Header().Set("accept-ranges", "bytes") + var b strings.Builder + defer func() { + if b.Len() > 0 { + t.Log(b.String()) + } + }() + switch { + case br == `bytes=-1`: + // Trigger our dumb behavior. + if _, err := io.Copy(w, io.LimitReader(s.zero, s.size)); err != nil { + t.Logf("expected error writing last byte: %v", err) + } + case br == "": + panic("no range") + case !strings.HasPrefix(br, `bytes=`): + panic("non-bytes range: " + br) + default: + _, req, ok := strings.Cut(br, "=") + if !ok { + panic("weird range: " + br) + } + firstStr, lastStr, ok := strings.Cut(req, "-") + if !ok { + panic("weird range: " + br) + } + first, _ := strconv.ParseInt(firstStr, 10, 64) + reqLast, _ := strconv.ParseInt(lastStr, 10, 64) + fmt.Fprintf(&b, "req: %s, len: %d, ", req, (reqLast+1)-first) + if first >= s.size { + b.WriteString("serving: 0") + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + return + } + if first == s.prev { + w.WriteHeader(http.StatusBadRequest) + return + } + s.prev = first + last := min(reqLast, s.size-1) + fmt.Fprintf(&b, "serving: %d", (last+1)-first) + w.Header().Set("content-range", fmt.Sprintf("bytes %d-%d/*", first, last)) + w.WriteHeader(http.StatusPartialContent) + if n, err := io.Copy(w, io.LimitReader(s.zero, (last+1)-first)); err != nil && n != 0 { + t.Error(err) + } + } +} + +func TestCompliant(t *testing.T) { + t.Parallel() + randfile := ensureRandfile(t) + + ctx := test.Logging(t) + h := fileserver(t, randfile) + srv := httptest.NewUnstartedServer(h) + srv.Config.BaseContext = func(_ net.Listener) context.Context { + return context.WithValue(ctx, tKey, t) + } + srv.Start() + t.Cleanup(srv.Close) + rd, err := New(ctx, srv.Client(), srv.URL) + if err != nil { + t.Fatal(err) + } + defer rd.Close() + f, err := os.Open(randfile) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + if err := quick.Check(checkEq(t, f, rd), &eqConfig); err != nil { + t.Error(err) + } +} + +func TestLengthSearch(t *testing.T) { + t.Parallel() + ctx := test.Logging(t) + + doSearch := func(sz int64) bool { + srv := httptest.NewUnstartedServer(newNoLength(t, sz)) + srv.Config.BaseContext = func(_ net.Listener) context.Context { + return context.WithValue(ctx, tKey, t) + } + srv.Start() + defer srv.Close() + rd, err := New(ctx, srv.Client(), srv.URL) + if err != nil { + t.Fatal(err) + } + defer rd.Close() + + got, want := rd.Size(), sz + if got != want { + t.Logf("got: %d (%d MiB), want: %d (%d MiB)", got, got/(1<<20), want, want/(1<<20)) + return false + } + return true + } + + if err := quick.Check(doSearch, &quick.Config{ + MaxCount: 10, + MaxCountScale: 5, + Values: func(args []reflect.Value, rng *rand.Rand) { + const max = 10240 + sz := rng.Int63n(max) + 1 // Make this [1, max] + args[0] = reflect.ValueOf((1 << 20) * sz) + }, + }); err != nil { + t.Error(err) + } +} + +func TestMulti(t *testing.T) { + t.Parallel() + randfile := ensureRandfile(t) + ctx := test.Logging(t) + h := fileserver(t, randfile) + srv := httptest.NewUnstartedServer(h) + srv.Config.BaseContext = func(_ net.Listener) context.Context { + return context.WithValue(ctx, tKey, t) + } + srv.Start() + t.Cleanup(srv.Close) + rd, err := New(ctx, srv.Client(), srv.URL) + if err != nil { + t.Fatal(err) + } + defer rd.Close() + var wg sync.WaitGroup + start := make(chan struct{}) + cfg := eqConfig + cfg.MaxCount = 4 + + for range 4 { + wg.Add(1) + f, err := os.Open(randfile) + if err != nil { + t.Fatal(err) + } + defer f.Close() + go func() { + defer wg.Done() + <-start + if err := quick.Check(checkEq(t, f, rd), &cfg); err != nil { + t.Error(err) + } + }() + } + + close(start) + wg.Wait() +} + +const randfileSz = (1 << 20) * 50 + +var ( + randfilePath string + randfileOnce sync.Once +) + +func ensureRandfile(t testing.TB) string { + randfileOnce.Do(func() { + stamp := test.Modtime(t, ".") + randfilePath = test.GenerateFixture(t, "randfile", stamp, genRandfile) + }) + if t.Failed() { + return "" + } + return randfilePath +} + +func genRandfile(t testing.TB, f *os.File) { + defer f.Close() + s := rand.NewSource(660096000) + rng := rand.New(s) + if err := f.Truncate(0); err != nil { + t.Fatal(err) + } + if _, err := f.ReadFrom(io.LimitReader(rng, randfileSz)); err != nil { + t.Fatal(err) + } +} + +func checkEq(t *testing.T, f *os.File, rd *Reader) func(int64, int64) bool { + var ct atomic.Int64 + t.Helper() + t.Cleanup(func() { + t.Logf("ran %d times", ct.Load()) + }) + return func(sz, off int64) bool { + ct.Add(1) + // Read our want value: + var want bytes.Buffer + r := io.NewSectionReader(f, off, sz) + if _, err := io.Copy(&want, r); err != nil { + t.Error(err) + return false + } + t.Logf("read %d bytes at offset %d", sz, off) + + // Read our got value: + r = io.NewSectionReader(rd, off, sz) + if err := iotest.TestReader(r, want.Bytes()); err != nil { + t.Error(err) + return false + } + return true + } +} + +var eqConfig = quick.Config{ + MaxCount: 10, + MaxCountScale: 5, + Values: func(args []reflect.Value, rng *rand.Rand) { + const max = 8192 + sz := rng.Int63n(max) + 1 // Make this [1, max] + args[0] = reflect.ValueOf(sz) + args[1] = reflect.ValueOf(rng.Int63n(randfileSz)) // Offset + }, +} diff --git a/internal/zreader/zreader.go b/internal/zreader/zreader.go index 0afdee23c..400c5014d 100644 --- a/internal/zreader/zreader.go +++ b/internal/zreader/zreader.go @@ -89,9 +89,9 @@ var ( // DetectCompression reports the compression type indicated based on the header // contained in the passed byte slice. // -// "CmpNone" is returned if all detectors report false, but it's possible that +// [KindNone] is returned if all detectors report false, but it's possible that // it's just a scheme unsupported by this package. -func detectCompression(b []byte) Compression { +func DetectCompression(b []byte) Compression { t := make([]byte, len(b)) for c, d := range detectors { n, l := copy(t, b), len(d.Mask) @@ -153,7 +153,7 @@ func detect(r io.Reader) (io.ReadCloser, Compression, error) { // // All the return types are a little different, so they're handled in the // switch arms. - switch c := detectCompression(b); c { + switch c := DetectCompression(b); c { case KindGzip: z, err := gzip.NewReader(br) return z, c, err diff --git a/libindex/fetcher.go b/libindex/fetcher.go index a92523771..ed9cd4ee1 100644 --- a/libindex/fetcher.go +++ b/libindex/fetcher.go @@ -17,12 +17,14 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" "go.opentelemetry.io/otel/trace" "golang.org/x/sync/errgroup" "github.com/quay/claircore" "github.com/quay/claircore/indexer" "github.com/quay/claircore/internal/cache" + "github.com/quay/claircore/internal/httpreader" "github.com/quay/claircore/internal/httputil" "github.com/quay/claircore/internal/wart" "github.com/quay/claircore/internal/zreader" @@ -125,6 +127,43 @@ func (a *RemoteFetchArena) fetchInto(ctx context.Context, l *claircore.Layer, cl span.End() }() + var d details + d, err = a.inspect(ctx, desc) + if err != nil { + return err + } + + HTTPReader: + switch { // Switch for a single case to be able to break. + case d.Uncompressed: + var opts []httpreader.Option + if d.ContentLength > 0 { + opts = append(opts, httpreader.WithSize(d.ContentLength)) + } + if len(desc.Headers) != 0 { + opts = append(opts, httpreader.WithHeaders(desc.Headers)) + } + var rd *httpreader.Reader + rd, err = httpreader.New(ctx, a.wc, desc.URI, opts...) + switch { + case err == nil: + if err = l.Init(ctx, desc, rd); err != nil { + return err + } + *cl = closeFunc(func() (err error) { + err = errors.Join(l.Close(), rd.Close()) + return err + }) + a.logger(desc).DebugContext(ctx, "using httpreader") + return nil + case errors.Is(err, errors.ErrUnsupported): + err = nil + break HTTPReader + default: + return err + } + } + // NB This is not closed on purpose. The [io.Closer] populated by this // function holds the pointer until that function is cleaned up. Once // nothing has a copy of this [*os.File], the runtime will run all the @@ -135,19 +174,20 @@ func (a *RemoteFetchArena) fetchInto(ctx context.Context, l *claircore.Layer, cl var spool *os.File spool, err = a.files.Get(ctx, key, func(ctx context.Context, _ string) (*os.File, error) { cacheHit = false - return a.fetchFileForCache(ctx, desc) + return a.fetchFileForCache(ctx, desc, d) }) if err != nil { return err } + var f *os.File // This is an owned, independent descriptor for the passed [*os.File]. - f, err := reopen(a.root, spool) + f, err = reopen(a.root, spool) if err != nil { return err } // If this succeeds, "f" is now owned by "l" - if err := l.Init(ctx, desc, f); err != nil { + if err = l.Init(ctx, desc, f); err != nil { return errors.Join(err, f.Close()) } *cl = closeFunc(func() (err error) { @@ -173,12 +213,105 @@ func (f closeFunc) Close() error { return f() } +// Inspect makes a request to the layer's URI and examines the response for +// useful information. +func (a *RemoteFetchArena) inspect(ctx context.Context, desc *claircore.LayerDescription) (d details, err error) { + ctx, span := tracer.Start(ctx, "RemoteFetchArena.inspect") + defer func() { + a.logger(desc).DebugContext(ctx, "inspected resource", "ok", err == nil, "details", &d) + span.RecordError(err) + span.End() + }() + span.SetStatus(codes.Error, "") + + var req *http.Request + var res *http.Response + req, err = http.NewRequestWithContext(ctx, http.MethodGet, desc.URI, nil) + req.Header = http.Header(desc.Headers).Clone() + req.Header.Set(`claircore-reason`, `inspect`) + req.Header.Set(`range`, `bytes=0-15`) + if err != nil { + return d, fmt.Errorf("fetcher: failed to construct request: %w", err) + } + res, err = a.wc.Do(req) + if err != nil { + return d, fmt.Errorf("fetcher: request failed: %w", err) + } + err = httputil.CheckResponse(res, http.StatusOK, http.StatusPartialContent) + if err != nil { + return d, fmt.Errorf("fetcher: %w", err) + } + head := make([]byte, 16) + _, err = io.ReadFull(res.Body, head) + _ = res.Body.Close() + if err != nil { + return d, fmt.Errorf("fetcher: unexpected read: %w", err) + } + + const ( + ctKey = `content-type` + arKey = `accept-ranges` + ) + ct := res.Header.Get(ctKey) + span.SetAttributes( + semconv.HTTPResponseStatusCode(res.StatusCode), + semconv.HTTPResponseBodySize(int(res.ContentLength)), + semconv.HTTPResponseHeader(ctKey, ct), + semconv.HTTPResponseHeader(arKey, res.Header.Get(arKey)), + ) + if ct == `application/octect-stream` { + if zreader.DetectCompression(head) == zreader.KindNone { + ct = `application/x-tar` + } + } + switch res.StatusCode { + case http.StatusOK: + d.ContentLength = res.ContentLength + case http.StatusPartialContent: + var cr httpreader.ContentRange + if err := cr.Parse(res.Header.Get(`content-range`)); err == nil { + d.ContentLength = cr.Length + } + } + d.Uncompressed = ct == "application/x-tar" || strings.HasSuffix(ct, ".tar") + d.RangeOK = res.Header.Get(arKey) == `bytes` || res.StatusCode == http.StatusPartialContent + + span.SetStatus(codes.Ok, "") + return d, nil +} + +var _ slog.LogValuer = (*details)(nil) + +// Details is the useful information reported by [RemoteFetchArena.inspect]. +type details struct { + // The content length, if known. + ContentLength int64 + // The resource reports to be known non-compressed contents. + Uncompressed bool + // The server reports supporting "bytes" via the "Accept-Ranges" header. + RangeOK bool +} + +// LogValue implements [slog.LogValuer]. +func (d *details) LogValue() slog.Value { + return slog.GroupValue( + slog.Int64("content-length", d.ContentLength), + slog.Bool("uncompressed", d.Uncompressed), + slog.Bool("range-ok", d.RangeOK), + ) +} + +// Logger returns a [*slog.Logger] with predefined attributes. +func (a *RemoteFetchArena) logger(desc *claircore.LayerDescription) *slog.Logger { + return slog.With("arena", a.root.Name(), "layer", desc.Digest, "uri", desc.URI) +} + // FetchFileForCache is the inner function used inside the [cache.Live]. // // Because we know we're the only concurrent call that's dealing with this blob, // we can be a bit more lax. -func (a *RemoteFetchArena) fetchFileForCache(ctx context.Context, desc *claircore.LayerDescription) (*os.File, error) { - log := slog.With("arena", a.root.Name(), "layer", desc.Digest, "uri", desc.URI) +func (a *RemoteFetchArena) fetchFileForCache(ctx context.Context, desc *claircore.LayerDescription, _ details) (*os.File, error) { + log := a.logger(desc) ctx, span := tracer.Start(ctx, "RemoteFetchArena.fetchFileForCache") defer span.End() span.SetStatus(codes.Error, "") @@ -210,6 +343,7 @@ func (a *RemoteFetchArena) fetchFileForCache(ctx context.Context, desc *claircor URL: url, Header: http.Header(desc.Headers).Clone(), }).WithContext(ctx) + req.Header.Set(`claircore-reason`, `fetch`) resp, err := a.wc.Do(req) if err != nil { return nil, fmt.Errorf("fetcher: request failed: %w", err) diff --git a/libindex/fetcher_test.go b/libindex/fetcher_test.go index ff9508baa..cf973c598 100644 --- a/libindex/fetcher_test.go +++ b/libindex/fetcher_test.go @@ -2,19 +2,25 @@ package libindex import ( "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" "context" "crypto/sha256" "fmt" "io" + "io/fs" "math/rand" "net/http" "net/http/httptest" "os" - "path/filepath" + "path" "runtime" + "slices" "strconv" "sync/atomic" "testing" + "time" "github.com/quay/claircore" "github.com/quay/claircore/internal/wart" @@ -111,144 +117,198 @@ func TestFetchInvalid(t *testing.T) { func TestFetchConcurrent(t *testing.T) { t.Parallel() - ctx := test.Logging(t) - descs, h := commonLayerServer(t, 25) - srv := httptest.NewUnstartedServer(h) - srv.Start() - for i := range descs { - descs[i].URI = srv.URL + descs[i].URI - } - t.Cleanup(srv.Close) - a := NewRemoteFetchArena(srv.Client(), t.TempDir()) - t.Cleanup(func() { - if err := a.Close(ctx); err != nil { - t.Error(err) + setup := func(t *testing.T, ct int, gz bool) (*RemoteFetchArena, []claircore.LayerDescription) { + t.Helper() + stamp := test.Modtime(t, ".") + name := test.GenerateFixture(t, fmt.Sprintf("layers_%02d_%v.zip", ct, gz), stamp, generateTarballs(ct, gz)) + f, err := os.Open(name) + if err != nil { + t.Fatal(err) } - }) - - t.Run("OldInterface", func(t *testing.T) { - t.Run("Thread", func(t *testing.T) { - run := func(a *RemoteFetchArena, ls []claircore.LayerDescription) func(*testing.T) { - ps := wart.DescriptionsToLayers(ls) - // Leave the bottom half the same, shuffle the top half. - off := len(ps) / 2 - rand.Shuffle(off, func(i, j int) { - i, j = i+off, j+off - ps[i], ps[j] = ps[j], ps[i] - }) - return func(t *testing.T) { - t.Parallel() - ctx := test.Logging(t) - f := a.Realizer(ctx) - t.Cleanup(func() { - if err := f.Close(); err != nil { - t.Error(err) - } - }) - if err := f.Realize(ctx, ps); err != nil { - t.Error(err) - } - } - } - for i := 0; i < runtime.GOMAXPROCS(0); i++ { - t.Run(strconv.Itoa(i), run(a, descs)) + t.Cleanup(func() { + if err := f.Close(); err != nil { + t.Fatal(err) } }) - }) - - t.Run("NewInterface", func(t *testing.T) { - t.Run("Thread", func(t *testing.T) { - run := func(a *RemoteFetchArena, descs []claircore.LayerDescription) func(*testing.T) { - ds := make([]claircore.LayerDescription, len(descs)) - copy(ds, descs) - // Leave the bottom half the same, shuffle the top half. - off := len(ds) / 2 - rand.Shuffle(off, func(i, j int) { - i, j = i+off, j+off - ds[i], ds[j] = ds[j], ds[i] - }) - return func(t *testing.T) { - t.Parallel() - ctx := test.Logging(t) - f := a.Realizer(ctx).(*FetchProxy) - defer func() { - if err := f.Close(); err != nil { - t.Error(err) - } - }() - ls, err := f.RealizeDescriptions(ctx, ds) - if err != nil { - t.Errorf("RealizeDescriptions error: %v", err) - } - t.Logf("layers: %v", ls) - } - } - for i := 0; i < runtime.GOMAXPROCS(0); i++ { - t.Run(strconv.Itoa(i), run(a, descs)) - } - }) - }) -} - -func commonLayerServer(t testing.TB, ct int) ([]claircore.LayerDescription, http.Handler) { - // TODO(hank) Cache all this? The contents are basically static. - t.Helper() - dir := t.TempDir() - descs := make([]claircore.LayerDescription, ct) - fetch := make(map[string]*uint64, ct) - for i := range ct { - n := strconv.Itoa(i) - f, err := os.Create(filepath.Join(dir, strconv.Itoa(i))) + fi, err := f.Stat() if err != nil { t.Fatal(err) } - h := sha256.New() - w := tar.NewWriter(io.MultiWriter(f, h)) - if err := w.WriteHeader(&tar.Header{ - Name: n, - Size: 33, - }); err != nil { + z, err := zip.NewReader(f, fi.Size()) + if err != nil { t.Fatal(err) } - fmt.Fprintf(w, "%032d\n", i) + desc := make([]claircore.LayerDescription, len(z.File)) + reqCt := make(map[string]*uint64) + contentType := `application/vnd.oci.image.layer.nondistributable.v1.tar` + if gz { + contentType += `+gzip` + } + now := time.Now() + h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := path.Base(r.URL.EscapedPath()) + ct := reqCt[n] + atomic.AddUint64(ct, 1) + w.Header().Set(`content-type`, contentType) + b, err := fs.ReadFile(z, n) + if err != nil { + t.Error(err) + w.WriteHeader(http.StatusInternalServerError) + return + } + http.ServeContent(w, r, n, now, bytes.NewReader(b)) + }) - if err := w.Close(); err != nil { - t.Fatal(err) + srv := httptest.NewUnstartedServer(h) + srv.EnableHTTP2 = true + srv.StartTLS() + for i, zf := range z.File { + reqCt[zf.Name] = new(uint64) + d := &desc[i] + d.URI = srv.URL + "/" + zf.Name + d.Digest = zf.Comment + d.Headers = make(http.Header) + d.MediaType = contentType } - if err := f.Close(); err != nil { - t.Fatal(err) + + t.Cleanup(srv.Close) + a := NewRemoteFetchArena(srv.Client(), t.TempDir()) + ctx := test.Logging(t) + t.Cleanup(func() { + if err := a.Close(ctx); err != nil { + t.Error(err) + } + }) + return a, desc + } + oldInterface := func(a *RemoteFetchArena, ls []claircore.LayerDescription) func(*testing.T) { + ps := wart.DescriptionsToLayers(ls) + shuffleSlice(ps) + return func(t *testing.T) { + t.Parallel() + ctx := test.Logging(t) + f := a.Realizer(ctx) + t.Cleanup(func() { + if err := f.Close(); err != nil { + t.Error(err) + } + }) + if err := f.Realize(ctx, ps); err != nil { + t.Error(err) + } } - l := &descs[i] - l.URI = "/" + strconv.Itoa(i) - fetch[l.URI] = new(uint64) - l.Digest = fmt.Sprintf("sha256:%x", h.Sum(nil)) - l.Headers = make(http.Header) - l.MediaType = `application/vnd.oci.image.layer.nondistributable.v1.tar` - if err != nil { - t.Fatal(err) + } + newInterface := func(a *RemoteFetchArena, descs []claircore.LayerDescription) func(*testing.T) { + ds := slices.Clone(descs) + shuffleSlice(ds) + return func(t *testing.T) { + t.Parallel() + ctx := test.Logging(t) + f := a.Realizer(ctx).(*FetchProxy) + defer func() { + if err := f.Close(); err != nil { + t.Error(err) + } + }() + ls, err := f.RealizeDescriptions(ctx, ds) + if err != nil { + t.Errorf("RealizeDescriptions error: %v", err) + } + t.Logf("layers: %v", ls) } } - t.Cleanup(func() { - // We know we're doing 2 sets of fetches. - limit := ct * 2 * runtime.GOMAXPROCS(0) - var total int - for _, v := range fetch { - total += int(*v) - } - switch { - case total > limit: - t.Errorf("more fetches than should be possible: %d > %d", total, limit) - case total == limit: - t.Errorf("prevented no fetches: %d == %d", total, limit) - case total < limit: - t.Logf("prevented %[3]d fetches: %[1]d < %d", total, limit, limit-total) + for i := range 2 { + gz := i == 1 + n := `Uncompressed` + if gz { + n = `Compressed` } + t.Run(n, func(t *testing.T) { + t.Parallel() + a, descs := setup(t, 25, gz) + t.Run("OldInterface", func(t *testing.T) { + for i := 0; i < runtime.GOMAXPROCS(0); i++ { + t.Run(strconv.Itoa(i), oldInterface(a, descs)) + } + }) + + t.Run("NewInterface", func(t *testing.T) { + for i := 0; i < runtime.GOMAXPROCS(0); i++ { + t.Run(strconv.Itoa(i), newInterface(a, descs)) + } + }) + }) + } +} + +// Leave the bottom half the same, shuffle the top half. +func shuffleSlice[S ~[]E, E any](s S) { + off := len(s) / 2 + rand.Shuffle(off, func(i, j int) { + i, j = i+off, j+off + s[i], s[j] = s[j], s[i] }) - inner := http.FileServer(http.Dir(dir)) - return descs, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ct := fetch[r.URL.Path] - atomic.AddUint64(ct, 1) - inner.ServeHTTP(w, r) - }) +} + +func generateTarballs(count int, compressed bool) func(testing.TB, *os.File) { + return func(t testing.TB, f *os.File) { + defer f.Close() + defer f.Sync() + t.Attr("count", strconv.Itoa(count)) + t.Attr("compressed", strconv.FormatBool(compressed)) + stamp := test.Modtime(t, ".") + z := zip.NewWriter(f) + defer z.Close() + + var buf bytes.Buffer + buf.Grow(2048) + h := sha256.New() + for i := range count { + h.Reset() + buf.Reset() + n := fmt.Sprintf("%04d", i) + w := io.MultiWriter(&buf, h) + var gz *gzip.Writer + var tw *tar.Writer + if compressed { + gz = gzip.NewWriter(w) + tw = tar.NewWriter(gz) + } else { + tw = tar.NewWriter(w) + } + + err := tw.WriteHeader(&tar.Header{ + Name: n, + Size: 33, + }) + if err != nil { + t.Fatal(err) + } + fmt.Fprintf(tw, "%032d\n", i) + + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if gz != nil { + if err := gz.Close(); err != nil { + t.Fatal(err) + } + } + + fh := &zip.FileHeader{ + Name: n, + Comment: fmt.Sprintf("sha256:%x", h.Sum(nil)), + Modified: stamp, + UncompressedSize64: uint64(buf.Len()), + } + zw, err := z.CreateHeader(fh) + if err != nil { + t.Fatal(err) + } + if _, err := io.Copy(zw, &buf); err != nil { + t.Fatal(err) + } + } + } }