diff --git a/datastore/postgres/index_e2e_test.go b/datastore/postgres/index_e2e_test.go index 7ef33a339..56ebbfc11 100644 --- a/datastore/postgres/index_e2e_test.go +++ b/datastore/postgres/index_e2e_test.go @@ -4,6 +4,7 @@ import ( "context" "strings" "testing" + "time" "github.com/google/go-cmp/cmp" "github.com/jackc/pgx/v5/pgxpool" @@ -61,6 +62,118 @@ func TestIndexE2E(t *testing.T) { } } +func TestRequeueIndexPartialsBatchAndSharedLayers(t *testing.T) { + integration.NeedDB(t) + ctx := test.Logging(t) + pool := pgtest.TestIndexerDB(ctx, t) + store := NewIndexerStore(pool) + scnr := mockScnr{name: "test-scanner", kind: "test", version: "v0.0.1"} + scnrs := indexer.VersionedScanners{scnr} + if err := store.RegisterScanners(ctx, scnrs); err != nil { + t.Fatalf("failed to register scanner: %v", err) + } + + partialOnlyA := &claircore.Layer{Hash: test.RandomSHA256Digest(t)} + sharedWithFinished := &claircore.Layer{Hash: test.RandomSHA256Digest(t)} + partialOnlyB := &claircore.Layer{Hash: test.RandomSHA256Digest(t)} + partialOnlyC := &claircore.Layer{Hash: test.RandomSHA256Digest(t)} + + partialA := claircore.Manifest{ + Hash: test.RandomSHA256Digest(t), + Layers: []*claircore.Layer{partialOnlyA, sharedWithFinished}, + } + finished := claircore.Manifest{ + Hash: test.RandomSHA256Digest(t), + Layers: []*claircore.Layer{sharedWithFinished}, + } + partialB := claircore.Manifest{ + Hash: test.RandomSHA256Digest(t), + Layers: []*claircore.Layer{partialOnlyB}, + } + partialC := claircore.Manifest{ + Hash: test.RandomSHA256Digest(t), + Layers: []*claircore.Layer{partialOnlyC}, + } + + for _, m := range []claircore.Manifest{partialA, finished, partialB, partialC} { + if err := store.PersistManifest(ctx, m); err != nil { + t.Fatalf("failed to persist manifest %s: %v", m.Hash, err) + } + for _, l := range m.Layers { + if err := store.SetLayerScanned(ctx, l.Hash, scnr); err != nil { + t.Fatalf("failed to set layer %s scanned: %v", l.Hash, err) + } + } + } + + for _, m := range []claircore.Manifest{partialA, partialB, partialC} { + ir := &claircore.IndexReport{Hash: m.Hash, State: "IndexPartial"} + if err := store.SetIndexPartial(ctx, ir, scnrs); err != nil { + t.Fatalf("failed to set partial index report %s: %v", m.Hash, err) + } + if err := ageIndexReport(ctx, pool, m.Hash, 2*time.Hour); err != nil { + t.Fatalf("failed to age partial index report %s: %v", m.Hash, err) + } + } + ir := &claircore.IndexReport{Hash: finished.Hash, State: "IndexFinished"} + if err := store.SetIndexFinished(ctx, ir, scnrs); err != nil { + t.Fatalf("failed to set finished index report: %v", err) + } + + n, err := store.RequeueIndexPartials(ctx, time.Hour, 2) + if err != nil { + t.Fatalf("failed to requeue partial index reports: %v", err) + } + if n != 2 { + t.Fatalf("requeued %d partial index reports, want 2", n) + } + + assertManifestScanned := func(m claircore.Manifest, want bool) { + t.Helper() + got, err := store.ManifestScanned(ctx, m.Hash, scnrs) + if err != nil { + t.Fatalf("failed to query manifest %s scanned: %v", m.Hash, err) + } + if got != want { + t.Fatalf("manifest %s scanned = %v, want %v", m.Hash, got, want) + } + } + assertLayerScanned := func(l *claircore.Layer, want bool) { + t.Helper() + got, err := store.LayerScanned(ctx, l.Hash, scnr) + if err != nil { + t.Fatalf("failed to query layer %s scanned: %v", l.Hash, err) + } + if got != want { + t.Fatalf("layer %s scanned = %v, want %v", l.Hash, got, want) + } + } + + assertManifestScanned(partialA, false) + assertManifestScanned(partialB, false) + assertManifestScanned(partialC, true) + assertManifestScanned(finished, true) + + assertLayerScanned(partialOnlyA, false) + assertLayerScanned(partialOnlyB, false) + assertLayerScanned(partialOnlyC, true) + assertLayerScanned(sharedWithFinished, true) +} + +func ageIndexReport(ctx context.Context, pool *pgxpool.Pool, hash claircore.Digest, age time.Duration) error { + const query = ` +UPDATE indexreport ir +SET + updated_at = now() - $2::interval +FROM manifest m +WHERE + ir.manifest_id = m.id + AND m.hash = $1; +` + _, err := pool.Exec(ctx, query, hash, age) + return err +} + // RunAll executes all test steps in sequence func (e *indexE2e) RunAll(t testing.TB) { steps := []struct { diff --git a/datastore/postgres/migrations/indexer/10-indexreport-partial.sql b/datastore/postgres/migrations/indexer/10-indexreport-partial.sql new file mode 100644 index 000000000..976e704b5 --- /dev/null +++ b/datastore/postgres/migrations/indexer/10-indexreport-partial.sql @@ -0,0 +1,13 @@ +ALTER TABLE indexreport +ADD COLUMN IF NOT EXISTS updated_at timestamptz NOT NULL DEFAULT now(); + +UPDATE indexreport +SET + state = COALESCE(NULLIF(state, ''), scan_result ->> 'state') +WHERE + state IS NULL + OR state = ''; + +CREATE INDEX IF NOT EXISTS indexreport_partial_retry_idx ON indexreport (updated_at) +WHERE + state = 'IndexPartial'; diff --git a/datastore/postgres/setindexfinished.go b/datastore/postgres/setindexfinished.go index 7d5baa00f..63873035a 100644 --- a/datastore/postgres/setindexfinished.go +++ b/datastore/postgres/setindexfinished.go @@ -51,7 +51,8 @@ INSERT INTO scanned_manifest (manifest_id, scanner_id) VALUES - ((SELECT manifest_id FROM manifests), $2); + ((SELECT manifest_id FROM manifests), $2) +ON CONFLICT DO NOTHING; ` upsertIndexReport = ` WITH @@ -66,13 +67,15 @@ WITH ) INSERT INTO - indexreport (manifest_id, scan_result) + indexreport (manifest_id, state, scan_result, updated_at) VALUES - ((SELECT manifest_id FROM manifests), $2) + ((SELECT manifest_id FROM manifests), $2, $3, now()) ON CONFLICT (manifest_id) DO - UPDATE SET scan_result = excluded.scan_result; + UPDATE SET state = excluded.state, + scan_result = excluded.scan_result, + updated_at = excluded.updated_at; ` ) @@ -99,7 +102,7 @@ DO } start := time.Now() - _, err = tx.Exec(ctx, upsertIndexReport, ir.Hash, ir) + _, err = tx.Exec(ctx, upsertIndexReport, ir.Hash, ir.State, ir) if err != nil { return fmt.Errorf("failed to upsert scan result: %w", err) } diff --git a/datastore/postgres/setindexpartial.go b/datastore/postgres/setindexpartial.go new file mode 100644 index 000000000..5194eb047 --- /dev/null +++ b/datastore/postgres/setindexpartial.go @@ -0,0 +1,83 @@ +package postgres + +import ( + "context" + "fmt" + "time" + + "github.com/quay/claircore" + "github.com/quay/claircore/indexer" +) + +// SetIndexPartial persists a degraded index report and marks the manifest as +// scanned to prevent immediate retry loops. +func (s *IndexerStore) SetIndexPartial(ctx context.Context, ir *claircore.IndexReport, scnrs indexer.VersionedScanners) error { + return s.SetIndexFinished(ctx, ir, scnrs) +} + +func (s *IndexerStore) RequeueIndexPartials(ctx context.Context, minAge time.Duration, limit int) (int64, error) { + if limit < 1 { + limit = 1 + } + const query = ` +WITH partial_manifests AS ( + SELECT + ir.manifest_id + FROM + indexreport ir + WHERE + ir.state = 'IndexPartial' + AND ir.updated_at <= now() - $1::interval + ORDER BY + ir.updated_at ASC, + ir.manifest_id ASC + LIMIT $2 +), +partial_layers AS ( + SELECT DISTINCT + ml.layer_id + FROM + manifest_layer ml + JOIN partial_manifests pm ON pm.manifest_id = ml.manifest_id + WHERE + NOT EXISTS ( + SELECT + 1 + FROM + manifest_layer ml2 + LEFT JOIN indexreport ir2 ON ir2.manifest_id = ml2.manifest_id + WHERE + ml2.layer_id = ml.layer_id + AND ml2.manifest_id NOT IN ( + SELECT + manifest_id + FROM + partial_manifests + ) + AND COALESCE(ir2.state, '') <> 'IndexPartial' + ) +), +deleted_manifest_scans AS ( + DELETE FROM scanned_manifest sm + USING partial_manifests pm + WHERE sm.manifest_id = pm.manifest_id + RETURNING sm.manifest_id +), +deleted_layer_scans AS ( + DELETE FROM scanned_layer sl + USING partial_layers pl + WHERE sl.layer_id = pl.layer_id + RETURNING sl.layer_id +) +UPDATE indexreport ir +SET + updated_at = now() +FROM partial_manifests pm +WHERE ir.manifest_id = pm.manifest_id; +` + tag, err := s.pool.Exec(ctx, query, minAge, limit) + if err != nil { + return 0, fmt.Errorf("failed to requeue partial index reports: %w", err) + } + return tag.RowsAffected(), nil +} diff --git a/datastore/postgres/setindexreport.go b/datastore/postgres/setindexreport.go index 2a85be4a3..a14327bcd 100644 --- a/datastore/postgres/setindexreport.go +++ b/datastore/postgres/setindexreport.go @@ -47,16 +47,18 @@ WITH ) INSERT INTO - indexreport (manifest_id, scan_result) + indexreport (manifest_id, state, scan_result, updated_at) VALUES - ((SELECT manifest_id FROM manifests), $2) + ((SELECT manifest_id FROM manifests), $2, $3, now()) ON CONFLICT (manifest_id) DO - UPDATE SET scan_result = excluded.scan_result; + UPDATE SET state = excluded.state, + scan_result = excluded.scan_result, + updated_at = excluded.updated_at; ` start := time.Now() - _, err := s.pool.Exec(ctx, query, ir.Hash, ir) + _, err := s.pool.Exec(ctx, query, ir.Hash, ir.State, ir) if err != nil { return fmt.Errorf("failed to upsert index report: %w", err) } diff --git a/indexer/controller/controller.go b/indexer/controller/controller.go index e8b4aa814..a96534041 100644 --- a/indexer/controller/controller.go +++ b/indexer/controller/controller.go @@ -26,6 +26,8 @@ type Controller struct { report *claircore.IndexReport // a fatal error halting the scanning process err error + // partial records whether the scan completed with degraded scanner data. + partial bool // the current state of the controller currentState State // Realizer is scoped to a single request diff --git a/indexer/controller/indexfinished.go b/indexer/controller/indexfinished.go index d96525f16..b147394ce 100644 --- a/indexer/controller/indexfinished.go +++ b/indexer/controller/indexfinished.go @@ -10,9 +10,20 @@ import ( // indexer to the IndexFinished state the indexer will no longer transition // and return an IndexReport to the caller func indexFinished(ctx context.Context, s *Controller) (State, error) { - s.report.Success = true slog.InfoContext(ctx, "finishing scan") + if s.partial { + s.report.Success = false + s.report.State = IndexPartial.String() + err := s.Store.SetIndexPartial(ctx, s.report, s.Vscnrs) + if err != nil { + return Terminal, fmt.Errorf("failed finish partial scan: %w", err) + } + slog.InfoContext(ctx, "manifest partially scanned") + return Terminal, nil + } + + s.report.Success = true err := s.Store.SetIndexFinished(ctx, s.report, s.Vscnrs) if err != nil { return Terminal, fmt.Errorf("failed finish scan: %w", err) diff --git a/indexer/controller/scanlayers.go b/indexer/controller/scanlayers.go index 21c612635..1b053aabe 100644 --- a/indexer/controller/scanlayers.go +++ b/indexer/controller/scanlayers.go @@ -2,8 +2,11 @@ package controller import ( "context" + "errors" "fmt" "log/slog" + + "github.com/quay/claircore/indexer" ) // scanLayers will run all scanner types against all layers if deemed necessary @@ -13,6 +16,12 @@ func scanLayers(ctx context.Context, c *Controller) (State, error) { defer slog.InfoContext(ctx, "layers scan done") err := c.LayerScanner.Scan(ctx, c.manifest.Hash, c.manifest.Layers) if err != nil { + if errors.Is(err, indexer.ErrScanPartial) { + c.partial = true + c.report.Err = err.Error() + slog.WarnContext(ctx, "layers scan completed with partial results", "reason", err) + return Coalesce, nil + } return Terminal, fmt.Errorf("failed to scan all layer contents: %w", err) } slog.DebugContext(ctx, "layers scan ok") diff --git a/indexer/controller/state.go b/indexer/controller/state.go index 240f1ec8b..c00ccf122 100644 --- a/indexer/controller/state.go +++ b/indexer/controller/state.go @@ -39,6 +39,9 @@ const ( // to the caller of Scan() // Transitions: Terminal IndexFinished + // IndexPartial is a terminal state indicating indexing completed with + // degraded scanner data and should be retried later. + IndexPartial ) func (ss State) String() string { @@ -51,6 +54,7 @@ func (ss State) String() string { "IndexManifest", "IndexError", "IndexFinished", + "IndexPartial", } return names[ss] } @@ -73,6 +77,8 @@ func (ss *State) FromString(state string) { *ss = IndexError case "IndexFinished": *ss = IndexFinished + case "IndexPartial": + *ss = IndexPartial } } diff --git a/indexer/layerscanner.go b/indexer/layerscanner.go index d93de7865..a978c2c33 100644 --- a/indexer/layerscanner.go +++ b/indexer/layerscanner.go @@ -7,6 +7,7 @@ import ( "log/slog" "net" "runtime" + "sync/atomic" "github.com/quay/claircore/toolkit/log" "golang.org/x/sync/errgroup" @@ -116,6 +117,7 @@ func (ls *LayerScanner) Scan(ctx context.Context, manifest claircore.Digest, lay ctx = log.With(ctx, "manifest", manifest) g, ctx := errgroup.WithContext(ctx) + var partial atomic.Bool // Using the goroutine's built-in limit is worst-case the same as using an // external semaphore (spawn N goroutines and immediately wait on M of them, // waits canceling when the first error is returned) but putting the @@ -131,7 +133,7 @@ func (ls *LayerScanner) Scan(ctx context.Context, manifest claircore.Digest, lay return context.Cause(ctx) default: } - if err := ls.scanLayer(ctx, l, s); err != nil { + if err := ls.scanLayer(ctx, l, s, &partial); err != nil { return fmt.Errorf("layer %q: %w", l.Hash, err) } return nil @@ -163,12 +165,18 @@ Layers: } } - return g.Wait() + if err := g.Wait(); err != nil { + return err + } + if partial.Load() { + return ErrScanPartial + } + return nil } // ScanLayer (along with the result type) handles an individual (scanner, layer) // pair. -func (ls *LayerScanner) scanLayer(ctx context.Context, l *claircore.Layer, s VersionedScanner) error { +func (ls *LayerScanner) scanLayer(ctx context.Context, l *claircore.Layer, s VersionedScanner, partial *atomic.Bool) error { ctx = log.With(ctx, "scanner", s.Name(), "kind", s.Kind(), "layer", l.Hash) slog.DebugContext(ctx, "scan start") defer slog.DebugContext(ctx, "scan done") @@ -184,7 +192,12 @@ func (ls *LayerScanner) scanLayer(ctx context.Context, l *claircore.Layer, s Ver var result result if err := result.Do(ctx, s, l); err != nil { - return err + if errors.Is(err, ErrScanPartial) { + partial.Store(true) + slog.WarnContext(ctx, "scanner returned partial results", "reason", err) + } else { + return err + } } if err = result.Store(ctx, ls.store, s, l); err != nil { @@ -232,6 +245,8 @@ func (r *result) Do(ctx context.Context, s VersionedScanner, l *claircore.Layer) var addrErr *net.AddrError switch { case errors.Is(err, nil): + case errors.Is(err, ErrScanPartial): + slog.WarnContext(ctx, "scanner produced degraded results", "scanner", s.Name(), "reason", err) case errors.As(err, &addrErr): slog.WarnContext(ctx, "scanner not able to access resources", "scanner", s.Name(), "reason", err) return nil diff --git a/indexer/partial.go b/indexer/partial.go new file mode 100644 index 000000000..decdd6781 --- /dev/null +++ b/indexer/partial.go @@ -0,0 +1,41 @@ +package indexer + +import ( + "errors" + "fmt" +) + +// ErrScanPartial indicates a scanner returned degraded but usable results. +var ErrScanPartial = errors.New("partial scan") + +// PartialError wraps an error that should mark the containing index report as +// partial without failing the entire index operation. +type PartialError struct { + Err error +} + +func (e *PartialError) Error() string { + if e == nil || e.Err == nil { + return ErrScanPartial.Error() + } + return fmt.Sprintf("%s: %v", ErrScanPartial, e.Err) +} + +func (e *PartialError) Unwrap() error { + if e == nil { + return nil + } + return e.Err +} + +func (e *PartialError) Is(target error) bool { + return target == ErrScanPartial +} + +// Partial wraps err as a partial scanner error. +func Partial(err error) error { + if err == nil { + return nil + } + return &PartialError{Err: err} +} diff --git a/indexer/store.go b/indexer/store.go index 6c7c4a5c8..c9b654180 100644 --- a/indexer/store.go +++ b/indexer/store.go @@ -2,6 +2,7 @@ package indexer import ( "context" + "time" "github.com/quay/claircore" ) @@ -48,6 +49,17 @@ type Setter interface { // Also a call to Querier.IndexReport with the manifest hash represted in the provided IndexReport must return the IndexReport // in finished state. SetIndexFinished(ctx context.Context, sr *claircore.IndexReport, scnrs VersionedScanners) error + // SetIndexPartial marks a scan completed with degraded data. + // + // After this method returns ManifestScanned must return true, so callers do + // not immediately retry the same manifest. Implementations should preserve + // enough state for RequeueIndexPartials to make the manifest eligible for a + // later retry. + SetIndexPartial(ctx context.Context, sr *claircore.IndexReport, scnrs VersionedScanners) error + // RequeueIndexPartials makes up to limit IndexPartial manifests eligible for + // re-indexing when their persisted report has not been updated within + // minAge. + RequeueIndexPartials(ctx context.Context, minAge time.Duration, limit int) (int64, error) } // Querier interface provides the method set to ascertain indexed artifacts and query whether a layer diff --git a/libindex/libindex.go b/libindex/libindex.go index 75f93d83c..646aa9606 100644 --- a/libindex/libindex.go +++ b/libindex/libindex.go @@ -63,6 +63,8 @@ type Libindex struct { // indexerOptions hold construction context for the layerScanner and the // controller factory. indexerOptions *indexer.Options + // partialRetryCancel stops the background IndexPartial retry scheduler. + partialRetryCancel context.CancelFunc } // New creates a new instance of libindex. @@ -88,6 +90,12 @@ func New(ctx context.Context, opts *Options, cl *http.Client) (*Libindex, error) if opts.LayerScanConcurrency == 0 { opts.LayerScanConcurrency = DefaultLayerScanConcurrency } + if opts.IndexPartialRetryInterval == 0 { + opts.IndexPartialRetryInterval = DefaultIndexPartialRetryInterval + } + if opts.IndexPartialRetryLimit == 0 { + opts.IndexPartialRetryLimit = DefaultIndexPartialRetryLimit + } if opts.ControllerFactory == nil { opts.ControllerFactory = controller.New } @@ -159,17 +167,49 @@ func New(ctx context.Context, opts *Options, cl *http.Client) (*Libindex, error) return nil, err } + if opts.IndexPartialRetryInterval > 0 { + l.startPartialRetryScheduler(opts.IndexPartialRetryInterval, opts.IndexPartialRetryLimit) + } + return l, nil } // Close releases held resources. func (l *Libindex) Close(ctx context.Context) error { + if l.partialRetryCancel != nil { + l.partialRetryCancel() + } l.locker.Close(ctx) l.store.Close(ctx) l.fa.Close(ctx) return nil } +func (l *Libindex) startPartialRetryScheduler(interval time.Duration, limit int) { + ctx := log.With(context.Background(), "component", "index-partial-retry") + ctx, cancel := context.WithCancel(ctx) + l.partialRetryCancel = cancel + go func() { + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + n, err := l.store.RequeueIndexPartials(ctx, interval, limit) + if err != nil { + slog.WarnContext(ctx, "failed to requeue partial index reports", "reason", err) + continue + } + if n > 0 { + slog.InfoContext(ctx, "requeued partial index reports", "count", n) + } + } + } + }() +} + // Index performs a scan and index of each layer within the provided Manifest. // // If the index operation cannot start an error will be returned. diff --git a/libindex/options.go b/libindex/options.go index d1c97f7bb..d44428093 100644 --- a/libindex/options.go +++ b/libindex/options.go @@ -13,6 +13,12 @@ const ( // DefaultLayerScanConcurrency is the default number of concurrent layer // "scans". DefaultLayerScanConcurrency = 10 + // DefaultIndexPartialRetryInterval is the default interval between attempts + // to make partial index reports eligible for re-indexing. + DefaultIndexPartialRetryInterval = 24 * time.Hour + // DefaultIndexPartialRetryLimit is the default maximum number of partial + // index reports to requeue per retry scheduler tick. + DefaultIndexPartialRetryLimit = 100 ) // Options are dependencies and options for constructing an instance of libindex @@ -33,6 +39,12 @@ type Options struct { LayerScanConcurrency int // LayerFetchOpt is unused and kept here for backwards compatibility. LayerFetchOpt any + // IndexPartialRetryInterval specifies how often IndexPartial reports should + // be made eligible for re-indexing. A negative value disables the scheduler. + IndexPartialRetryInterval time.Duration + // IndexPartialRetryLimit specifies the maximum number of IndexPartial + // reports made eligible for re-indexing on each scheduler tick. + IndexPartialRetryLimit int // NoLayerValidation controls whether layers are checked to actually be // content-addressed. With this option toggled off, callers can trigger // layers to be indexed repeatedly by changing the identifier in the diff --git a/rhel/repositoryscanner.go b/rhel/repositoryscanner.go index b4e463c8d..6253faabf 100644 --- a/rhel/repositoryscanner.go +++ b/rhel/repositoryscanner.go @@ -172,11 +172,14 @@ func (r *RepositoryScanner) Scan(ctx context.Context, l *claircore.Layer) ([]*cl defer done() cmi, err := r.upd.Get(tctx, r.client) if err != nil && cmi == nil { - return []*claircore.Repository{}, err + slog.WarnContext(ctx, "rhel: unable to fetch mapping file, skipping repository enrichment", "error", err) + return []*claircore.Repository{}, indexer.Partial(err) } cm, ok := cmi.(*mappingFile) if !ok || cm == nil { - return []*claircore.Repository{}, fmt.Errorf("rhel: unable to create a mappingFile object") + err := fmt.Errorf("rhel: unable to create a mappingFile object") + slog.WarnContext(ctx, "rhel: unable to create mappingFile object, skipping repository enrichment") + return []*claircore.Repository{}, indexer.Partial(err) } var repoids []string diff --git a/rhel/repositoryscanner_test.go b/rhel/repositoryscanner_test.go index ce390145a..e148e780f 100644 --- a/rhel/repositoryscanner_test.go +++ b/rhel/repositoryscanner_test.go @@ -2,6 +2,7 @@ package rhel import ( "bytes" + "context" "encoding/json" "errors" "io" @@ -11,8 +12,11 @@ import ( "sort" "strings" "testing" + "time" + "github.com/quay/claircore/indexer" "github.com/quay/claircore/pkg/tmp" + "github.com/quay/claircore/rhel/internal/common" "github.com/google/go-cmp/cmp" @@ -366,6 +370,51 @@ func TestRepositoryScanner(t *testing.T) { } } +func TestRepositoryScannerPartialMappingFileFailure(t *testing.T) { + t.Parallel() + + table := []struct { + name string + scanner *RepositoryScanner + client *http.Client + }{ + { + name: "fetch error with no cached mapping", + scanner: &RepositoryScanner{ + upd: common.NewUpdater("://bad-url", (*mappingFile)(nil)), + cfg: RepositoryScannerConfig{Timeout: time.Second}, + }, + client: http.DefaultClient, + }, + { + name: "unexpected mapping object", + scanner: &RepositoryScanner{ + upd: common.NewUpdater("", (*struct{})(nil)), + cfg: RepositoryScannerConfig{Timeout: time.Second}, + }, + client: http.DefaultClient, + }, + } + + for _, tt := range table { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx := test.Logging(t) + layer := repositoryScannerTestLayer(t, ctx, "testdata/layer-with-embedded-cs.tar") + tt.scanner.client = tt.client + + got, err := tt.scanner.Scan(ctx, layer) + if !errors.Is(err, indexer.ErrScanPartial) { + t.Fatalf("Scan error = %v, want %v", err, indexer.ErrScanPartial) + } + if len(got) != 0 { + t.Fatalf("Scan returned %d repositories, want 0", len(got)) + } + }) + } +} + func TestLabelError(t *testing.T) { err := missingLabel("test") t.Log(err) @@ -392,3 +441,34 @@ func TestBugURL(t *testing.T) { t.Fail() } } + +func repositoryScannerTestLayer(t *testing.T, ctx context.Context, layerPath string) *claircore.Layer { + t.Helper() + + f, err := os.Open(layerPath) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := f.Close(); err != nil { + t.Error(err) + } + }) + + var l claircore.Layer + desc := claircore.LayerDescription{ + Digest: `sha256:` + strings.Repeat(`beef`, 16), + URI: `file:///dev/null`, + MediaType: test.MediaType, + Headers: make(map[string][]string), + } + if err := l.Init(ctx, &desc, f); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := l.Close(); err != nil { + t.Error(err) + } + }) + return &l +} diff --git a/rhel/rhcc/scanner.go b/rhel/rhcc/scanner.go index 916cf0f70..c8b487110 100644 --- a/rhel/rhcc/scanner.go +++ b/rhel/rhcc/scanner.go @@ -179,11 +179,14 @@ func (s *scanner) Scan(ctx context.Context, l *claircore.Layer) ([]*claircore.Pa defer done() vi, err := s.upd.Get(tctx, s.client) if err != nil && vi == nil { - return nil, err + slog.WarnContext(ctx, "rhcc: unable to fetch mapping file, skipping RHCC mapping enrichment", "error", err) + return pkgs, indexer.Partial(err) } v, ok := vi.(*mappingFile) if !ok || v == nil { - return nil, fmt.Errorf("rhcc: unable to create a mappingFile object") + err := fmt.Errorf("rhcc: unable to create a mappingFile object") + slog.WarnContext(ctx, "rhcc: unable to create mappingFile object, skipping RHCC mapping enrichment") + return pkgs, indexer.Partial(err) } repos, ok := v.Data[name] if ok { diff --git a/test/mock/indexer/mocks.go b/test/mock/indexer/mocks.go index 074448df0..73109fa2c 100644 --- a/test/mock/indexer/mocks.go +++ b/test/mock/indexer/mocks.go @@ -12,6 +12,7 @@ package mock_indexer import ( context "context" reflect "reflect" + time "time" claircore "github.com/quay/claircore" indexer "github.com/quay/claircore/indexer" @@ -295,6 +296,21 @@ func (mr *MockStoreMockRecorder) RepositoriesByLayer(ctx, hash, scnrs any) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RepositoriesByLayer", reflect.TypeOf((*MockStore)(nil).RepositoriesByLayer), ctx, hash, scnrs) } +// RequeueIndexPartials mocks base method. +func (m *MockStore) RequeueIndexPartials(ctx context.Context, minAge time.Duration, limit int) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RequeueIndexPartials", ctx, minAge, limit) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RequeueIndexPartials indicates an expected call of RequeueIndexPartials. +func (mr *MockStoreMockRecorder) RequeueIndexPartials(ctx, minAge, limit any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RequeueIndexPartials", reflect.TypeOf((*MockStore)(nil).RequeueIndexPartials), ctx, minAge, limit) +} + // SetIndexFinished mocks base method. func (m *MockStore) SetIndexFinished(ctx context.Context, sr *claircore.IndexReport, scnrs indexer.VersionedScanners) error { m.ctrl.T.Helper() @@ -309,6 +325,20 @@ func (mr *MockStoreMockRecorder) SetIndexFinished(ctx, sr, scnrs any) *gomock.Ca return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetIndexFinished", reflect.TypeOf((*MockStore)(nil).SetIndexFinished), ctx, sr, scnrs) } +// SetIndexPartial mocks base method. +func (m *MockStore) SetIndexPartial(ctx context.Context, sr *claircore.IndexReport, scnrs indexer.VersionedScanners) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetIndexPartial", ctx, sr, scnrs) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetIndexPartial indicates an expected call of SetIndexPartial. +func (mr *MockStoreMockRecorder) SetIndexPartial(ctx, sr, scnrs any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetIndexPartial", reflect.TypeOf((*MockStore)(nil).SetIndexPartial), ctx, sr, scnrs) +} + // SetIndexReport mocks base method. func (m *MockStore) SetIndexReport(arg0 context.Context, arg1 *claircore.IndexReport) error { m.ctrl.T.Helper()