diff --git a/.changeset/fetch-all-pages-cursor-forwarding.md b/.changeset/fetch-all-pages-cursor-forwarding.md new file mode 100644 index 000000000..961e59803 --- /dev/null +++ b/.changeset/fetch-all-pages-cursor-forwarding.md @@ -0,0 +1,9 @@ +--- +'@openchoreo/backstage-plugin-catalog-backend-module': patch +--- + +Fix three `fetchAllPages` call sites in the scheduled entity provider that +ignored the pagination cursor (workflowplanes, observabilityplanes and +deploymentpipelines). Each closure now forwards the cursor and requests +`limit: 100`, so namespaces with more than one page of these resources are +fully ingested instead of silently stopping after the first page. diff --git a/.changeset/fetch-all-pages-hardening.md b/.changeset/fetch-all-pages-hardening.md new file mode 100644 index 000000000..7d9d4d996 --- /dev/null +++ b/.changeset/fetch-all-pages-hardening.md @@ -0,0 +1,18 @@ +--- +'@openchoreo/openchoreo-client-node': minor +--- + +Harden `fetchAllPages` with an optional options bag: `maxPages` caps how +many pages are fetched (throwing instead of silently truncating, and kept +opt-in with no default so existing callers see no new failure modes), +`timeoutMs` gives the whole run a wall-clock budget (chosen as a finite +60s default so unbounded pagination cannot hang a backend, with `0` as +the escape hatch that disables it), and `signal` lets callers abort the +run at entry and between pages. The helper now also detects stuck +cursors (a page returning the same non-empty cursor it was fetched with) +and malformed page responses (a nullish page or a missing `items` +array), throwing descriptive errors that name the page index, cursor, +and collected item count. `PaginatedResponse` and the new +`FetchAllPagesOptions` type are now exported. Behavior is unchanged for +callers that pass only `fetchPage`, apart from the new default timeout +kicking in. diff --git a/.changeset/incremental-ingestion-package.md b/.changeset/incremental-ingestion-package.md new file mode 100644 index 000000000..e31266188 --- /dev/null +++ b/.changeset/incremental-ingestion-package.md @@ -0,0 +1,14 @@ +--- +'@openchoreo/backstage-plugin-catalog-backend-module-openchoreo-incremental': minor +--- + +Add `@openchoreo/backstage-plugin-catalog-backend-module-openchoreo-incremental` +— a new package providing burst-based, cursor-resumable incremental catalog +ingestion on the namespace model (namespaces → projects → components). The +module is config-gated off by default via +`openchoreo.features.incrementalIngestion.enabled`; when the flag is true the +scheduled full-sync `OpenChoreoEntityProvider` in the sibling catalog module +stands down so entities are not double-ingested. Ingestion state persists +through knex migrations on the plugin database (SQLite in dev, Postgres in +prod), applied lazily on first enable. The package depends on the hardened +`fetchAllPages`-era `@openchoreo/openchoreo-client-node` client (PR-A). diff --git a/app-config.local.yaml.example b/app-config.local.yaml.example index a61dd07e2..c02dbd109 100644 --- a/app-config.local.yaml.example +++ b/app-config.local.yaml.example @@ -77,6 +77,17 @@ openchoreo: enabled: true # Set to false to hide Secrets settings tab assistant: enabled: true # Opt-in. Set to true after deploying perch-agent and OPENCHOREO_PERCH_AGENT_URL + incrementalIngestion: + enabled: false # Opt-in; replaces the scheduled full-sync provider when true + + # Incremental ingestion tuning (optional). Only read when + # features.incrementalIngestion.enabled is true; uncomment to override + # the built-in defaults shown here. + # incremental: + # burstLength: 10 # seconds per ingestion burst (default: 10) + # burstInterval: 30 # seconds between bursts (default: 30) + # chunkSize: 100 # entities per page; the API caps page size at 100 (default: 100) + # restLength: 30 # minutes of rest after a completed ingestion (default: 30) # Thunder IDP configuration (k3d cluster) thunder: diff --git a/app-config.production.yaml b/app-config.production.yaml index 8e5e5cd98..01c30e41b 100644 --- a/app-config.production.yaml +++ b/app-config.production.yaml @@ -219,6 +219,14 @@ openchoreo: events: enabled: ${OPENCHOREO_EVENTS_ENABLED} + # Incremental ingestion tuning. Only read when + # openchoreo.features.incrementalIngestion.enabled is true. + incremental: + burstLength: 16 # seconds per ingestion burst + burstInterval: 8 # seconds between bursts + chunkSize: 100 # entities per page; the API caps page size at 100 + restLength: 60 # minutes of rest after a completed ingestion + # Observability tuning. # Hard cap (seconds) on a single wirelogs SSE stream before the backend ends # it; the UI shows soft warnings at ~1/3 and ~2/3 of this value and a toast @@ -254,6 +262,11 @@ openchoreo: # LLM API key Secret to be deployed. assistant: enabled: ${OPENCHOREO_FEATURES_ASSISTANT_ENABLED} + # Incremental catalog ingestion - burst-based, cursor-resumable ingestion. + # When enabled, replaces the scheduled full-sync entity provider. + # Environment variable: OPENCHOREO_FEATURES_INCREMENTAL_INGESTION_ENABLED + incrementalIngestion: + enabled: ${OPENCHOREO_FEATURES_INCREMENTAL_INGESTION_ENABLED} # Secret Management configuration # When disabled, hides the Secrets settings tab and disables secret APIs secretManagement: diff --git a/app-config.yaml b/app-config.yaml index 043f41d46..2840eab21 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -195,6 +195,14 @@ openchoreo: frequency: 300 # seconds between runs (default: 300, the periodic full sync acts as a safety net behind the event-forwarder's real-time deltas) timeout: 120 # seconds for timeout (default: 120) + # Incremental ingestion tuning. Only read when + # openchoreo.features.incrementalIngestion.enabled is true. + incremental: + burstLength: 16 # seconds per ingestion burst + burstInterval: 8 # seconds between bursts + chunkSize: 100 # entities per page; the API caps page size at 100 + restLength: 60 # minutes of rest after a completed ingestion + # Observability tuning (optional). # Hard cap (seconds) on a single wirelogs SSE stream before the backend ends # it; the UI shows soft warnings at ~1/3 and ~2/3 of this value and a toast @@ -235,6 +243,11 @@ openchoreo: # FailedBuildSnackbar, and the build / logs / component-create launchers. assistant: enabled: ${OPENCHOREO_FEATURES_ASSISTANT_ENABLED} + # Incremental catalog ingestion - burst-based, cursor-resumable ingestion. + # When enabled, replaces the scheduled full-sync entity provider. + # Environment variable: OPENCHOREO_FEATURES_INCREMENTAL_INGESTION_ENABLED + incrementalIngestion: + enabled: ${OPENCHOREO_FEATURES_INCREMENTAL_INGESTION_ENABLED} # Component Type Mappings (optional - defaults provided) # Maps OpenChoreo component types to Backstage page variants diff --git a/packages/openchoreo-client-node/src/index.ts b/packages/openchoreo-client-node/src/index.ts index d893ad949..822976522 100644 --- a/packages/openchoreo-client-node/src/index.ts +++ b/packages/openchoreo-client-node/src/index.ts @@ -52,7 +52,11 @@ export { } from './resource-utils'; // Export pagination utilities (new API) -export { fetchAllPages } from './pagination-utils'; +export { + fetchAllPages, + type FetchAllPagesOptions, + type PaginatedResponse, +} from './pagination-utils'; // Export generated types as namespaces export * as OpenChoreoAPI from './generated/openchoreo'; diff --git a/packages/openchoreo-client-node/src/pagination-utils.test.ts b/packages/openchoreo-client-node/src/pagination-utils.test.ts index bbc89ca1a..66732a1e5 100644 --- a/packages/openchoreo-client-node/src/pagination-utils.test.ts +++ b/packages/openchoreo-client-node/src/pagination-utils.test.ts @@ -1,3 +1,4 @@ +import { getEventListeners } from 'events'; import { fetchAllPages } from './pagination-utils'; describe('fetchAllPages', () => { @@ -99,3 +100,255 @@ describe('fetchAllPages', () => { expect(fetchPage).toHaveBeenCalledTimes(2); }); }); + +async function captureError(run: () => Promise): Promise { + try { + await run(); + } catch (error) { + return error as Error; + } + throw new Error('Expected the operation to reject, but it resolved'); +} + +describe('fetchAllPages hardening', () => { + it('throws when fetchPage resolves undefined, naming the page index', async () => { + const fetchPage = jest.fn().mockResolvedValue(undefined); + + const error = await captureError(() => fetchAllPages(fetchPage)); + + expect(error).toBeInstanceOf(Error); + expect(error.message).toContain('page 0'); + expect(error.message).toContain('undefined'); + }); + + it('throws when a page has no items array', async () => { + const fetchPage = jest.fn().mockResolvedValue({}); + + const error = await captureError(() => fetchAllPages(fetchPage)); + + expect(error).toBeInstanceOf(Error); + expect(error.message).toContain('page 0'); + expect(error.message).toContain('items'); + }); + + it('throws when the next cursor repeats the cursor used to fetch the page', async () => { + const fetchPage = jest + .fn() + .mockResolvedValueOnce({ + items: [{ id: 1 }], + pagination: { nextCursor: 'cursor-1' }, + }) + .mockResolvedValueOnce({ + items: [{ id: 2 }], + pagination: { nextCursor: 'cursor-1' }, + }); + + const error = await captureError(() => fetchAllPages(fetchPage)); + + expect(error).toBeInstanceOf(Error); + expect(error.message).toContain('stuck'); + // The error must name the cursor value that is not advancing. + expect(error.message).toContain('"cursor-1"'); + expect(fetchPage).toHaveBeenCalledTimes(2); + }); + + it('stops when nextCursor is null', async () => { + const fetchPage = jest + .fn() + .mockResolvedValueOnce({ + items: [{ id: 1 }], + pagination: { nextCursor: 'cursor-1' }, + }) + .mockResolvedValueOnce({ + items: [{ id: 2 }], + pagination: { nextCursor: null }, + }); + + const result = await fetchAllPages(fetchPage); + + expect(result).toEqual([{ id: 1 }, { id: 2 }]); + expect(fetchPage).toHaveBeenCalledTimes(2); + }); + + it('stops when nextCursor is an empty string', async () => { + const fetchPage = jest + .fn() + .mockResolvedValueOnce({ + items: [{ id: 1 }], + pagination: { nextCursor: 'cursor-1' }, + }) + .mockResolvedValueOnce({ + items: [{ id: 2 }], + pagination: { nextCursor: '' }, + }); + + const result = await fetchAllPages(fetchPage); + + expect(result).toEqual([{ id: 1 }, { id: 2 }]); + expect(fetchPage).toHaveBeenCalledTimes(2); + }); + + it('throws when fetching more pages than maxPages allows', async () => { + const fetchPage = jest + .fn() + .mockResolvedValueOnce({ + items: [{ id: 1 }, { id: 2 }], + pagination: { nextCursor: 'cursor-1' }, + }) + .mockResolvedValueOnce({ + items: [{ id: 3 }, { id: 4 }], + pagination: { nextCursor: 'cursor-2' }, + }) + .mockResolvedValueOnce({ + items: [{ id: 5 }, { id: 6 }], + pagination: {}, + }); + + const error = await captureError(() => + fetchAllPages(fetchPage, { maxPages: 2 }), + ); + + expect(error).toBeInstanceOf(Error); + expect(error.message).toContain('maxPages of 2'); + expect(error.message).toContain('6 items'); + expect(fetchPage).toHaveBeenCalledTimes(3); + }); + + it('does not throw when the page count exactly reaches maxPages', async () => { + const fetchPage = jest + .fn() + .mockResolvedValueOnce({ + items: [{ id: 1 }], + pagination: { nextCursor: 'cursor-1' }, + }) + .mockResolvedValueOnce({ + items: [{ id: 2 }], + pagination: {}, + }); + + const result = await fetchAllPages(fetchPage, { maxPages: 2 }); + + expect(result).toEqual([{ id: 1 }, { id: 2 }]); + expect(fetchPage).toHaveBeenCalledTimes(2); + }); +}); + +describe('fetchAllPages timeout', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + if (jest.getTimerCount() !== 0) { + throw new Error(`leaked ${jest.getTimerCount()} fake timer(s)`); + } + jest.useRealTimers(); + }); + + it('rejects when the default overall timeout expires', async () => { + const fetchPage = jest.fn().mockImplementation( + () => + new Promise(resolve => { + setTimeout( + () => resolve({ items: [{ id: 1 }], pagination: {} }), + 120_000, + ); + }), + ); + + const settled = fetchAllPages(fetchPage).catch(error => error); + + await jest.advanceTimersByTimeAsync(60_000); + const error = (await settled) as Error; + + expect(error).toBeInstanceOf(Error); + expect(error.message).toBe('Pagination timed out after 60000 ms'); + + // Let the abandoned fetch settle so no fake timers are left pending. + await jest.advanceTimersByTimeAsync(60_000); + }); + + it('does not time out when timeoutMs is 0', async () => { + const fetchPage = jest.fn().mockImplementation( + () => + new Promise(resolve => { + setTimeout( + () => resolve({ items: [{ id: 1 }], pagination: {} }), + 120_000, + ); + }), + ); + + const pending = fetchAllPages(fetchPage, { timeoutMs: 0 }); + + // Far beyond the 60s default budget; the run must still complete. + await jest.advanceTimersByTimeAsync(120_000); + + await expect(pending).resolves.toEqual([{ id: 1 }]); + }); +}); + +describe('fetchAllPages cancellation', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + if (jest.getTimerCount() !== 0) { + throw new Error(`leaked ${jest.getTimerCount()} fake timer(s)`); + } + jest.useRealTimers(); + }); + + it('rejects without fetching when the signal is already aborted', async () => { + const controller = new AbortController(); + controller.abort(); + const fetchPage = jest.fn(); + + const error = await captureError(() => + fetchAllPages(fetchPage, { signal: controller.signal }), + ); + + expect(error.name).toBe('AbortError'); + expect(error.message).toContain('Pagination aborted'); + expect(fetchPage).not.toHaveBeenCalled(); + }); + + it('rejects when the signal is aborted after the first page', async () => { + const controller = new AbortController(); + const fetchPage = jest + .fn() + .mockResolvedValueOnce({ + items: [{ id: 1 }], + pagination: { nextCursor: 'cursor-1' }, + }) + .mockImplementationOnce(() => { + controller.abort(); + return new Promise(() => {}); + }); + + const error = await captureError(() => + fetchAllPages(fetchPage, { signal: controller.signal }), + ); + + expect(error.name).toBe('AbortError'); + expect(error.message).toContain('Pagination aborted'); + expect(fetchPage).toHaveBeenCalledTimes(2); + }); + + it('removes its timer and abort listener once it completes', async () => { + const controller = new AbortController(); + const fetchPage = jest.fn().mockResolvedValue({ + items: [{ id: 1 }], + pagination: {}, + }); + + const result = await fetchAllPages(fetchPage, { + signal: controller.signal, + }); + + expect(result).toEqual([{ id: 1 }]); + expect(jest.getTimerCount()).toBe(0); + expect(getEventListeners(controller.signal, 'abort')).toHaveLength(0); + }); +}); diff --git a/packages/openchoreo-client-node/src/pagination-utils.ts b/packages/openchoreo-client-node/src/pagination-utils.ts index f5b5bf82d..7c95e969b 100644 --- a/packages/openchoreo-client-node/src/pagination-utils.ts +++ b/packages/openchoreo-client-node/src/pagination-utils.ts @@ -4,17 +4,53 @@ * @packageDocumentation */ -interface PaginatedResponse { +/** + * A single page of items from a cursor-based paginated API endpoint. + */ +export interface PaginatedResponse { items: T[]; pagination?: { nextCursor?: string; }; } +/** + * Optional guards that protect {@link fetchAllPages} against runaway + * pagination. All of them throw rather than silently truncating results. + */ +export interface FetchAllPagesOptions { + /** Hard cap on pages fetched. Exceeding it throws; it never silently truncates. */ + maxPages?: number; + /** Wall-clock budget for the entire run. Defaults to 60_000; 0 disables the timeout. */ + timeoutMs?: number; + /** Caller cancellation, checked at entry and between pages. */ + signal?: AbortSignal; +} + +/** Default wall-clock budget for an entire pagination run, in milliseconds. */ +const DEFAULT_TIMEOUT_MS = 60_000; + +function describeCursor(cursor: string | undefined): string { + return cursor === undefined ? 'undefined' : `"${cursor}"`; +} + +function abortError(message: string): Error { + const error = new Error(message); + error.name = 'AbortError'; + return error; +} + /** * Fetches all pages from a cursor-based paginated API endpoint. * + * A page whose `nextCursor` is `undefined`, `null` or `''` terminates the + * loop. Broken or runaway pagination never silently truncates: a nullish + * page, a page without an `items` array, a cursor that stops advancing, + * more pages than `options.maxPages`, an expired `options.timeoutMs` + * budget, and an aborted `options.signal` all throw. + * * @param fetchPage - Function that fetches a single page given an optional cursor. + * @param options - Optional guards; see {@link FetchAllPagesOptions}. * @returns All items concatenated across every page. * * @example @@ -33,16 +69,94 @@ interface PaginatedResponse { * ``` */ export async function fetchAllPages( - fetchPage: (cursor?: string) => Promise>, + fetchPage: ( + cursor?: string, + ) => Promise | null | undefined>, + options?: FetchAllPagesOptions, ): Promise { + const maxPages = options?.maxPages; + const signal = options?.signal; + const timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS; + + if (signal?.aborted) { + throw abortError('Pagination aborted before the first page was fetched'); + } + const allItems: T[] = []; let cursor: string | undefined; + let pageIndex = 0; + + let timeoutId: ReturnType | undefined; + let timeoutPromise: Promise | undefined; + if (timeoutMs > 0) { + timeoutPromise = new Promise((_, reject) => { + timeoutId = setTimeout( + () => reject(new Error(`Pagination timed out after ${timeoutMs} ms`)), + timeoutMs, + ); + }); + } + + let onAbort: (() => void) | undefined; + let abortPromise: Promise | undefined; + if (signal) { + abortPromise = new Promise((_, reject) => { + onAbort = () => reject(abortError('Pagination aborted')); + signal.addEventListener('abort', onAbort, { once: true }); + }); + } + + try { + do { + const page = await Promise.race([ + fetchPage(cursor), + ...(timeoutPromise ? [timeoutPromise] : []), + ...(abortPromise ? [abortPromise] : []), + ]); + + const cursorDesc = describeCursor(cursor); + + if (page === null || page === undefined) { + const returned = page === null ? 'null' : 'undefined'; + throw new Error( + `Pagination failed: fetchPage returned ${returned} for page ${pageIndex} (cursor: ${cursorDesc})`, + ); + } + + if (!Array.isArray(page.items)) { + throw new Error( + `Pagination failed: page ${pageIndex} did not return an items array (cursor: ${cursorDesc})`, + ); + } + + const nextCursor = page.pagination?.nextCursor; + if (cursor && nextCursor === cursor) { + // nextCursor equals cursor here, so cursorDesc describes both. + throw new Error( + `Pagination is stuck: page ${pageIndex} returned nextCursor ${cursorDesc}, which is the cursor that was already used to fetch it`, + ); + } + + allItems.push(...page.items); + pageIndex += 1; + + if (maxPages !== undefined && pageIndex > maxPages) { + throw new Error( + `Pagination exceeded maxPages of ${maxPages} after ${pageIndex} pages and ${allItems.length} items collected`, + ); + } - do { - const page = await fetchPage(cursor); - allItems.push(...page.items); - cursor = page.pagination?.nextCursor; - } while (cursor); + // A nextCursor of undefined, null or '' all terminate the loop. + cursor = nextCursor; + } while (cursor); - return allItems; + return allItems; + } finally { + if (timeoutId !== undefined) { + clearTimeout(timeoutId); + } + if (signal && onAbort) { + signal.removeEventListener('abort', onAbort); + } + } } diff --git a/packages/portal-backend/package.json b/packages/portal-backend/package.json index ae8f5f627..e5864b8d5 100644 --- a/packages/portal-backend/package.json +++ b/packages/portal-backend/package.json @@ -52,6 +52,7 @@ "@openchoreo/backstage-plugin-auth-backend-module-openchoreo-auth": "workspace:^", "@openchoreo/backstage-plugin-backend": "workspace:^", "@openchoreo/backstage-plugin-catalog-backend-module": "workspace:^", + "@openchoreo/backstage-plugin-catalog-backend-module-openchoreo-incremental": "workspace:^", "@openchoreo/backstage-plugin-catalog-backend-module-openchoreo-users": "workspace:^", "@openchoreo/backstage-plugin-openchoreo-ci-backend": "workspace:^", "@openchoreo/backstage-plugin-openchoreo-observability-backend": "workspace:^", diff --git a/packages/portal-backend/src/features.test.ts b/packages/portal-backend/src/features.test.ts index 210d8203f..be68b86a4 100644 --- a/packages/portal-backend/src/features.test.ts +++ b/packages/portal-backend/src/features.test.ts @@ -28,9 +28,28 @@ describe('portalBackendFeatures', () => { it('composes the full portal plugin set', () => { // If you add or remove a feature in features.ts, update this count — // the composition is this package's contract. - expect(portalFeatureLoaders).toHaveLength(28); + expect(portalFeatureLoaders).toHaveLength(29); for (const load of portalFeatureLoaders) { expect(typeof load).toBe('function'); } }); + + it('registers the incremental ingestion module between the catalog module and openchoreo-backend', () => { + // Ordering contract (see features.ts): the incremental ingestion module + // must load after '@openchoreo/backstage-plugin-catalog-backend-module' + // (index 19) and before '@openchoreo/backstage-plugin-backend' + // (index 21); it sits at index 20, right between the two. Jest's VM + // sandbox cannot invoke dynamic imports (that needs + // --experimental-vm-modules), so the thunks are identified by their + // source text instead. If you add or remove a feature in features.ts, + // update these indexes. + const loaderName = (i: number) => portalFeatureLoaders[i].toString(); + expect(loaderName(19)).toContain( + "'@openchoreo/backstage-plugin-catalog-backend-module'", + ); + expect(loaderName(20)).toContain( + "'@openchoreo/backstage-plugin-catalog-backend-module-openchoreo-incremental'", + ); + expect(loaderName(21)).toContain("'@openchoreo/backstage-plugin-backend'"); + }); }); diff --git a/packages/portal-backend/src/features.ts b/packages/portal-backend/src/features.ts index 011578739..69969384e 100644 --- a/packages/portal-backend/src/features.ts +++ b/packages/portal-backend/src/features.ts @@ -75,6 +75,14 @@ export const portalFeatureLoaders = [ // because openchoreo-backend depends on the AnnotationStore which is initialized // by the catalog module. () => import('@openchoreo/backstage-plugin-catalog-backend-module'), + // Incremental ingestion follows the same ordering rule as above (after + // catalog-backend-module, before openchoreo-backend). Inert unless + // openchoreo.features.incrementalIngestion.enabled is true, in which case + // it replaces the scheduled full-sync entity provider. + () => + import( + '@openchoreo/backstage-plugin-catalog-backend-module-openchoreo-incremental' + ), () => import('@openchoreo/backstage-plugin-backend'), () => import('@openchoreo/backstage-plugin-scaffolder-backend-module'), () => diff --git a/plugins/catalog-backend-module-openchoreo-incremental/.eslintrc.js b/plugins/catalog-backend-module-openchoreo-incremental/.eslintrc.js new file mode 100644 index 000000000..e2a53a6ad --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/catalog-backend-module-openchoreo-incremental/README.md b/plugins/catalog-backend-module-openchoreo-incremental/README.md new file mode 100644 index 000000000..53779bf08 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/README.md @@ -0,0 +1,247 @@ +# OpenChoreo Incremental Provider + +The OpenChoreo Incremental Provider processes entities in small batches using cursor-based pagination with burst and rest cycles, providing optimal memory consumption, scalability, and controlled load for large OpenChoreo installations. + +## Enabling / disabling + +Incremental ingestion is **disabled by default**. The module is inert unless `openchoreo.features.incrementalIngestion.enabled` is `true` (default `false`): it registers no providers, runs no migrations, and exposes no admin routes — the scheduled full-sync `OpenChoreoEntityProvider` from `@openchoreo/backstage-plugin-catalog-backend-module` keeps running as usual. + +Enable it with: + +```yaml +openchoreo: + features: + incrementalIngestion: + enabled: true # or set the env var OPENCHOREO_FEATURES_INCREMENTAL_INGESTION_ENABLED=true +``` + +When enabled, the scheduled full-sync entity provider in the sibling catalog module **stands down** (it logs `scheduled OpenChoreoEntityProvider standing down` and is not registered), so entities are not double-ingested. + +Tuning keys under `openchoreo.incremental` (only read while the feature is enabled): + +| Key | Default | Meaning | +| --------------- | ------- | ------------------------------------------------- | +| `burstLength` | `10` | seconds per ingestion burst | +| `burstInterval` | `30` | seconds between bursts | +| `chunkSize` | `100` | entities per page (the API caps page size at 100) | +| `restLength` | `30` | minutes of rest after a completed ingestion | + +Database migrations on the plugin database (SQLite in dev, Postgres in prod) run lazily the first time the feature is enabled and a provider connects — not at install or startup while disabled. + +## Installation + +### 1. Add Workspace Dependency + +First, add the incremental provider as a workspace dependency: + +```bash +# From your Backstage root directory +yarn add @openchoreo/plugin-catalog-backend-module-openchoreo-incremental +``` + +### 2. Register the Module + +Add the incremental provider module to your backend: + +```typescript +// packages/backend/src/index.ts +backend.add( + import('@openchoreo/plugin-catalog-backend-module-openchoreo-incremental'), +); +``` + +## Configuration + +```yaml +openchoreo: + baseUrl: ${OPENCHOREO_API_URL} + token: ${OPENCHOREO_TOKEN} # optional + defaultOwner: openchoreo-users # optional; defaults to 'openchoreo-users' + incremental: + burstLength: 10 # seconds - duration of each processing burst + burstInterval: 30 # seconds - interval between bursts during active ingestion + restLength: 30 # minutes - rest period after completing full ingestion + chunkSize: 50 # entities per API request (max 100) + rejectRemovalsAbovePercentage: 80 # reject sync if removals exceed this percentage + rejectEmptySourceCollections: false # reject removals from empty collections +``` + +## How It Works + +### Burst-Based Processing + +The provider uses a burst-and-rest cycle to control load: + +1. **Burst Phase**: Processes entities continuously for `burstLength` seconds +2. **Interstitial Phase**: Pauses for `burstInterval` seconds between bursts +3. **Rest Phase**: After completing a full ingestion cycle, rests for `restLength` minutes before starting again + +This approach prevents overwhelming the API server while ensuring regular catalog updates. + +### Cursor-Based Pagination + +The provider traverses OpenChoreo resources in three phases using cursor-based pagination: + +1. **Namespaces Phase**: Fetches all namespaces and builds a namespace queue +2. **Projects Phase**: For each namespace, fetches all projects +3. **Components Phase**: For each namespace, fetches all components (flat, without a project filter) + +Each phase maintains its own API cursor (`namespaceApiCursor`, `projectApiCursor`, `componentApiCursor`) allowing safe resumption after interruptions. The cursor state tracks: + +- Current phase (`namespaces`, `projects`, `components`) +- API pagination cursors for each resource type +- Queue of namespaces to process +- Current position in the queue + +#### Pagination Mechanism + +The provider uses cursor-based pagination with the following characteristics: + +- **Cursors**: Opaque `nextCursor` tokens that mark the position in the result set +- **Limit Parameter**: Controls the number of items per page (1-100, default 100) + +Example response structure: + +```json +{ + "items": [...], + "pagination": { + "nextCursor": "opaque-token", + "remainingCount": 42 + } +} +``` + +`pagination.nextCursor` is absent when there are no more items. + +#### Stuck-Cursor Guard + +If the API ever returns the exact cursor that was just sent (which would loop +the traversal forever), the provider throws an error for that batch. The +ingestion engine then backs off and retries. + +### Requirements + +Your OpenChoreo backend must support cursor-based pagination on the +`/api/v1/namespaces`, `/api/v1/namespaces/{namespaceName}/projects`, and +`/api/v1/namespaces/{namespaceName}/components` list endpoints: + +- `pagination.nextCursor` field in list responses +- Support for `limit` and `cursor` query parameters + +### State Persistence + +All ingestion state is persisted to the database: + +- Cursors are saved after each burst +- Entity references are tracked for staleness detection +- Progress can resume from the last successful checkpoint +- Removed entities are detected by comparing current and previous ingestion snapshots + +## Management API + +The module provides REST API endpoints for monitoring and managing incremental ingestion: + +- `GET /api/catalog/incremental/health` - Health check status for all providers +- `GET /api/catalog/incremental/providers` - List all registered incremental providers +- `GET /api/catalog/incremental/providers/{name}/status` - Get detailed status for a specific provider +- `POST /api/catalog/incremental/providers/{name}/reset` - Reset provider state to start fresh ingestion +- `POST /api/catalog/incremental/providers/{name}/refresh` - Trigger immediate refresh of provider data + +## Database Migrations + +The module includes automatic database migrations to create the necessary tables for state persistence: + +- `openchoreo_incremental_ingestion_state` - Stores cursor state and ingestion metadata +- `openchoreo_incremental_entity_refs` - Tracks entity references for staleness detection + +These migrations run automatically when the module is first loaded. + +## Migration from Legacy Provider + +If you were previously using the basic `catalog-backend-module-openchoreo` provider: + +1. **Remove the old provider**: Remove the basic OpenChoreo provider module from your backend +2. **Add this incremental module**: Register this module as shown in the Installation section +3. **Update configuration**: Add the `incremental` configuration block (or use defaults) +4. **Verify API support**: Ensure your OpenChoreo API supports cursor-based pagination endpoints + +## Extension Points + +The module provides extension points for advanced use cases: + +### Incremental Provider Extension Point + +You can extend the module with custom incremental entity providers: + +```typescript +import { + openchoreoIncrementalProvidersExtensionPoint, + type OpenChoreoIncrementalProviderExtensionPoint, +} from '@openchoreo/plugin-catalog-backend-module-openchoreo-incremental'; + +// In your backend module +export default createBackendModule({ + pluginId: 'catalog', + moduleId: 'custom-incremental-provider', + register(env) { + env.registerInit({ + deps: { + providers: openchoreoIncrementalProvidersExtensionPoint, + }, + async init({ providers }) { + providers.addIncrementalEntityProvider(new CustomIncrementalProvider()); + }, + }); + }, +}); +``` + +### Custom Provider Implementation + +Implement the `IncrementalEntityProvider` interface for custom providers: + +```typescript +import { + IncrementalEntityProvider, + EntityIteratorResult, +} from '@openchoreo/plugin-catalog-backend-module-openchoreo-incremental'; + +class CustomIncrementalProvider + implements IncrementalEntityProvider +{ + getProviderName(): string { + return 'custom-provider'; + } + + async around(burst: (context: MyContext) => Promise): Promise { + // Setup and teardown logic + await burst(context); + } + + async next( + context: MyContext, + cursor?: MyCursor, + ): Promise> { + // Return batch of entities and next cursor + } +} +``` + +## Features + +- **Burst-Based Processing**: Controlled load with configurable burst and rest cycles +- **Three-Phase Traversal**: Systematic ingestion of namespaces → projects → components +- **Cursor-Based Pagination**: Stable API cursors for efficient, resumable pagination +- **Memory Efficient**: Processes entities in small chunks without loading large datasets +- **Scalable**: Handles very large datasets efficiently with constant memory usage +- **Fault Tolerant**: Resumes from last successful checkpoint after interruptions +- **Configurable**: Customizable burst intervals, rest periods, chunk sizes, and retry backoff +- **Error Resilient**: Exponential backoff strategy with configurable retry intervals +- **Staleness Detection**: Automatically removes entities that no longer exist in OpenChoreo +- **Metrics & Observability**: OpenTelemetry metrics for monitoring ingestion progress +- **Event-Driven Updates**: Supports delta updates via Backstage events system +- **Management API**: REST endpoints for monitoring and controlling ingestion processes +- **Database Persistence**: Automatic migrations and state management +- **Extension Points**: Pluggable architecture for custom incremental providers +- **Health Monitoring**: Built-in health checks and provider status reporting diff --git a/plugins/catalog-backend-module-openchoreo-incremental/config.d.ts b/plugins/catalog-backend-module-openchoreo-incremental/config.d.ts new file mode 100644 index 000000000..253e79e90 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/config.d.ts @@ -0,0 +1,31 @@ +export interface Config { + openchoreo?: { + /** + * Incremental catalog ingestion tuning. Only read when + * openchoreo.features.incrementalIngestion.enabled is true. + */ + incremental?: { + /** + * Duration of each ingestion burst in seconds. + * @visibility backend + */ + burstLength?: number; + /** + * Interval between ingestion bursts in seconds. + * @visibility backend + */ + burstInterval?: number; + /** + * Number of entities to fetch per page. The OpenChoreo API caps page + * size at 100 (LimitParam maximum), so values above 100 are clamped. + * @visibility backend + */ + chunkSize?: number; + /** + * Rest period after a completed ingestion, in minutes. + * @visibility backend + */ + restLength?: number; + }; + }; +} diff --git a/plugins/catalog-backend-module-openchoreo-incremental/dev/index.ts b/plugins/catalog-backend-module-openchoreo-incremental/dev/index.ts new file mode 100644 index 000000000..b350fb5c2 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/dev/index.ts @@ -0,0 +1,92 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Development setup for testing the OpenChoreo incremental ingestion plugin. + * Creates a backend with a dummy provider to simulate incremental entity processing. + */ + +import { createBackend } from '@backstage/backend-defaults'; +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { mockServices } from '@backstage/backend-test-utils'; +import { + IncrementalEntityProvider, + openchoreoIncrementalProvidersExtensionPoint, +} from '../src'; + +const dummyProvider = createBackendModule({ + pluginId: 'catalog', + moduleId: 'openchoreo-test-provider', + register(reg) { + reg.registerInit({ + deps: { + logger: coreServices.logger, + providers: openchoreoIncrementalProvidersExtensionPoint, + }, + async init({ logger, providers }) { + const provider: IncrementalEntityProvider = { + getProviderName: () => 'test-provider', + around: burst => burst(0), + next: async (_context, cursor) => { + await new Promise(resolve => setTimeout(resolve, 500)); + if (cursor === undefined || cursor < 3) { + logger.info(`### Returning batch #${cursor}`); + return { done: false, entities: [], cursor: (cursor ?? 0) + 1 }; + } + + logger.info('### Last batch reached, stopping'); + return { done: true }; + }, + }; + + providers.addProvider({ + provider: provider, + options: { + burstInterval: { seconds: 1 }, + burstLength: { seconds: 10 }, + restLength: { seconds: 10 }, + }, + }); + }, + }); + }, +}); + +async function main(): Promise { + const backend = createBackend(); + backend.add( + mockServices.rootConfig.factory({ + data: { + backend: { + baseUrl: 'http://localhost:7007', + listen: ':7007', + database: { client: 'better-sqlite3', connection: ':memory:' }, + }, + }, + }), + ); + // Note: @backstage/plugin-catalog-backend is intentionally not added here; + // it is not a dependency of this module package. + backend.add(import('../src')); + backend.add(dummyProvider); + await backend.start(); +} + +// Entry point for `yarn start` in this package. Not executed on import. +void main(); diff --git a/plugins/catalog-backend-module-openchoreo-incremental/migrations/20221116073152_init.js b/plugins/catalog-backend-module-openchoreo-incremental/migrations/20221116073152_init.js new file mode 100644 index 000000000..6d74f7937 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/migrations/20221116073152_init.js @@ -0,0 +1,188 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Provenance: copied from @backstage/plugin-catalog-backend-module-incremental-ingestion +// (https://github.com/backstage/backstage), Apache-2.0. Kept byte-compatible so +// databases migrated by that module roll forward cleanly. + +// @ts-check + +/** + * Database migration to initialize tables for incremental ingestion. + * Creates ingestions, ingestion_marks, and ingestion_mark_entities tables + * to support resumable, burst-based processing of large entity datasets. + */ + +/** + * @param { import("knex").Knex } knex + */ +exports.up = async function up(knex) { + /** + * Sets up the ingestions table + */ + await knex.schema.createTable('ingestions', table => { + table.comment('Tracks ingestion streams for very large data sets'); + + table + .uuid('id') + .notNullable() + .comment('Auto-generated ID of the ingestion'); + + table + .string('provider_name') + .notNullable() + .comment('each provider gets its own identifiable name'); + + table + .string('status') + .notNullable() + .comment( + 'One of "interstitial" | "bursting" | "backing off" | "resting" | "complete"', + ); + + table + .string('next_action') + .notNullable() + .comment("what will this, 'ingest', 'rest', 'backoff', 'nothing (done)'"); + + table + .timestamp('next_action_at') + .notNullable() + .defaultTo(knex.fn.now()) + .comment('the moment in time at which point ingestion can begin again'); + + table + .string('last_error') + .comment('records any error that occurred in the previous burst attempt'); + + table + .integer('attempts') + .notNullable() + .defaultTo(0) + .comment('how many attempts have been made to burst without success'); + + table + .timestamp('created_at') + .notNullable() + .defaultTo(knex.fn.now()) + .comment('when did this ingestion actually begin'); + + table + .timestamp('ingestion_completed_at') + .comment('when did the ingestion actually end'); + + table + .timestamp('rest_completed_at') + .comment('when did the rest period actually end'); + + table + .string('completion_ticket') + .notNullable() + .comment( + 'indicates whether the ticket is still open or stamped complete', + ); + }); + + await knex.schema.alterTable('ingestions', t => { + t.primary(['id']); + t.index('provider_name', 'ingestion_provider_name_idx'); + t.unique(['provider_name', 'completion_ticket'], { + indexName: 'ingestion_composite_index', + }); + }); + + /** + * Sets up the ingestion_marks table + */ + await knex.schema.createTable('ingestion_marks', table => { + table.comment('tracks each step of an iterative ingestion'); + + table + .uuid('id') + .notNullable() + .comment('Auto-generated ID of the ingestion mark'); + + table + .uuid('ingestion_id') + .notNullable() + .references('id') + .inTable('ingestions') + .onDelete('CASCADE') + .comment('The id of the ingestion in which this mark took place'); + + table + .json('cursor') + .comment( + 'the current data associated with this iteration wherever it is in this moment in time', + ); + + table + .integer('sequence') + .notNullable() + .defaultTo(0) + .comment('what is the order of this mark'); + + table.timestamp('created_at').notNullable().defaultTo(knex.fn.now()); + }); + + await knex.schema.alterTable('ingestion_marks', t => { + t.primary(['id']); + t.index('ingestion_id', 'ingestion_mark_ingestion_id_idx'); + }); + + /** + * Set up the ingestion_mark_entities table + */ + await knex.schema.createTable('ingestion_mark_entities', table => { + table.comment( + 'tracks the entities recorded in each step of an iterative ingestion', + ); + + table + .uuid('id') + .notNullable() + .comment('Auto-generated ID of the marked entity'); + + table + .uuid('ingestion_mark_id') + .notNullable() + .references('id') + .inTable('ingestion_marks') + .onDelete('CASCADE') + .comment( + 'Every time a mark happens during an ingestion, there are a list of entities marked.', + ); + + table + .string('ref') + .notNullable() + .comment('the entity reference of the marked entity'); + }); + + await knex.schema.alterTable('ingestion_mark_entities', t => { + t.primary(['id']); + t.index('ingestion_mark_id', 'ingestion_mark_entity_ingestion_mark_id_idx'); + }); +}; + +/** + * @param { import("knex").Knex } knex + */ +exports.down = async function down(knex) { + await knex.schema.dropTable('ingestion_mark_entities'); + await knex.schema.dropTable('ingestion_marks'); + await knex.schema.dropTable('ingestions'); +}; diff --git a/plugins/catalog-backend-module-openchoreo-incremental/migrations/20240110000001_add_performance_indexes.js b/plugins/catalog-backend-module-openchoreo-incremental/migrations/20240110000001_add_performance_indexes.js new file mode 100644 index 000000000..a99ffc733 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/migrations/20240110000001_add_performance_indexes.js @@ -0,0 +1,184 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Provenance: adapted from the OpenChoreo incremental ingestion module of +// https://github.com/openchoreo/backstage-plugins, Apache-2.0. All statements +// targeting the catalog backend's own tables were dropped: this module only +// manages the `ingestions`, `ingestion_marks`, and +// `ingestion_mark_entities` tables it creates itself. + +// @ts-check + +/** + * Performance optimization migration for OpenChoreo incremental ingestion + * This migration adds database indexes to improve query performance for large datasets + * + * Expected performance improvements: + * - 50-70% faster ingestion time + * - 5-10x faster database queries + * - Reduced memory pressure during large ingestions + */ + +// Disable transactions for this migration due to CREATE INDEX CONCURRENTLY commands +// PostgreSQL CONCURRENTLY operations cannot run inside transaction blocks +exports.config = { transaction: false }; + +/** + * @param { import("knex").Knex } knex + */ +exports.up = async function up(knex) { + const isPostgres = knex.client.config.client === 'pg'; + + if (isPostgres) { + console.log('Applying PostgreSQL performance indexes...'); + + // Create indexes concurrently to avoid blocking production traffic + await knex.raw(` + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_ingestion_mark_entities_ref + ON ingestion_mark_entities(ref); + `); + + await knex.raw(` + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_ingestion_marks_ingestion_id + ON ingestion_marks(ingestion_id); + `); + + await knex.raw(` + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_ingestions_provider_name + ON ingestions(provider_name); + `); + + await knex.raw(` + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_ingestions_completion_ticket + ON ingestions(completion_ticket); + `); + + await knex.raw(` + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_ingestion_mark_entities_composite + ON ingestion_mark_entities(ingestion_mark_id, ref); + `); + + // Update table statistics for query optimizer + await knex.raw('ANALYZE ingestion_mark_entities'); + await knex.raw('ANALYZE ingestion_marks'); + await knex.raw('ANALYZE ingestions'); + + // Create performance monitoring view + await knex.raw(` + CREATE OR REPLACE VIEW ingestion_performance_stats AS + SELECT + i.provider_name, + COUNT(DISTINCT ime.ref) as total_entities, + COUNT(DISTINCT im.id) as total_marks, + MAX(i.created_at) as last_ingestion_start, + MAX(i.ingestion_completed_at) as last_ingestion_complete, + CASE + WHEN i.status = 'resting' THEN 'RESTING' + WHEN i.status = 'bursting' THEN 'ACTIVE' + WHEN i.status = 'backing off' THEN 'ERROR' + ELSE 'UNKNOWN' + END as current_status + FROM ingestions i + LEFT JOIN ingestion_marks im ON i.id = im.ingestion_id + LEFT JOIN ingestion_mark_entities ime ON im.id = ime.ingestion_mark_id + WHERE i.completion_ticket = 'open' + GROUP BY i.provider_name, i.status + `); + + console.log('PostgreSQL performance indexes created successfully'); + } else { + // SQLite for development/testing + console.log('Applying SQLite performance indexes...'); + + await knex.schema.raw(` + CREATE INDEX IF NOT EXISTS idx_ingestion_mark_entities_ref + ON ingestion_mark_entities(ref); + `); + + await knex.schema.raw(` + CREATE INDEX IF NOT EXISTS idx_ingestion_marks_ingestion_id + ON ingestion_marks(ingestion_id); + `); + + await knex.schema.raw(` + CREATE INDEX IF NOT EXISTS idx_ingestions_provider_name + ON ingestions(provider_name); + `); + + await knex.schema.raw(` + CREATE INDEX IF NOT EXISTS idx_ingestions_completion_ticket + ON ingestions(completion_ticket); + `); + + await knex.schema.raw(` + CREATE INDEX IF NOT EXISTS idx_ingestion_mark_entities_composite + ON ingestion_mark_entities(ingestion_mark_id, ref); + `); + + console.log('SQLite performance indexes created successfully'); + } +}; + +/** + * @param { import("knex").Knex } knex + */ +exports.down = async function down(knex) { + const isPostgres = knex.client.config.client === 'pg'; + + if (isPostgres) { + console.log('Removing PostgreSQL performance indexes...'); + + // Drop indexes concurrently + await knex.raw( + 'DROP INDEX CONCURRENTLY IF EXISTS idx_ingestion_mark_entities_ref', + ); + await knex.raw( + 'DROP INDEX CONCURRENTLY IF EXISTS idx_ingestion_marks_ingestion_id', + ); + await knex.raw( + 'DROP INDEX CONCURRENTLY IF EXISTS idx_ingestions_provider_name', + ); + await knex.raw( + 'DROP INDEX CONCURRENTLY IF EXISTS idx_ingestions_completion_ticket', + ); + await knex.raw( + 'DROP INDEX CONCURRENTLY IF EXISTS idx_ingestion_mark_entities_composite', + ); + + // Drop monitoring view + await knex.raw('DROP VIEW IF EXISTS ingestion_performance_stats'); + + console.log('PostgreSQL performance indexes removed'); + } else { + console.log('Removing SQLite performance indexes...'); + + await knex.schema.raw( + 'DROP INDEX IF EXISTS idx_ingestion_mark_entities_ref', + ); + await knex.schema.raw( + 'DROP INDEX IF EXISTS idx_ingestion_marks_ingestion_id', + ); + await knex.schema.raw('DROP INDEX IF EXISTS idx_ingestions_provider_name'); + await knex.schema.raw( + 'DROP INDEX IF EXISTS idx_ingestions_completion_ticket', + ); + await knex.schema.raw( + 'DROP INDEX IF EXISTS idx_ingestion_mark_entities_composite', + ); + + console.log('SQLite performance indexes removed'); + } +}; diff --git a/plugins/catalog-backend-module-openchoreo-incremental/migrations/20240110000003_expand_last_error_field.js b/plugins/catalog-backend-module-openchoreo-incremental/migrations/20240110000003_expand_last_error_field.js new file mode 100644 index 000000000..71fa7867f --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/migrations/20240110000003_expand_last_error_field.js @@ -0,0 +1,44 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * Database migration to expand the last_error field from VARCHAR(255) to TEXT. + * This allows storing full error stack traces and detailed error messages + * without truncation. + */ + +/** + * @param { import("knex").Knex } knex + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('ingestions', table => { + // Change last_error from VARCHAR(255) to TEXT to accommodate long error messages + table.text('last_error').alter(); + }); +}; + +/** + * @param { import("knex").Knex } knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('ingestions', table => { + // Revert back to VARCHAR(255) + // Note: This may truncate existing error messages longer than 255 characters + table.string('last_error', 255).alter(); + }); +}; diff --git a/plugins/catalog-backend-module-openchoreo-incremental/package.json b/plugins/catalog-backend-module-openchoreo-incremental/package.json new file mode 100644 index 000000000..be478d375 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/package.json @@ -0,0 +1,82 @@ +{ + "name": "@openchoreo/backstage-plugin-catalog-backend-module-openchoreo-incremental", + "version": "0.1.0", + "license": "Apache-2.0", + "description": "OpenChoreo incremental ingestion backend module for the Backstage catalog plugin", + "main": "src/index.ts", + "types": "src/index.ts", + "exports": { + ".": "./src/index.ts", + "./package.json": "./package.json" + }, + "typesVersions": { + "*": { + "package.json": [ + "package.json" + ] + } + }, + "repository": { + "type": "git", + "url": "https://github.com/openchoreo/backstage-plugins.git", + "directory": "plugins/catalog-backend-module-openchoreo-incremental" + }, + "publishConfig": { + "access": "restricted", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts", + "registry": "https://npm.pkg.github.com" + }, + "backstage": { + "role": "backend-plugin-module", + "pluginId": "catalog", + "pluginPackage": "@backstage/plugin-catalog-backend", + "features": { + ".": "@backstage/BackendFeature" + } + }, + "scripts": { + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack" + }, + "dependencies": { + "@backstage/backend-plugin-api": "^1.9.1", + "@backstage/catalog-model": "^1.9.0", + "@backstage/config": "^1.3.8", + "@backstage/errors": "^1.3.1", + "@backstage/plugin-catalog-node": "^2.2.1", + "@backstage/plugin-events-node": "^0.4.22", + "@backstage/types": "^1.2.2", + "@openchoreo/backstage-plugin-catalog-backend-module": "workspace:^", + "@openchoreo/backstage-plugin-common": "workspace:^", + "@openchoreo/openchoreo-client-node": "workspace:^", + "@opentelemetry/api": "^1.9.0", + "express": "4.21.2", + "express-promise-router": "^4.1.0", + "knex": "3.1.0", + "luxon": "^3.0.0", + "uuid": "^11.0.0", + "zod": "^4.1.12" + }, + "devDependencies": { + "@backstage/backend-defaults": "^0.17.1", + "@backstage/backend-test-utils": "^1.11.3", + "@backstage/cli": "^0.36.2", + "@types/express": "4.17.23", + "@types/luxon": "^3.4.2", + "@types/supertest": "2.0.16", + "supertest": "6.3.4" + }, + "files": [ + "dist", + "config.d.ts", + "migrations/**/*.{js,d.ts}", + "dev/**/*.{ts,js}" + ], + "configSchema": "config.d.ts" +} diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/config.test.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/config.test.ts new file mode 100644 index 000000000..3a63ce823 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/config.test.ts @@ -0,0 +1,197 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Test suite for the zod configuration schemas. + * Verifies default values, partial inputs, type enforcement, optional keys, + * and the API and top-level validation schemas. + */ +import { ZodError } from 'zod'; +import { + openchoreoApiConfigSchema, + openchoreoIncrementalConfigSchema, + openchoreoIncrementalConfigValidation, +} from './config'; + +describe('openchoreoIncrementalConfigSchema', () => { + it('fills in all defaults for an empty object', () => { + expect(openchoreoIncrementalConfigSchema.parse({})).toEqual({ + burstLength: 10, + burstInterval: 30, + restLength: 30, + chunkSize: 100, + rejectEmptySourceCollections: false, + maxConcurrentRequests: 5, + batchDelayMs: 100, + }); + }); + + it('accepts partial objects and merges them with the defaults', () => { + expect( + openchoreoIncrementalConfigSchema.parse({ + burstLength: 60, + restLength: 5, + }), + ).toEqual({ + burstLength: 60, + burstInterval: 30, + restLength: 5, + chunkSize: 100, + rejectEmptySourceCollections: false, + maxConcurrentRequests: 5, + batchDelayMs: 100, + }); + }); + + it('rejects wrong types for every tuned field', () => { + expect(() => + openchoreoIncrementalConfigSchema.parse({ burstLength: 'ten' }), + ).toThrow(ZodError); + expect(() => + openchoreoIncrementalConfigSchema.parse({ burstInterval: true }), + ).toThrow(ZodError); + expect(() => + openchoreoIncrementalConfigSchema.parse({ restLength: null }), + ).toThrow(ZodError); + expect(() => + openchoreoIncrementalConfigSchema.parse({ chunkSize: '100' }), + ).toThrow(ZodError); + expect(() => + openchoreoIncrementalConfigSchema.parse({ + rejectEmptySourceCollections: 'yes', + }), + ).toThrow(ZodError); + expect(() => + openchoreoIncrementalConfigSchema.parse({ backoff: [10, 'x'] }), + ).toThrow(ZodError); + expect(() => + openchoreoIncrementalConfigSchema.parse({ + maxConcurrentRequests: '5', + }), + ).toThrow(ZodError); + }); + + it('enforces the documented ranges', () => { + expect(() => + openchoreoIncrementalConfigSchema.parse({ chunkSize: 101 }), + ).toThrow(ZodError); + expect(() => + openchoreoIncrementalConfigSchema.parse({ burstInterval: 4 }), + ).toThrow(ZodError); + expect(() => + openchoreoIncrementalConfigSchema.parse({ restLength: 1441 }), + ).toThrow(ZodError); + + // The boundary values themselves are accepted. + expect( + openchoreoIncrementalConfigSchema.parse({ + chunkSize: 100, + burstInterval: 300, + restLength: 1440, + }), + ).toMatchObject({ + chunkSize: 100, + burstInterval: 300, + restLength: 1440, + }); + }); + + it('keeps backoff and rejectRemovalsAbovePercentage optional', () => { + const withoutOptionals = openchoreoIncrementalConfigSchema.parse({}); + expect(withoutOptionals.backoff).toBeUndefined(); + expect(withoutOptionals.rejectRemovalsAbovePercentage).toBeUndefined(); + + const withOptionals = openchoreoIncrementalConfigSchema.parse({ + backoff: [30, 60, 300], + rejectRemovalsAbovePercentage: 25, + }); + expect(withOptionals.backoff).toEqual([30, 60, 300]); + expect(withOptionals.rejectRemovalsAbovePercentage).toBe(25); + + expect(() => + openchoreoIncrementalConfigSchema.parse({ + rejectRemovalsAbovePercentage: 101, + }), + ).toThrow(ZodError); + expect(() => + openchoreoIncrementalConfigSchema.parse({ backoff: [0] }), + ).toThrow(ZodError); + }); +}); + +describe('openchoreoApiConfigSchema', () => { + it('requires a valid base URL and keeps the token optional', () => { + expect(() => openchoreoApiConfigSchema.parse({})).toThrow(ZodError); + + expect( + openchoreoApiConfigSchema.parse({ + baseUrl: 'https://api.openchoreo.example.com', + }), + ).toEqual({ baseUrl: 'https://api.openchoreo.example.com' }); + + expect( + openchoreoApiConfigSchema.parse({ + baseUrl: 'http://localhost:8080', + token: 'secret', + }), + ).toEqual({ baseUrl: 'http://localhost:8080', token: 'secret' }); + + expect(() => + openchoreoApiConfigSchema.parse({ baseUrl: 'not a url' }), + ).toThrow(ZodError); + }); +}); + +describe('openchoreoIncrementalConfigValidation', () => { + it('requires an openchoreo section but accepts it empty', () => { + expect(() => openchoreoIncrementalConfigValidation.parse({})).toThrow( + ZodError, + ); + expect( + openchoreoIncrementalConfigValidation.parse({ openchoreo: {} }), + ).toEqual({ openchoreo: {} }); + }); + + it('validates api and incremental sections together', () => { + expect( + openchoreoIncrementalConfigValidation.parse({ + openchoreo: { + api: { baseUrl: 'https://api.openchoreo.example.com' }, + incremental: { burstLength: 20 }, + }, + }), + ).toEqual({ + openchoreo: { + api: { baseUrl: 'https://api.openchoreo.example.com' }, + incremental: { + burstLength: 20, + burstInterval: 30, + restLength: 30, + chunkSize: 100, + rejectEmptySourceCollections: false, + maxConcurrentRequests: 5, + batchDelayMs: 100, + }, + }, + }); + + expect(() => + openchoreoIncrementalConfigValidation.parse({ + openchoreo: { incremental: { burstLength: 301 } }, + }), + ).toThrow(ZodError); + }); +}); diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/config.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/config.ts new file mode 100644 index 000000000..c55f1fc00 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/config.ts @@ -0,0 +1,167 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { z } from 'zod'; + +/** + * Configuration options for the OpenChoreo API connection. + */ +export const openchoreoApiConfigSchema = z.object({ + /** + * Base URL for the OpenChoreo API. + */ + baseUrl: z.string().url().describe('OpenChoreo API base URL'), + + /** + * Optional authentication token for API access. + */ + token: z.string().optional().describe('OpenChoreo API authentication token'), +}); + +/** + * Configuration options for incremental ingestion behavior. + */ +export const openchoreoIncrementalConfigSchema = z.object({ + /** + * Duration of each ingestion burst in seconds. Must be between 1 and 300. + * @default 10 + */ + burstLength: z + .number() + .min(1) + .max(300) + .default(10) + .describe('Duration of ingestion bursts in seconds'), + + /** + * Interval between ingestion bursts in seconds. Must be between 5 and 300. + * @default 30 + */ + burstInterval: z + .number() + .min(5) + .max(300) + .default(30) + .describe('Interval between ingestion bursts in seconds'), + + /** + * Rest period after successful ingestion in minutes. Must be between 1 and 1440. + * @default 30 + */ + restLength: z + .number() + .min(1) + .max(1440) + .default(30) + .describe('Rest period after ingestion in minutes'), + + /** + * Number of entities to process in each batch. Must be between 1 and 100. + * The OpenChoreo API caps page size at 100 (LimitParam maximum), so values + * above 100 are rejected. + * @default 100 + */ + chunkSize: z + .number() + .min(1) + .max(100) + .default(100) + .describe('Number of entities per batch (max 100, API page size cap)'), + + /** + * Backoff strategy for failed ingestion attempts in seconds. + */ + backoff: z + .array(z.number().positive()) + .optional() + .describe('Backoff durations in seconds'), + + /** + * Percentage threshold above which entity removals will be rejected (0-100). + */ + rejectRemovalsAbovePercentage: z + .number() + .min(0) + .max(100) + .optional() + .describe('Removal rejection threshold percentage'), + + /** + * Whether to reject removals when source collections are empty. + * @default false + */ + rejectEmptySourceCollections: z + .boolean() + .default(false) + .describe('Reject removals from empty collections'), + + /** + * Maximum number of concurrent API requests during batch processing. + * Must be between 1 and 50. + * @default 5 + */ + maxConcurrentRequests: z + .number() + .min(1) + .max(50) + .default(5) + .describe('Maximum concurrent API requests during batch processing'), + + /** + * Delay in milliseconds between batch processing requests. + * Must be between 0 and 10000. + * @default 100 + */ + batchDelayMs: z + .number() + .min(0) + .max(10000) + .default(100) + .describe('Delay in milliseconds between batch processing requests'), +}); + +/** + * Complete configuration schema for OpenChoreo incremental plugin. + */ +export const openchoreoIncrementalConfigValidation = z.object({ + openchoreo: z.object({ + api: openchoreoApiConfigSchema.optional(), + incremental: openchoreoIncrementalConfigSchema.optional(), + }), +}); + +/** + * TypeScript interface for the complete OpenChoreo configuration. + */ +export interface OpenChoreoIncrementalConfig { + openchoreo: { + api?: { + baseUrl: string; + token?: string; + }; + incremental?: { + burstLength: number; + burstInterval: number; + restLength: number; + chunkSize: number; + backoff?: number[]; + rejectRemovalsAbovePercentage?: number; + rejectEmptySourceCollections: boolean; + maxConcurrentRequests: number; + batchDelayMs: number; + }; + }; +} diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/database/OpenChoreoIncrementalIngestionDatabaseManager.test.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/database/OpenChoreoIncrementalIngestionDatabaseManager.test.ts new file mode 100644 index 000000000..1cee0a1b3 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/database/OpenChoreoIncrementalIngestionDatabaseManager.test.ts @@ -0,0 +1,1155 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Test suite for OpenChoreoIncrementalIngestionDatabaseManager. + * Verifies database operations for incremental ingestion, including mark + * storage and retrieval, cascading deletes, error truncation, and migration + * idempotence. Runs against the real migrations directory. + */ +import { + TestDatabases, + mockServices, + type TestDatabaseId, +} from '@backstage/backend-test-utils'; +import type { SchedulerService } from '@backstage/backend-plugin-api'; +import type { Entity } from '@backstage/catalog-model'; +import type { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import type { EventsService } from '@backstage/plugin-events-node'; +import type { Knex } from 'knex'; +import { Duration } from 'luxon'; +import { v4 as uuid } from 'uuid'; +import { WrapperProviders } from '../module/WrapperProviders'; +import type { + IncrementalEntityProvider, + IncrementalEntityProviderOptions, +} from '../types'; +import { OpenChoreoIncrementalIngestionDatabaseManager } from './OpenChoreoIncrementalIngestionDatabaseManager'; +import { applyDatabaseMigrations } from './migrations'; +import { DB_MIGRATIONS_TABLE } from './tables'; + +const migrationsDir = `${__dirname}/../../migrations`; + +jest.setTimeout(60_000); + +const EXPECTED_MIGRATION_NAMES = [ + '20221116073152_init.js', + '20240110000001_add_performance_indexes.js', + '20240110000003_expand_last_error_field.js', +]; + +function makeEntity(name: string): Entity { + return { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { namespace: 'default', name }, + }; +} + +/** Inserts a completed ingestion row through the manager's insert path. */ +async function insertFinishedIngestion( + manager: OpenChoreoIncrementalIngestionDatabaseManager, + provider: string, + options: { restCompletedAt: Date; completionTicket?: string }, +): Promise { + const id = uuid(); + await manager.insertIngestionRecord({ + id, + provider_name: provider, + status: 'complete', + next_action: 'nothing (done)', + completion_ticket: options.completionTicket ?? uuid(), + ingestion_completed_at: options.restCompletedAt, + rest_completed_at: options.restCompletedAt, + }); + return id; +} + +/** Creates a mark and attaches raw entity refs to it in one go. */ +async function addMarkWithRefs( + knex: Knex, + ingestionId: string, + sequence: number, + refs: string[], +): Promise { + const markId = uuid(); + await knex('ingestion_marks').insert({ + id: markId, + ingestion_id: ingestionId, + sequence, + cursor: JSON.stringify({ sequence }), + }); + if (refs.length > 0) { + await knex('ingestion_mark_entities').insert( + refs.map(ref => ({ id: uuid(), ingestion_mark_id: markId, ref })), + ); + } + return markId; +} + +describe('OpenChoreoIncrementalIngestionDatabaseManager', () => { + // SQLITE_3 runs everywhere with no docker dependency; the PostgreSQL + // variants of the migration are exercised in deployment environments. + const databases = TestDatabases.create({ + ids: ['SQLITE_3'], + }); + + it.each(databases.eachSupportedId())( + 'creates and returns the current ingestion record, %p', + async databaseId => { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + + const manager = new OpenChoreoIncrementalIngestionDatabaseManager({ + client: knex, + logger: mockServices.logger.mock(), + }); + + const created = await manager.createProviderIngestionRecord('myProvider'); + expect(created).toBeDefined(); + + const record = await manager.getCurrentIngestionRecord('myProvider'); + expect(record).toBeDefined(); + expect(record!.id).toEqual(created!.ingestionId); + expect(record!.provider_name).toEqual('myProvider'); + expect(record!.status).toEqual('bursting'); + expect(record!.completion_ticket).toEqual('open'); + + // Completing the record closes the ticket: it is no longer current, + // but remains retrievable as the previous record. + await manager.setProviderComplete(created!.ingestionId); + + await expect( + manager.getCurrentIngestionRecord('myProvider'), + ).resolves.toBeUndefined(); + + const previous = await manager.getPreviousIngestionRecord('myProvider'); + expect(previous).toBeDefined(); + expect(previous!.id).toEqual(created!.ingestionId); + expect(previous!.completion_ticket).not.toEqual('open'); + }, + ); + + it.each(databases.eachSupportedId())( + 'cascades mark entity deletion when the ingestion is deleted, %p', + async databaseId => { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + + const manager = new OpenChoreoIncrementalIngestionDatabaseManager({ + client: knex, + logger: mockServices.logger.mock(), + }); + + const { ingestionId } = (await manager.createProviderIngestionRecord( + 'myProvider', + ))!; + + const markId = uuid(); + await manager.createMark({ + record: { + id: markId, + ingestion_id: ingestionId, + sequence: 1, + cursor: { data: 1 }, + }, + }); + await manager.createMarkEntities(markId, [ + { entity: makeEntity('one') }, + { entity: makeEntity('two') }, + ]); + + await expect( + knex('ingestion_mark_entities').where({ ingestion_mark_id: markId }), + ).resolves.toHaveLength(2); + + // Deleting the ingestion row must cascade through marks to entities. + await knex('ingestions').where({ id: ingestionId }).delete(); + + await expect( + knex('ingestion_marks').where({ ingestion_id: ingestionId }), + ).resolves.toHaveLength(0); + await expect( + knex('ingestion_mark_entities').where({ ingestion_mark_id: markId }), + ).resolves.toHaveLength(0); + }, + ); + + it.each(databases.eachSupportedId())( + 'stores long last_error values on the expanded column and truncates at the storage limit, %p', + async databaseId => { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + + const manager = new OpenChoreoIncrementalIngestionDatabaseManager({ + client: knex, + logger: mockServices.logger.mock(), + }); + + const { ingestionId } = (await manager.createProviderIngestionRecord( + 'myProvider', + ))!; + + // The expand migration turned last_error into a TEXT column, so a long + // raw write round-trips without loss. + await knex('ingestions') + .where({ id: ingestionId }) + .update({ last_error: 'x'.repeat(10_000) }); + const rawRow = await knex('ingestions') + .where({ id: ingestionId }) + .first(); + expect(rawRow.last_error.length).toEqual(10_000); + + // The manager itself truncates error text at its 2000-char limit. + await manager.setProviderBackoff( + ingestionId, + 0, + new Error('y'.repeat(5_000)), + 1_000, + ); + + const row = await knex('ingestions').where({ id: ingestionId }).first(); + expect(row.last_error.length).toBeLessThanOrEqual(2_000); + expect(row.last_error).toEqual( + `Error: ${'y'.repeat(1_943)}... [error truncated]`, + ); + expect(row.status).toEqual('backing off'); + }, + ); + + it.each(databases.eachSupportedId())( + 'round-trips mark cursors so ingestion can resume, %p', + async databaseId => { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + + const manager = new OpenChoreoIncrementalIngestionDatabaseManager({ + client: knex, + logger: mockServices.logger.mock(), + }); + + const { ingestionId } = (await manager.createProviderIngestionRecord( + 'myProvider', + ))!; + + await manager.createMark({ + record: { + id: uuid(), + ingestion_id: ingestionId, + sequence: 1, + cursor: { data: 1 }, + }, + }); + await manager.createMark({ + record: { + id: uuid(), + ingestion_id: ingestionId, + sequence: 2, + cursor: { data: 2, nextPage: 'cursor-2' }, + }, + }); + + await expect(manager.getFirstMark(ingestionId)).resolves.toMatchObject({ + sequence: 1, + cursor: { data: 1 }, + }); + await expect(manager.getLastMark(ingestionId)).resolves.toMatchObject({ + sequence: 2, + cursor: { data: 2, nextPage: 'cursor-2' }, + }); + + const marks = await manager.getAllMarks(ingestionId); + expect(marks).toHaveLength(2); + expect(marks.map((m: any) => m.sequence)).toEqual([2, 1]); + }, + ); + + it.each(databases.eachSupportedId())( + 'applies migrations idempotently and records each migration exactly once, %p', + async databaseId => { + const knex = await databases.init(databaseId); + + // Running the real migrations directory twice must be safe and must + // not record duplicate entries. + await applyDatabaseMigrations(knex); + await applyDatabaseMigrations(knex); + + const names = (await knex(DB_MIGRATIONS_TABLE).select('name')).map( + (row: { name: string }) => row.name, + ); + expect(names.sort()).toEqual(EXPECTED_MIGRATION_NAMES); + expect(new Set(names).size).toEqual(names.length); + + // The performance index migration's SQLite branch must have run. + const indexes = ( + await knex.raw( + "SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_%'", + ) + ).map((row: { name: string }) => row.name); + expect(indexes.sort()).toEqual([ + 'idx_ingestion_mark_entities_composite', + 'idx_ingestion_mark_entities_ref', + 'idx_ingestion_marks_ingestion_id', + 'idx_ingestions_completion_ticket', + 'idx_ingestions_provider_name', + ]); + }, + ); + + it.each(databases.eachSupportedId())( + 'runs migrations exactly once across concurrent WrapperProviders connects, %p', + async databaseId => { + const knex = await databases.init(databaseId); + + let applierCalls = 0; + const countingApplier: typeof applyDatabaseMigrations = async client => { + applierCalls += 1; + await applyDatabaseMigrations(client); + }; + + const makeWrapper = () => + new WrapperProviders({ + config: mockServices.rootConfig.mock(), + logger: mockServices.logger.mock(), + client: knex, + scheduler: { + scheduleTask: jest.fn(), + } as unknown as SchedulerService, + applyDatabaseMigrations: countingApplier, + events: { + subscribe: jest.fn(), + } as unknown as EventsService, + }); + + const connection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const makeProvider = ( + name: string, + ): IncrementalEntityProvider => ({ + getProviderName: () => name, + next: async () => ({ done: true }), + around: async burst => { + await burst(undefined); + }, + }); + + const providerOptions: IncrementalEntityProviderOptions = { + burstInterval: { seconds: 30 }, + burstLength: { seconds: 30 }, + restLength: { seconds: 30 }, + }; + + // Two separate wrapper instances sharing one database client connect + // concurrently; both must await the same single migration run. + await Promise.all([ + makeWrapper() + .wrap(makeProvider('provider-a'), providerOptions) + .connect(connection), + makeWrapper() + .wrap(makeProvider('provider-b'), providerOptions) + .connect(connection), + ]); + + expect(applierCalls).toEqual(1); + + const names = (await knex(DB_MIGRATIONS_TABLE).select('name')).map( + (row: { name: string }) => row.name, + ); + expect(names.sort()).toEqual(EXPECTED_MIGRATION_NAMES); + }, + ); + + it.each(databases.eachSupportedId())( + 'rolls back all migrations cleanly, %p', + async databaseId => { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + + const [, rolledBack] = await knex.migrate.rollback({ + directory: migrationsDir, + tableName: DB_MIGRATIONS_TABLE, + }); + expect(rolledBack).toHaveLength(3); + + await expect( + knex(DB_MIGRATIONS_TABLE).select('name'), + ).resolves.toHaveLength(0); + }, + ); + + // Shared setup for the suites below: a migrated SQLite database plus a + // manager instance wired to a mock logger. + async function setup(databaseId: TestDatabaseId) { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + const manager = new OpenChoreoIncrementalIngestionDatabaseManager({ + client: knex, + logger: mockServices.logger.mock(), + }); + return { knex, manager }; + } + + describe('ingestion record updates', () => { + it.each(databases.eachSupportedId())( + 'updates records by id and ignores unknown ids, %p', + async databaseId => { + const { knex, manager } = await setup(databaseId); + const { ingestionId } = (await manager.createProviderIngestionRecord( + 'myProvider', + ))!; + + await manager.setProviderBursting(ingestionId); + let row = await knex('ingestions').where({ id: ingestionId }).first(); + expect(row.status).toEqual('bursting'); + expect(row.next_action).toEqual('ingest'); + expect(row.attempts).toEqual(0); + + await manager.updateIngestionRecordById({ + ingestionId, + update: { status: 'resting', next_action: 'rest' }, + }); + await manager.setProviderIngesting(ingestionId); + row = await knex('ingestions').where({ id: ingestionId }).first(); + expect(row.status).toEqual('resting'); + expect(row.next_action).toEqual('ingest'); + + // Updating a nonexistent id is a silent no-op. + await expect( + manager.updateIngestionRecordById({ + ingestionId: uuid(), + update: { status: 'bursting' }, + }), + ).resolves.toBeUndefined(); + row = await knex('ingestions').where({ id: ingestionId }).first(); + expect(row.status).toEqual('resting'); + }, + ); + + it.each(databases.eachSupportedId())( + 'resets attempts when entering the interstitial state, %p', + async databaseId => { + const { knex, manager } = await setup(databaseId); + const { ingestionId } = (await manager.createProviderIngestionRecord( + 'myProvider', + ))!; + + const before = Date.now(); + await manager.setProviderBackoff( + ingestionId, + 2, + new Error('source unavailable'), + 5_000, + ); + let row = await knex('ingestions').where({ id: ingestionId }).first(); + expect(row.status).toEqual('backing off'); + expect(row.next_action).toEqual('backoff'); + expect(row.attempts).toEqual(3); + expect(row.last_error).toContain('source unavailable'); + const backoffAt = new Date(row.next_action_at).getTime(); + expect(backoffAt).toBeGreaterThanOrEqual(before + 4_000); + expect(backoffAt).toBeLessThanOrEqual(Date.now() + 6_000); + + await manager.setProviderInterstitial(ingestionId); + row = await knex('ingestions').where({ id: ingestionId }).first(); + expect(row.status).toEqual('interstitial'); + expect(row.attempts).toEqual(0); + }, + ); + + it.each(databases.eachSupportedId())( + 'schedules the rest period when the burst completes, %p', + async databaseId => { + const { knex, manager } = await setup(databaseId); + const { ingestionId } = (await manager.createProviderIngestionRecord( + 'myProvider', + ))!; + + const before = Date.now(); + await manager.setProviderResting( + ingestionId, + Duration.fromObject({ minutes: 30 }), + ); + const row = await knex('ingestions').where({ id: ingestionId }).first(); + expect(row.next_action).toEqual('rest'); + expect(row.status).toEqual('resting'); + expect(row.ingestion_completed_at).toBeTruthy(); + const restAt = new Date(row.next_action_at).getTime(); + expect(restAt).toBeGreaterThanOrEqual(before + 29 * 60_000); + expect(restAt).toBeLessThanOrEqual(Date.now() + 31 * 60_000); + }, + ); + + it.each(databases.eachSupportedId())( + 'records the cancel reason and completes the cancellation, %p', + async databaseId => { + const { knex, manager } = await setup(databaseId); + const { ingestionId } = (await manager.createProviderIngestionRecord( + 'myProvider', + ))!; + + await manager.setProviderCanceling(ingestionId, 'stop requested'); + let row = await knex('ingestions').where({ id: ingestionId }).first(); + expect(row.next_action).toEqual('cancel'); + expect(row.status).toEqual('canceling'); + expect(row.last_error).toEqual('stop requested'); + expect(new Date(row.next_action_at).getTime()).toBeLessThanOrEqual( + Date.now(), + ); + + await manager.setProviderCanceled(ingestionId); + row = await knex('ingestions').where({ id: ingestionId }).first(); + expect(row.next_action).toEqual('nothing (canceled)'); + expect(row.status).toEqual('complete'); + expect(row.rest_completed_at).toBeTruthy(); + expect(row.completion_ticket).not.toEqual('open'); + + await expect( + manager.getCurrentIngestionRecord('myProvider'), + ).resolves.toBeUndefined(); + await expect( + manager.getPreviousIngestionRecord('myProvider'), + ).resolves.toMatchObject({ id: ingestionId }); + }, + ); + + it.each(databases.eachSupportedId())( + 'canceling without a message leaves last_error untouched, %p', + async databaseId => { + const { knex, manager } = await setup(databaseId); + const { ingestionId } = (await manager.createProviderIngestionRecord( + 'myProvider', + ))!; + + await manager.setProviderCanceling(ingestionId); + const row = await knex('ingestions').where({ id: ingestionId }).first(); + expect(row.status).toEqual('canceling'); + expect(row.next_action).toEqual('cancel'); + expect(row.last_error).toBeFalsy(); + }, + ); + + it.each(databases.eachSupportedId())( + 'provider updates and triggered actions only affect open tickets, %p', + async databaseId => { + const { knex, manager } = await setup(databaseId); + + const { ingestionId } = (await manager.createProviderIngestionRecord( + 'myProvider', + ))!; + const farFuture = new Date(Date.now() + 3_600_000); + await knex('ingestions') + .where({ id: ingestionId }) + .update({ next_action_at: farFuture }); + + const finishedId = await insertFinishedIngestion( + manager, + 'myProvider', + { restCompletedAt: new Date(Date.now() - 60_000) }, + ); + await knex('ingestions') + .where({ id: finishedId }) + .update({ next_action_at: farFuture }); + + await manager.triggerNextProviderAction('myProvider'); + let openRow = await knex('ingestions') + .where({ id: ingestionId }) + .first(); + let finishedRow = await knex('ingestions') + .where({ id: finishedId }) + .first(); + expect(new Date(openRow.next_action_at).getTime()).toBeLessThanOrEqual( + Date.now(), + ); + expect(new Date(finishedRow.next_action_at).getTime()).toBeGreaterThan( + Date.now() + 3_000_000, + ); + + await manager.updateByName('myProvider', { status: 'resting' }); + openRow = await knex('ingestions').where({ id: ingestionId }).first(); + finishedRow = await knex('ingestions') + .where({ id: finishedId }) + .first(); + expect(openRow.status).toEqual('resting'); + expect(finishedRow.status).toEqual('complete'); + }, + ); + }); + + describe('ingestion cleanup', () => { + it.each(databases.eachSupportedId())( + 'returns undefined when the provider already has an open ingestion, %p', + async databaseId => { + const { knex, manager } = await setup(databaseId); + const first = await manager.createProviderIngestionRecord('myProvider'); + + // The (provider_name, completion_ticket) uniqueness rejects the + // second open record, which the manager reports as undefined. + await expect( + manager.createProviderIngestionRecord('myProvider'), + ).resolves.toBeUndefined(); + + const record = await manager.getCurrentIngestionRecord('myProvider'); + expect(record!.id).toEqual(first!.ingestionId); + await expect(knex('ingestions')).resolves.toHaveLength(1); + }, + ); + + it.each(databases.eachSupportedId())( + 'clears stale duplicate active ingestions but keeps fresh ones, %p', + async databaseId => { + const { knex, manager } = await setup(databaseId); + const { ingestionId } = (await manager.createProviderIngestionRecord( + 'myProvider', + ))!; + + const staleId = uuid(); + await manager.insertIngestionRecord({ + id: staleId, + provider_name: 'myProvider', + status: 'bursting', + next_action: 'ingest', + completion_ticket: 'stale-ticket', + }); + await knex('ingestions') + .where({ id: staleId }) + .update({ + created_at: new Date(Date.now() - 2 * 3_600_000), + }); + await addMarkWithRefs(knex, staleId, 1, [ + 'component:default/stale-one', + 'component:default/stale-two', + ]); + + const freshId = uuid(); + await manager.insertIngestionRecord({ + id: freshId, + provider_name: 'myProvider', + status: 'bursting', + next_action: 'ingest', + completion_ticket: 'fresh-ticket', + }); + + await manager.clearDuplicateIngestions(ingestionId, 'myProvider'); + + const ids = (await knex('ingestions').select('id')).map( + (row: { id: string }) => row.id, + ); + expect(ids.sort()).toEqual([freshId, ingestionId].sort()); + await expect( + knex('ingestion_marks').where({ ingestion_id: staleId }), + ).resolves.toHaveLength(0); + await expect( + knex('ingestion_mark_entities').where({ + ref: 'component:default/stale-one', + }), + ).resolves.toHaveLength(0); + }, + ); + + it.each(databases.eachSupportedId())( + 'keeps the newest finished ingestion and the running one when clearing, %p', + async databaseId => { + const { knex, manager } = await setup(databaseId); + + const olderId = await insertFinishedIngestion(manager, 'myProvider', { + restCompletedAt: new Date(Date.now() - 2 * 3_600_000), + }); + await addMarkWithRefs(knex, olderId, 1, [ + 'component:default/older-one', + 'component:default/older-two', + ]); + const newerId = await insertFinishedIngestion(manager, 'myProvider', { + restCompletedAt: new Date(Date.now() - 3_600_000), + }); + await addMarkWithRefs(knex, newerId, 1, [ + 'component:default/newer-one', + ]); + const { ingestionId: runningId } = + (await manager.createProviderIngestionRecord('myProvider'))!; + + const { deletions } = await manager.clearFinishedIngestions( + 'myProvider', + ); + expect(deletions).toEqual({ + markEntitiesDeleted: 2, + marksDeleted: 1, + ingestionsDeleted: 1, + }); + + const ids = (await knex('ingestions').select('id')).map( + (row: { id: string }) => row.id, + ); + expect(ids.sort()).toEqual([newerId, runningId].sort()); + await expect( + knex('ingestion_marks').where({ ingestion_id: olderId }), + ).resolves.toHaveLength(0); + await expect( + knex('ingestion_marks').where({ ingestion_id: newerId }), + ).resolves.toHaveLength(1); + await expect( + knex('ingestion_mark_entities').where({ + ref: 'component:default/newer-one', + }), + ).resolves.toHaveLength(1); + + await expect( + manager.getPreviousIngestionRecord('myProvider'), + ).resolves.toMatchObject({ id: newerId }); + }, + ); + + it.each(databases.eachSupportedId())( + 'deletes nothing when only a running ingestion exists, %p', + async databaseId => { + const { manager } = await setup(databaseId); + const { ingestionId } = (await manager.createProviderIngestionRecord( + 'myProvider', + ))!; + + const { deletions } = await manager.clearFinishedIngestions( + 'myProvider', + ); + expect(deletions).toEqual({ + markEntitiesDeleted: 0, + marksDeleted: 0, + ingestionsDeleted: 0, + }); + await expect( + manager.getCurrentIngestionRecord('myProvider'), + ).resolves.toMatchObject({ id: ingestionId }); + }, + ); + }); + + describe('purge and reset', () => { + it.each(databases.eachSupportedId())( + 'purges all provider data and leaves it resting, %p', + async databaseId => { + const { knex, manager } = await setup(databaseId); + const { ingestionId } = (await manager.createProviderIngestionRecord( + 'myProvider', + ))!; + + const markId = uuid(); + await manager.createMark({ + record: { + id: markId, + ingestion_id: ingestionId, + sequence: 1, + cursor: { page: 1 }, + }, + }); + await manager.createMarkEntities(markId, [ + { entity: makeEntity('one') }, + { entity: makeEntity('two') }, + { entity: makeEntity('three') }, + ]); + + const result = await manager.purgeAndResetProvider('myProvider'); + expect(result).toMatchObject({ + provider: 'myProvider', + ingestionsDeleted: 1, + marksDeleted: 1, + markEntitiesDeleted: 3, + }); + + await expect(knex('ingestion_marks')).resolves.toHaveLength(0); + await expect(knex('ingestion_mark_entities')).resolves.toHaveLength(0); + await expect(knex('ingestions')).resolves.toHaveLength(1); + + const reset = await manager.getCurrentIngestionRecord('myProvider'); + expect(reset).toMatchObject({ + provider_name: 'myProvider', + status: 'resting', + next_action: 'rest', + completion_ticket: 'open', + }); + const resetAt = new Date(reset!.next_action_at).getTime(); + expect(resetAt).toBeGreaterThan(Date.now() + 23 * 3_600_000); + expect(resetAt).toBeLessThan(Date.now() + 25 * 3_600_000); + }, + ); + + it.each(databases.eachSupportedId())( + 'seeds a resting record for providers without history, %p', + async databaseId => { + const { manager } = await setup(databaseId); + + const result = await manager.purgeAndResetProvider('ghost'); + expect(result).toMatchObject({ + provider: 'ghost', + ingestionsDeleted: 0, + marksDeleted: 0, + markEntitiesDeleted: 0, + }); + await expect( + manager.getCurrentIngestionRecord('ghost'), + ).resolves.toMatchObject({ status: 'resting' }); + }, + ); + + it.each(databases.eachSupportedId())( + 'cleans up every provider across all tables, %p', + async databaseId => { + const { knex, manager } = await setup(databaseId); + + for (const provider of ['provider-a', 'provider-b']) { + const { ingestionId } = (await manager.createProviderIngestionRecord( + provider, + ))!; + await addMarkWithRefs(knex, ingestionId, 1, [ + `component:default/${provider}-one`, + `component:default/${provider}-two`, + ]); + } + + const result = await manager.cleanupProviders(); + expect(result).toEqual({ + ingestionsDeleted: 2, + // The marks cascade away with their ingestions, so the explicit + // mark purges find nothing left to remove. + ingestionMarksDeleted: 0, + markEntitiesDeleted: 0, + }); + + await expect(knex('ingestion_marks')).resolves.toHaveLength(0); + await expect(knex('ingestion_mark_entities')).resolves.toHaveLength(0); + + for (const provider of ['provider-a', 'provider-b']) { + await expect( + knex('ingestions').where({ provider_name: provider }), + ).resolves.toHaveLength(1); + await expect( + manager.getCurrentIngestionRecord(provider), + ).resolves.toMatchObject({ + status: 'resting', + next_action: 'rest', + completion_ticket: 'open', + }); + } + }, + ); + }); + + describe('mark-and-sweep bookkeeping', () => { + it.each(databases.eachSupportedId())( + 'computes removed entities between consecutive ingestions, %p', + async databaseId => { + const { knex, manager } = await setup(databaseId); + + // First ingestion marks three entities, then completes. + const first = (await manager.createProviderIngestionRecord( + 'myProvider', + ))!; + const firstMark = uuid(); + await manager.createMark({ + record: { + id: firstMark, + ingestion_id: first.ingestionId, + sequence: 1, + cursor: { page: 1 }, + }, + }); + await manager.createMarkEntities(firstMark, [ + { entity: makeEntity('one') }, + { entity: makeEntity('two') }, + { entity: makeEntity('three') }, + ]); + await manager.setProviderComplete(first.ingestionId); + + // Second ingestion re-marks only two of them. + const second = (await manager.createProviderIngestionRecord( + 'myProvider', + ))!; + const secondMark = uuid(); + await manager.createMark({ + record: { + id: secondMark, + ingestion_id: second.ingestionId, + sequence: 1, + cursor: { page: 1 }, + }, + }); + await manager.createMarkEntities(secondMark, [ + { entity: makeEntity('one') }, + { entity: makeEntity('two') }, + ]); + + const { total, removed } = await manager.computeRemoved( + 'myProvider', + second.ingestionId, + ); + expect(total).toEqual(2); + expect(removed).toEqual([{ entityRef: 'component:default/three' }]); + + // The re-marked entities now belong to the newest mark only. + await expect( + knex('ingestion_mark_entities').where({ + ingestion_mark_id: secondMark, + }), + ).resolves.toHaveLength(2); + await expect( + knex('ingestion_mark_entities').where({ + ingestion_mark_id: firstMark, + }), + ).resolves.toHaveLength(1); + }, + ); + + it.each(databases.eachSupportedId())( + 'computes no removals without a previous ingestion, %p', + async databaseId => { + const { knex, manager } = await setup(databaseId); + const { ingestionId } = (await manager.createProviderIngestionRecord( + 'myProvider', + ))!; + + const markId = uuid(); + await manager.createMark({ + record: { + id: markId, + ingestion_id: ingestionId, + sequence: 1, + cursor: { page: 1 }, + }, + }); + await knex('ingestion_mark_entities').insert( + ['component:default/one', 'component:default/two'].map(ref => ({ + id: uuid(), + ingestion_mark_id: markId, + ref, + })), + ); + + const { total, removed } = await manager.computeRemoved( + 'myProvider', + ingestionId, + ); + expect(total).toEqual(2); + expect(removed).toEqual([]); + }, + ); + + it.each(databases.eachSupportedId())( + 'counts marked entities by kind and flags malformed refs, %p', + async databaseId => { + const { knex, manager } = await setup(databaseId); + const { ingestionId } = (await manager.createProviderIngestionRecord( + 'myProvider', + ))!; + + const markId = uuid(); + await manager.createMark({ + record: { + id: markId, + ingestion_id: ingestionId, + sequence: 1, + cursor: { page: 1 }, + }, + }); + const refs = [ + 'Component:default/one', + 'component:default/two', + 'api:default/three', + 'group:default/four', + 'missing-colon-ref', + ':empty-kind', + ]; + await knex('ingestion_mark_entities').insert( + refs.map(ref => ({ + id: uuid(), + ingestion_mark_id: markId, + ref, + })), + ); + + await expect( + manager.getEntityCountsByKind(ingestionId), + ).resolves.toEqual({ + total: 6, + component: 2, + api: 1, + group: 1, + invalid: 2, + }); + }, + ); + + it.each(databases.eachSupportedId())( + 'deletes entity records by ref, %p', + async databaseId => { + const { knex, manager } = await setup(databaseId); + const { ingestionId } = (await manager.createProviderIngestionRecord( + 'myProvider', + ))!; + + await addMarkWithRefs(knex, ingestionId, 1, [ + 'component:default/one', + 'component:default/two', + 'component:default/three', + ]); + + await manager.deleteEntityRecordsByRef([ + { entityRef: 'component:default/one' }, + { entityRef: 'component:default/three' }, + ]); + await expect( + knex('ingestion_mark_entities').select('ref'), + ).resolves.toEqual([{ ref: 'component:default/two' }]); + + // Empty input is a no-op. + await expect( + manager.deleteEntityRecordsByRef([]), + ).resolves.toBeUndefined(); + await expect( + knex('ingestion_mark_entities').select('ref'), + ).resolves.toEqual([{ ref: 'component:default/two' }]); + }, + ); + + it.each(databases.eachSupportedId())( + 'healthchecks the active ingestion rows, %p', + async databaseId => { + const { manager } = await setup(databaseId); + + await manager.createProviderIngestionRecord('provider-a'); + await manager.createProviderIngestionRecord('provider-b'); + await insertFinishedIngestion(manager, 'provider-a', { + restCompletedAt: new Date(Date.now() - 60_000), + }); + + const rows = await manager.healthcheck(); + expect(rows).toHaveLength(2); + expect( + rows + .map((row: { provider_name: string }) => row.provider_name) + .sort(), + ).toEqual(['provider-a', 'provider-b']); + for (const row of rows) { + expect(row.id).toBeDefined(); + } + }, + ); + }); + + describe('failure handling', () => { + it.each(databases.eachSupportedId())( + 'reports wrapped transaction errors when tables are missing, %p', + async databaseId => { + // No migrations are applied, so every statement fails and each + // method must surface a wrapped transaction error. + const knex = await databases.init(databaseId); + const manager = new OpenChoreoIncrementalIngestionDatabaseManager({ + client: knex, + logger: mockServices.logger.mock(), + }); + + // The failed insert makes record creation report undefined. + await expect( + manager.createProviderIngestionRecord('myProvider'), + ).resolves.toBeUndefined(); + + await expect( + manager.updateIngestionRecordById({ + ingestionId: uuid(), + update: { status: 'resting' }, + }), + ).rejects.toThrow(); + await expect( + manager.updateIngestionRecordByProvider('myProvider', { + status: 'resting', + }), + ).rejects.toThrow(); + await expect( + manager.insertIngestionRecord({ + provider_name: 'myProvider', + status: 'bursting', + next_action: 'ingest', + completion_ticket: 'open', + }), + ).rejects.toThrow(); + await expect( + manager.getCurrentIngestionRecord('myProvider'), + ).rejects.toThrow(); + await expect( + manager.getPreviousIngestionRecord('myProvider'), + ).rejects.toThrow(); + await expect( + manager.clearFinishedIngestions('myProvider'), + ).rejects.toThrow(); + await expect( + manager.clearDuplicateIngestions(uuid(), 'myProvider'), + ).rejects.toThrow(); + await expect( + manager.purgeAndResetProvider('myProvider'), + ).rejects.toThrow(); + await expect( + manager.deleteEntityRecordsByRef([ + { entityRef: 'component:default/x' }, + ]), + ).rejects.toThrow(); + await expect( + manager.computeRemoved('myProvider', uuid()), + ).rejects.toThrow(); + await expect(manager.getEntityCountsByKind(uuid())).rejects.toThrow(); + await expect(manager.healthcheck()).rejects.toThrow(); + await expect(manager.getLastMark(uuid())).rejects.toThrow(); + await expect(manager.getFirstMark(uuid())).rejects.toThrow(); + await expect(manager.getAllMarks(uuid())).rejects.toThrow(); + await expect( + manager.createMark({ + record: { + id: uuid(), + ingestion_id: uuid(), + sequence: 1, + cursor: {}, + }, + }), + ).rejects.toThrow(); + await expect( + manager.createMarkEntities(uuid(), [{ entity: makeEntity('x') }]), + ).rejects.toThrow(); + await expect(manager.purgeTable('ingestions')).rejects.toThrow(); + await expect(manager.listProviders()).rejects.toThrow(); + await expect(manager.cleanupProviders()).rejects.toThrow(); + }, + ); + + it.each(databases.eachSupportedId())( + 'rejects mark reads when the stored cursor is corrupt, %p', + async databaseId => { + const { knex, manager } = await setup(databaseId); + const { ingestionId } = (await manager.createProviderIngestionRecord( + 'myProvider', + ))!; + + await knex('ingestion_marks').insert({ + id: uuid(), + ingestion_id: ingestionId, + sequence: 1, + cursor: 'not-json', + }); + + await expect(manager.getLastMark(ingestionId)).rejects.toThrow( + /Failed to decode mark cursor/, + ); + }, + ); + }); +}); diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/database/OpenChoreoIncrementalIngestionDatabaseManager.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/database/OpenChoreoIncrementalIngestionDatabaseManager.ts new file mode 100644 index 000000000..e19b9de3f --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/database/OpenChoreoIncrementalIngestionDatabaseManager.ts @@ -0,0 +1,1244 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Database manager for incremental ingestion operations. + * Manages ingestion records, marks, and entity tracking to support + * resumable, burst-based processing of large entity datasets. + */ + +import { Knex } from 'knex'; +import type { DeferredEntity } from '@backstage/plugin-catalog-node'; +import { stringifyEntityRef } from '@backstage/catalog-model'; +import { Duration } from 'luxon'; +import { v4 } from 'uuid'; +import { LoggerService } from '@backstage/backend-plugin-api'; +import { + IngestionRecord, + IngestionRecordUpdate, + IngestionUpsert, + MarkRecord, + MarkRecordInsert, +} from './tables'; +import { + DatabaseTransactionError, + DeadlockError, + ConstraintViolationError, + TransientDatabaseError, +} from './errors'; + +const POST_PROVIDER_RESET_COOLDOWN_MS = 24 * 60 * 60 * 1000; +const MARK_ENTITY_DELETE_BATCH_SIZE = 100; +const MARK_ENTITY_INSERT_BATCH_SIZE = 100; +const DUPLICATE_INGESTION_AGE_THRESHOLD_MS = 60000; + +/** + * Database-specific SQL variable limits: + * - SQLite: 999 (default), can be up to 32,766 at compile time + * - PostgreSQL: 32,767 (hard limit from protocol) + * - MySQL: 65,535 + * Using conservative limits to ensure compatibility across all configurations + */ +const SQL_VARIABLE_LIMITS = { + sqlite3: 900, // Conservative limit for SQLite (default is 999) + pg: 30000, // Conservative limit for PostgreSQL (max is 32,767) + mysql: 60000, // Conservative limit for MySQL (max is 65,535) + mysql2: 60000, + default: 900, // Safe default for unknown databases +}; + +export class OpenChoreoIncrementalIngestionDatabaseManager { + private client: Knex; + private logger: LoggerService; + private readonly batchSize: number; + + constructor(options: { client: Knex; logger: LoggerService }) { + this.client = options.client; + this.logger = options.logger; + this.batchSize = this.determineBatchSize(); + this.logger.info( + `Initialized database manager with batch size: ${this.batchSize} for client: ${this.client.client.config.client}`, + ); + } + + /** + * Determines the appropriate batch size for SQL IN clause operations + * based on the database client type. + */ + private determineBatchSize(): number { + const clientType = this.client.client.config.client; + const batchSize = + SQL_VARIABLE_LIMITS[clientType as keyof typeof SQL_VARIABLE_LIMITS] || + SQL_VARIABLE_LIMITS.default; + return batchSize; + } + + /** + * Safely formats an error for database storage. + * Truncates the error message if it's too long to prevent database constraint violations. + * @param error - The error to format + * @param maxLength - Maximum length (default: 2000 for TEXT fields, set to safe limit) + * @returns Formatted error string + */ + private formatErrorForStorage( + error: Error | string, + maxLength = 2000, + ): string { + const errorString = String(error); + if (errorString.length <= maxLength) { + return errorString; + } + // Truncate with an indicator + return `${errorString.substring(0, maxLength - 50)}... [error truncated]`; + } + + /** + * Helper method to execute a batched whereIn query operation. + * Automatically chunks the values to stay within database limits. + * + * This method prevents "too many SQL variables" errors that occur when + * SQL IN clauses contain more parameters than the database can handle: + * - SQLite: 999 variables (default) + * - PostgreSQL: 32,767 variables (protocol limit) + * - MySQL: 65,535 variables + * + * @param tx - Knex transaction + * @param tableName - Name of the table to query + * @param column - Column name for the WHERE IN clause + * @param values - Array of values to use in the IN clause + * @param operation - Type of operation ('select', 'delete', or 'update') + * @param updateData - Data to update (required for 'update' operation) + * @returns Array of results for 'select' operations, empty array otherwise + */ + private async batchedWhereIn( + tx: Knex.Transaction, + tableName: string, + column: string, + values: any[], + operation: 'select' | 'delete' | 'update', + updateData?: any, + ): Promise { + if (values.length === 0) { + return []; + } + + if (values.length > this.batchSize) { + this.logger.debug( + `Batching ${operation} operation for ${values.length} values into chunks of ${this.batchSize}`, + ); + } + + const results: T[] = []; + + for (let i = 0; i < values.length; i += this.batchSize) { + const chunk = values.slice(i, i + this.batchSize); + const query = tx(tableName); + + if (operation === 'select') { + const batchResults = await query.select('*').whereIn(column, chunk); + results.push(...batchResults); + } else if (operation === 'delete') { + await query.delete().whereIn(column, chunk); + } else if (operation === 'update' && updateData) { + await query.update(updateData).whereIn(column, chunk); + } + } + + return results; + } + + private async executeWithRetry( + operation: string, + fn: (tx: Knex.Transaction) => Promise, + maxRetries = 3, + ): Promise { + let lastError: Error | undefined; + + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + return await this.client.transaction(async tx => { + return await fn(tx); + }); + } catch (error) { + lastError = error as Error; + const errorCode = (error as any).code; + + if (errorCode === 'ER_LOCK_DEADLOCK' || errorCode === '40P01') { + if (attempt < maxRetries) { + const delay = Math.min(100 * Math.pow(2, attempt), 2000); + this.logger.warn( + `Deadlock detected in ${operation}, retrying in ${delay}ms (attempt ${ + attempt + 1 + }/${maxRetries})`, + ); + await new Promise(resolve => setTimeout(resolve, delay)); + continue; + } + throw new DeadlockError(operation, error as Error); + } + + if (errorCode === '23503' || errorCode === 'ER_NO_REFERENCED_ROW_2') { + throw new ConstraintViolationError( + 'Foreign key constraint violation', + operation, + (error as any).constraint, + error as Error, + ); + } + + if (errorCode === '23505' || errorCode === 'ER_DUP_ENTRY') { + throw new ConstraintViolationError( + 'Unique constraint violation', + operation, + (error as any).constraint, + error as Error, + ); + } + + if (errorCode === 'ECONNRESET' || errorCode === 'ETIMEDOUT') { + if (attempt < maxRetries) { + const delay = Math.min(500 * Math.pow(2, attempt), 5000); + this.logger.warn( + `Connection error in ${operation}, retrying in ${delay}ms`, + ); + await new Promise(resolve => setTimeout(resolve, delay)); + continue; + } + throw new TransientDatabaseError(operation, error as Error); + } + + this.logger.error( + `Transaction failed in ${operation}: ${(error as Error).message}`, + error as Error, + ); + throw new DatabaseTransactionError( + `Transaction failed: ${(error as Error).message}`, + operation, + error as Error, + ); + } + } + + throw new DatabaseTransactionError( + lastError?.message ?? 'Unknown transaction error', + operation, + lastError, + ); + } + + /** + * Performs an update to the ingestion record with matching `id`. + * @param options - IngestionRecordUpdate + */ + async updateIngestionRecordById(options: IngestionRecordUpdate) { + const { ingestionId, update } = options; + try { + await this.executeWithRetry( + `updateIngestionRecordById(ingestionId=${ingestionId})`, + async tx => { + await tx('ingestions').where('id', ingestionId).update(update); + }, + ); + } catch (error) { + this.logger.error( + `Failed to update ingestion record ${ingestionId}`, + error as Error, + ); + throw error; + } + } + + /** + * Performs an update to the ingestion record with matching provider name. Will only update active records. + * @param provider - string + * @param update - Partial + */ + async updateIngestionRecordByProvider( + provider: string, + update: Partial, + ) { + try { + await this.executeWithRetry( + `updateIngestionRecordByProvider(provider=${provider})`, + async tx => { + await tx('ingestions') + .where('provider_name', provider) + .andWhere('completion_ticket', 'open') + .update(update); + }, + ); + } catch (error) { + this.logger.error( + `Failed to update ingestion record for provider ${provider}`, + error as Error, + ); + throw error; + } + } + + /** + * Performs an insert into the `ingestions` table with the supplied values. + * @param record - IngestionUpsertIFace + */ + async insertIngestionRecord(record: IngestionUpsert) { + try { + await this.executeWithRetry( + `insertIngestionRecord(id=${record.id})`, + async tx => { + await tx('ingestions').insert(record); + }, + ); + } catch (error) { + this.logger.error( + `Failed to insert ingestion record ${record.id}`, + error as Error, + ); + throw error; + } + } + + private async deleteMarkEntities( + tx: Knex.Transaction, + ids: { id: string }[], + ) { + const chunks: { id: string }[][] = []; + for (let i = 0; i < ids.length; i += MARK_ENTITY_DELETE_BATCH_SIZE) { + const chunk = ids.slice(i, i + MARK_ENTITY_DELETE_BATCH_SIZE); + chunks.push(chunk); + } + + let deleted = 0; + + for (const chunk of chunks) { + const chunkDeleted = await tx('ingestion_mark_entities') + .delete() + .whereIn( + 'id', + chunk.map(entry => entry.id), + ); + deleted += chunkDeleted; + } + + return deleted; + } + + /** + * Finds the current ingestion record for the named provider. + * @param provider - string + * @returns IngestionRecord | undefined + */ + async getCurrentIngestionRecord(provider: string) { + try { + return await this.executeWithRetry( + `getCurrentIngestionRecord(provider=${provider})`, + async tx => { + const record = await tx('ingestions') + .where('provider_name', provider) + .andWhere('completion_ticket', 'open') + .first(); + return record; + }, + ); + } catch (error) { + this.logger.error( + `Failed to get current ingestion record for provider ${provider}`, + error as Error, + ); + throw error; + } + } + + /** + * Finds the last ingestion record for the named provider. + * @param provider - string + * @returns IngestionRecord | undefined + */ + async getPreviousIngestionRecord(provider: string) { + try { + return await this.executeWithRetry( + `getPreviousIngestionRecord(provider=${provider})`, + async tx => { + return await tx('ingestions') + .where('provider_name', provider) + .andWhereNot('completion_ticket', 'open') + .orderBy('rest_completed_at', 'desc') + .first(); + }, + ); + } catch (error) { + this.logger.error( + `Failed to get previous ingestion record for provider ${provider}`, + error as Error, + ); + throw error; + } + } + + /** + * Removes all entries from `ingestion_marks_entities`, `ingestion_marks`, and `ingestions` + * for prior ingestions that completed (i.e., have a `completion_ticket` value other than 'open'), + * except for the most recent completed ingestion which is kept for mark-and-sweep comparison. + * @param provider - string + * @returns A count of deletions for each record type. + * + * Note: This method uses subqueries for deletion which doesn't require manual batching + * as the database handles the query execution internally. + */ + async clearFinishedIngestions(provider: string) { + try { + return await this.executeWithRetry( + `clearFinishedIngestions(provider=${provider})`, + async tx => { + const mostRecentCompleted = await tx('ingestions') + .where('provider_name', provider) + .andWhereNot('completion_ticket', 'open') + .orderBy('rest_completed_at', 'desc') + .first(); + + const subquery = tx('ingestions') + .select('id') + .where('provider_name', provider) + .andWhereNot('completion_ticket', 'open'); + + if (mostRecentCompleted) { + subquery.andWhereNot('id', mostRecentCompleted.id); + } + + const markEntitiesDeleted = await tx('ingestion_mark_entities') + .delete() + .whereIn( + 'ingestion_mark_id', + tx('ingestion_marks') + .select('id') + .whereIn('ingestion_id', subquery.clone()), + ); + + const marksDeleted = await tx('ingestion_marks') + .delete() + .whereIn('ingestion_id', subquery.clone()); + + const ingestionsDeleted = await tx('ingestions') + .delete() + .whereIn('id', subquery.clone()); + + return { + deletions: { + markEntitiesDeleted, + marksDeleted, + ingestionsDeleted, + }, + }; + }, + ); + } catch (error) { + this.logger.error( + `Failed to clear finished ingestions for provider ${provider}`, + error as Error, + ); + throw error; + } + } + + /** + * Automatically cleans up duplicate ingestion records if they were accidentally created. + * Any ingestion record where the `rest_completed_at` is null (meaning it is active) AND + * the ingestionId is incorrect is a duplicate ingestion record. + * @param ingestionId - string + * @param provider - string + * + * Note: This method does not require batching as it operates on a small number of + * ingestion metadata records, not entity data. + */ + async clearDuplicateIngestions(ingestionId: string, provider: string) { + try { + await this.executeWithRetry( + `clearDuplicateIngestions(ingestionId=${ingestionId}, provider=${provider})`, + async tx => { + const invalid = await tx('ingestions') + .where('provider_name', provider) + .andWhere('rest_completed_at', null) + .andWhereNot('id', ingestionId) + .andWhere( + 'created_at', + '<', + new Date(Date.now() - DUPLICATE_INGESTION_AGE_THRESHOLD_MS), + ); + + if (invalid.length > 0) { + await tx('ingestions') + .delete() + .whereIn( + 'id', + invalid.map(i => i.id), + ); + await tx('ingestion_mark_entities') + .delete() + .whereIn( + 'ingestion_mark_id', + tx('ingestion_marks') + .select('id') + .whereIn( + 'ingestion_id', + invalid.map(i => i.id), + ), + ); + await tx('ingestion_marks') + .delete() + .whereIn( + 'ingestion_id', + invalid.map(i => i.id), + ); + } + }, + ); + } catch (error) { + this.logger.error( + `Failed to clear duplicate ingestions for ${provider}`, + error as Error, + ); + throw error; + } + } + + /** + * This method fully purges and resets all ingestion records for the named provider, and + * leaves it in a paused state. + * @param provider - string + * @returns Counts of all deleted ingestion records + * + * Note: This method does not require batching for whereIn operations as it operates + * on a small number of ingestion and mark metadata records per provider. + */ + async purgeAndResetProvider(provider: string) { + try { + return await this.executeWithRetry( + `purgeAndResetProvider(provider=${provider})`, + async tx => { + const ingestionIDs: { id: string }[] = await tx('ingestions') + .select('id') + .where('provider_name', provider); + + const markIDs: { id: string }[] = + ingestionIDs.length > 0 + ? await tx('ingestion_marks') + .select('id') + .whereIn( + 'ingestion_id', + ingestionIDs.map(entry => entry.id), + ) + : []; + + const markEntityIDs: { id: string }[] = + markIDs.length > 0 + ? await tx('ingestion_mark_entities') + .select('id') + .whereIn( + 'ingestion_mark_id', + markIDs.map(entry => entry.id), + ) + : []; + + const markEntitiesDeleted = await this.deleteMarkEntities( + tx, + markEntityIDs, + ); + + const marksDeleted = + markIDs.length > 0 + ? await tx('ingestion_marks') + .delete() + .whereIn( + 'ingestion_id', + ingestionIDs.map(entry => entry.id), + ) + : 0; + + const ingestionsDeleted = await tx('ingestions') + .delete() + .where('provider_name', provider); + + const next_action_at = new Date(); + next_action_at.setTime( + next_action_at.getTime() + POST_PROVIDER_RESET_COOLDOWN_MS, + ); + + await tx('ingestions').insert({ + id: v4(), + next_action: 'rest', + provider_name: provider, + next_action_at, + ingestion_completed_at: new Date(), + status: 'resting', + completion_ticket: 'open', + }); + + return { + provider, + ingestionsDeleted, + marksDeleted, + markEntitiesDeleted, + }; + }, + ); + } catch (error) { + this.logger.error( + `Failed to purge and reset provider ${provider}`, + error as Error, + ); + throw error; + } + } + + /** + * This method is used to remove entity records from the ingestion_mark_entities + * table by their entity reference. + */ + async deleteEntityRecordsByRef(entities: { entityRef: string }[]) { + const refs = entities.map(e => e.entityRef); + try { + await this.executeWithRetry( + `deleteEntityRecordsByRef(count=${refs.length})`, + async tx => { + // Delete in batches to avoid "too many SQL variables" error + await this.batchedWhereIn( + tx, + 'ingestion_mark_entities', + 'ref', + refs, + 'delete', + ); + }, + ); + } catch (error) { + this.logger.error( + `Failed to delete ${refs.length} entity records`, + error as Error, + ); + throw error; + } + } + + /** + * Creates a new ingestion record. + * @param provider - string + * @returns A new ingestion record + */ + async createProviderIngestionRecord(provider: string) { + const ingestionId = v4(); + const nextAction = 'ingest'; + try { + await this.insertIngestionRecord({ + id: ingestionId, + next_action: nextAction, + provider_name: provider, + status: 'bursting', + completion_ticket: 'open', + }); + return { ingestionId, nextAction, attempts: 0, nextActionAt: Date.now() }; + } catch (error) { + this.logger.error( + `Failed to create ingestion record for provider ${provider} with ingestionId ${ingestionId}`, + error as Error, + ); + // Creating the ingestion record failed. Return undefined. + return undefined; + } + } + + /** + * Computes which entities to remove, if any, at the end of a burst. + * Implements proper mark-and-sweep by comparing previous ingestion entities + * against current ingestion entities to identify orphans. + * @param provider - string + * @param ingestionId - string + * @returns All entities to remove for this burst. + */ + async computeRemoved(provider: string, ingestionId: string) { + const previousIngestion = await this.getPreviousIngestionRecord(provider); + try { + return await this.executeWithRetry( + `computeRemoved(provider=${provider}, ingestionId=${ingestionId})`, + async tx => { + const count = await tx('ingestion_mark_entities') + .count({ total: 'ingestion_mark_entities.ref' }) + .join( + 'ingestion_marks', + 'ingestion_marks.id', + 'ingestion_mark_entities.ingestion_mark_id', + ) + .join('ingestions', 'ingestions.id', 'ingestion_marks.ingestion_id') + .where('ingestions.id', ingestionId); + + const total = count.reduce( + (acc, cur) => acc + (cur.total as number), + 0, + ); + + const removed: { entityRef: string }[] = []; + + const currentEntities: { ref: string }[] = await tx( + 'ingestion_mark_entities', + ) + .select('ingestion_mark_entities.ref') + .join( + 'ingestion_marks', + 'ingestion_marks.id', + 'ingestion_mark_entities.ingestion_mark_id', + ) + .join('ingestions', 'ingestions.id', 'ingestion_marks.ingestion_id') + .where('ingestions.id', ingestionId); + + const currentEntityRefs = new Set(currentEntities.map(e => e.ref)); + + if (previousIngestion) { + const previousEntities: { ref: string }[] = await tx( + 'ingestion_mark_entities', + ) + .select('ingestion_mark_entities.ref') + .join( + 'ingestion_marks', + 'ingestion_marks.id', + 'ingestion_mark_entities.ingestion_mark_id', + ) + .join( + 'ingestions', + 'ingestions.id', + 'ingestion_marks.ingestion_id', + ) + .where('ingestions.id', previousIngestion.id); + + const staleEntities = previousEntities.filter( + entity => !currentEntityRefs.has(entity.ref), + ); + + for (const entityRef of staleEntities) { + removed.push({ entityRef: entityRef.ref }); + } + } + + return { total, removed }; + }, + ); + } catch (error) { + this.logger.error( + `Failed to compute removed entities for ${provider}`, + error as Error, + ); + throw error; + } + } + + async getEntityCountsByKind(ingestionId: string) { + try { + return await this.executeWithRetry( + `getEntityCountsByKind(ingestionId=${ingestionId})`, + async tx => { + const entityRefs: { ref: string }[] = await tx( + 'ingestion_mark_entities', + ) + .select('ingestion_mark_entities.ref') + .join( + 'ingestion_marks', + 'ingestion_marks.id', + 'ingestion_mark_entities.ingestion_mark_id', + ) + .join('ingestions', 'ingestions.id', 'ingestion_marks.ingestion_id') + .where('ingestions.id', ingestionId); + + // Count entities by kind - parse kind from entity ref format: :/ + const counts: Record = { + total: entityRefs.length, + }; + + let invalid = 0; + + for (const { ref } of entityRefs) { + try { + // Entity refs are in format: kind:namespace/name + const colonIndex = ref.indexOf(':'); + if (colonIndex === -1) { + invalid++; + this.logger.warn( + `Invalid entity ref format (missing colon): ${ref} in ingestion ${ingestionId}`, + ); + continue; + } + + const kind = ref.substring(0, colonIndex).toLowerCase(); + + if (!kind) { + invalid++; + this.logger.warn( + `Invalid entity ref format (empty kind): ${ref} in ingestion ${ingestionId}`, + ); + continue; + } + + counts[kind] = (counts[kind] || 0) + 1; + } catch (error) { + invalid++; + this.logger.warn( + `Failed to parse entity ref ${ref} in ingestion ${ingestionId}: ${ + (error as Error).message + }`, + ); + } + } + + if (invalid > 0) { + counts.invalid = invalid; + this.logger.warn( + `Found ${invalid} entities with invalid ref format out of ${entityRefs.length} total entities in ingestion ${ingestionId}`, + ); + } + + return counts; + }, + ); + } catch (error) { + this.logger.error( + `Failed to get entity counts for ingestion ${ingestionId}`, + error as Error, + ); + throw error; + } + } + + /** + * Performs a lookup of all providers that have duplicate active ingestion records. + * @returns An array of all duplicate active ingestions + */ + async healthcheck() { + try { + return await this.executeWithRetry('healthcheck', async tx => { + const records = await tx<{ id: string; provider_name: string }>( + 'ingestions', + ) + .distinct('id', 'provider_name') + .where('rest_completed_at', null); + return records; + }); + } catch (error) { + this.logger.error('Failed to perform healthcheck', error as Error); + throw error; + } + } + + /** + * Skips any wait time for the next action to run. + * @param provider - string + */ + async triggerNextProviderAction(provider: string) { + await this.updateIngestionRecordByProvider(provider, { + next_action_at: new Date(), + }); + } + + /** + * Purges the following tables: + * * `ingestions` + * * `ingestion_marks` + * * `ingestion_mark_entities` + * + * This function leaves the ingestions table with all providers in a paused state. + * @returns Results from cleaning up all ingestion tables. + */ + async cleanupProviders() { + const providers = await this.listProviders(); + + const ingestionsDeleted = await this.purgeTable('ingestions'); + + const next_action_at = new Date(); + next_action_at.setTime( + next_action_at.getTime() + POST_PROVIDER_RESET_COOLDOWN_MS, + ); + + for (const provider of providers) { + await this.insertIngestionRecord({ + id: v4(), + next_action: 'rest', + provider_name: provider, + next_action_at, + ingestion_completed_at: new Date(), + status: 'resting', + completion_ticket: 'open', + }); + } + + const ingestionMarksDeleted = await this.purgeTable('ingestion_marks'); + const markEntitiesDeleted = await this.purgeTable( + 'ingestion_mark_entities', + ); + + return { ingestionsDeleted, ingestionMarksDeleted, markEntitiesDeleted }; + } + + /** + * Configures the current ingestion record to ingest a burst. + * @param ingestionId - string + */ + async setProviderIngesting(ingestionId: string) { + await this.updateIngestionRecordById({ + ingestionId, + update: { next_action: 'ingest' }, + }); + } + + /** + * Indicates the provider is currently ingesting a burst. + * @param ingestionId - string + */ + async setProviderBursting(ingestionId: string) { + await this.updateIngestionRecordById({ + ingestionId, + update: { status: 'bursting' }, + }); + } + + /** + * Finalizes the current ingestion record to indicate that the post-ingestion rest period is complete. + * @param ingestionId - string + */ + async setProviderComplete(ingestionId: string) { + await this.updateIngestionRecordById({ + ingestionId, + update: { + next_action: 'nothing (done)', + rest_completed_at: new Date(), + status: 'complete', + completion_ticket: v4(), + }, + }); + } + + /** + * Marks ingestion as complete and starts the post-ingestion rest cycle. + * @param ingestionId - string + * @param restLength - Duration + */ + async setProviderResting(ingestionId: string, restLength: Duration) { + await this.updateIngestionRecordById({ + ingestionId, + update: { + next_action: 'rest', + next_action_at: new Date(Date.now() + restLength.as('milliseconds')), + ingestion_completed_at: new Date(), + status: 'resting', + }, + }); + } + + /** + * Marks ingestion as paused after a burst completes. + * @param ingestionId - string + */ + async setProviderInterstitial(ingestionId: string) { + await this.updateIngestionRecordById({ + ingestionId, + update: { attempts: 0, status: 'interstitial' }, + }); + } + + /** + * Starts the cancel process for the current ingestion. + * @param ingestionId - string + * @param message - string (optional) + */ + async setProviderCanceling(ingestionId: string, message?: string) { + const update: Partial = { + next_action: 'cancel', + last_error: message ? this.formatErrorForStorage(message) : undefined, + next_action_at: new Date(), + status: 'canceling', + }; + await this.updateIngestionRecordById({ ingestionId, update }); + } + + /** + * Completes the cancel process and triggers a new ingestion. + * @param ingestionId - string + */ + async setProviderCanceled(ingestionId: string) { + await this.updateIngestionRecordById({ + ingestionId, + update: { + next_action: 'nothing (canceled)', + rest_completed_at: new Date(), + status: 'complete', + completion_ticket: v4(), + }, + }); + } + + /** + * Configures the current ingestion to wait and retry, due to a data source error. + * @param ingestionId - string + * @param attempts - number + * @param error - Error + * @param backoffLength - number + */ + async setProviderBackoff( + ingestionId: string, + attempts: number, + error: Error, + backoffLength: number, + ) { + await this.updateIngestionRecordById({ + ingestionId, + update: { + next_action: 'backoff', + attempts: attempts + 1, + last_error: this.formatErrorForStorage(error), + next_action_at: new Date(Date.now() + backoffLength), + status: 'backing off', + }, + }); + } + + /** + * Returns the last record from `ingestion_marks` for the supplied ingestionId. + * @param ingestionId - string + * @returns MarkRecord | undefined + */ + async getLastMark(ingestionId: string) { + try { + return await this.executeWithRetry( + `getLastMark(ingestionId=${ingestionId})`, + async tx => { + const mark = await tx('ingestion_marks') + .where('ingestion_id', ingestionId) + .orderBy('sequence', 'desc') + .first(); + return this.#decodeMark(this.client, mark); + }, + ); + } catch (error) { + this.logger.error( + `Failed to get last mark for ingestion ${ingestionId}`, + error as Error, + ); + throw error; + } + } + + /** + * Returns the first record from `ingestion_marks` for the supplied ingestionId. + * @param ingestionId - string + * @returns MarkRecord | undefined + */ + async getFirstMark(ingestionId: string) { + try { + return await this.executeWithRetry( + `getFirstMark(ingestionId=${ingestionId})`, + async tx => { + const mark = await tx('ingestion_marks') + .where('ingestion_id', ingestionId) + .orderBy('sequence', 'asc') + .first(); + return this.#decodeMark(this.client, mark); + }, + ); + } catch (error) { + this.logger.error( + `Failed to get first mark for ingestion ${ingestionId}`, + error as Error, + ); + throw error; + } + } + + async getAllMarks(ingestionId: string) { + try { + return await this.executeWithRetry( + `getAllMarks(ingestionId=${ingestionId})`, + async tx => { + const marks = await tx('ingestion_marks') + .where('ingestion_id', ingestionId) + .orderBy('sequence', 'desc'); + return marks.map(m => this.#decodeMark(this.client, m)); + }, + ); + } catch (error) { + this.logger.error( + `Failed to get all marks for ingestion ${ingestionId}`, + error as Error, + ); + throw error; + } + } + + /** + * Performs an insert into the `ingestion_marks` table with the supplied values. + * @param options - MarkRecordInsert + */ + async createMark(options: MarkRecordInsert) { + const { record } = options; + try { + await this.executeWithRetry( + `createMark(ingestionId=${record.ingestion_id})`, + async tx => { + await tx('ingestion_marks').insert(record); + }, + ); + } catch (error) { + this.logger.error( + `Failed to create mark for ingestion ${record.ingestion_id}`, + error as Error, + ); + throw error; + } + } + + // Handles the fact that sqlite does not support json columns; they just + // persist the stringified data instead + #decodeMark(knex: Knex, record: T): T { + if (record && knex.client.config.client.includes('sqlite3')) { + try { + return { + ...record, + cursor: JSON.parse(record.cursor as string), + }; + } catch (error) { + this.logger.error( + `Failed to parse cursor JSON for mark record ${record.id}: ${ + (error as Error).message + }. This indicates database corruption.`, + error as Error, + ); + throw new DatabaseTransactionError( + `Failed to decode mark cursor: ${(error as Error).message}`, + 'decodeMark', + error as Error, + ); + } + } + return record; + } + + /** + * Performs an upsert to the `ingestion_mark_entities` table for all deferred entities. + * @param markId - string + * @param entities - DeferredEntity[] + */ + async createMarkEntities(markId: string, entities: DeferredEntity[]) { + const refs = entities.map(e => stringifyEntityRef(e.entity)); + + try { + await this.executeWithRetry( + `createMarkEntities(markId=${markId}, count=${refs.length})`, + async tx => { + // Query existing refs in batches to avoid "too many SQL variables" error + const existingRefsSet = new Set(); + for (let i = 0; i < refs.length; i += this.batchSize) { + const chunk = refs.slice(i, i + this.batchSize); + const existingBatch = ( + await tx<{ ref: string }>('ingestion_mark_entities') + .select('ref') + .whereIn('ref', chunk) + ).map(e => e.ref); + existingBatch.forEach(ref => existingRefsSet.add(ref)); + } + + const existingRefsArray = Array.from(existingRefsSet); + const newRefs = refs.filter(e => !existingRefsSet.has(e)); + + // Update existing refs in batches + if (existingRefsArray.length > 0) { + await this.batchedWhereIn( + tx, + 'ingestion_mark_entities', + 'ref', + existingRefsArray, + 'update', + { ingestion_mark_id: markId }, + ); + } + + if (newRefs.length > 0) { + // Process newRefs in batches to avoid overwhelming the database + for ( + let i = 0; + i < newRefs.length; + i += MARK_ENTITY_INSERT_BATCH_SIZE + ) { + const chunk = newRefs.slice(i, i + MARK_ENTITY_INSERT_BATCH_SIZE); + await tx('ingestion_mark_entities').insert( + chunk.map(ref => ({ + id: v4(), + ingestion_mark_id: markId, + ref, + })), + ); + this.logger.info( + `Batch ${ + Math.floor(i / MARK_ENTITY_INSERT_BATCH_SIZE) + 1 + }/${Math.ceil( + newRefs.length / MARK_ENTITY_INSERT_BATCH_SIZE, + )} completed: inserted ${ + chunk.length + } entities for mark ${markId}`, + ); + } + } + }, + ); + } catch (error) { + this.logger.error( + `Failed to create mark entities for mark ${markId} (${refs.length} entities)`, + error as Error, + ); + throw error; + } + } + + /** + * Deletes the entire content of a table, and returns the number of records deleted. + * @param table - string + * @returns number + */ + async purgeTable(table: string) { + try { + return await this.executeWithRetry(`purgeTable(${table})`, async tx => { + return await tx(table).delete(); + }); + } catch (error) { + this.logger.error(`Failed to purge table ${table}`, error as Error); + throw error; + } + } + + /** + * Returns a list of all providers. + * @returns string[] + */ + async listProviders() { + try { + return await this.executeWithRetry('listProviders', async tx => { + const providers = await tx<{ provider_name: string }>( + 'ingestions', + ).distinct('provider_name'); + return providers.map(entry => entry.provider_name); + }); + } catch (error) { + this.logger.error('Failed to list providers', error as Error); + throw error; + } + } + + async updateByName(provider: string, update: Partial) { + await this.updateIngestionRecordByProvider(provider, update); + } +} diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/database/errors.test.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/database/errors.test.ts new file mode 100644 index 000000000..1aae6253b --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/database/errors.test.ts @@ -0,0 +1,118 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Test suite for the database error taxonomy. + * Verifies class hierarchy, error names, message formatting, and the + * operation/cause/constraint metadata each error class carries. + */ +import { + ConstraintViolationError, + DatabaseTransactionError, + DeadlockError, + OpenChoreoIncrementalIngestionError, + TransientDatabaseError, +} from './errors'; + +describe('database errors', () => { + it('formats DatabaseTransactionError with operation and cause', () => { + const cause = new Error('connection reset'); + const error = new DatabaseTransactionError( + 'Transaction failed: connection reset', + 'updateIngestionRecordById(ingestionId=abc)', + cause, + ); + + expect(error).toBeInstanceOf(DatabaseTransactionError); + expect(error).toBeInstanceOf(Error); + expect(error.name).toBe('DatabaseTransactionError'); + expect(error.message).toBe('Transaction failed: connection reset'); + expect(error.operation).toBe('updateIngestionRecordById(ingestionId=abc)'); + expect(error.cause).toBe(cause); + }); + + it('allows DatabaseTransactionError without a cause', () => { + const error = new DatabaseTransactionError('boom', 'purgeTable'); + expect(error.cause).toBeUndefined(); + expect(error.operation).toBe('purgeTable'); + }); + + it('formats DeadlockError with the fixed message', () => { + const cause = new Error('deadlock victim'); + const error = new DeadlockError('clearFinishedIngestions', cause); + + expect(error).toBeInstanceOf(DatabaseTransactionError); + expect(error.name).toBe('DeadlockError'); + expect(error.message).toBe('Transaction deadlock detected'); + expect(error.operation).toBe('clearFinishedIngestions'); + expect(error.cause).toBe(cause); + }); + + it('formats ConstraintViolationError with the constraint name', () => { + const cause = new Error('duplicate key'); + const error = new ConstraintViolationError( + 'Unique constraint violation', + 'insertIngestionRecord', + 'ingestion_composite_index', + cause, + ); + + expect(error).toBeInstanceOf(DatabaseTransactionError); + expect(error.name).toBe('ConstraintViolationError'); + expect(error.message).toBe('Unique constraint violation'); + expect(error.operation).toBe('insertIngestionRecord'); + expect(error.constraintName).toBe('ingestion_composite_index'); + expect(error.cause).toBe(cause); + + const withoutConstraint = new ConstraintViolationError( + 'Foreign key constraint violation', + 'createMark', + ); + expect(withoutConstraint.constraintName).toBeUndefined(); + }); + + it('formats TransientDatabaseError with the fixed message', () => { + const cause = new Error('ETIMEDOUT'); + const error = new TransientDatabaseError('healthcheck', cause); + + expect(error).toBeInstanceOf(DatabaseTransactionError); + expect(error.name).toBe('TransientDatabaseError'); + expect(error.message).toBe('Transient database error - retry possible'); + expect(error.operation).toBe('healthcheck'); + expect(error.cause).toBe(cause); + }); + + it('formats OpenChoreoIncrementalIngestionError with a code', () => { + const cause = new Error('boom'); + const error = new OpenChoreoIncrementalIngestionError( + 'Failed to validate configuration', + 'CONFIG_VALIDATION_ERROR', + cause, + ); + + expect(error).toBeInstanceOf(Error); + expect(error.name).toBe('OpenChoreoIncrementalIngestionError'); + expect(error.message).toBe('Failed to validate configuration'); + expect(error.code).toBe('CONFIG_VALIDATION_ERROR'); + expect(error.cause).toBe(cause); + + const withoutCause = new OpenChoreoIncrementalIngestionError( + 'Something failed', + 'UNKNOWN', + ); + expect(withoutCause.cause).toBeUndefined(); + }); +}); diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/database/errors.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/database/errors.ts new file mode 100644 index 000000000..b22d5f5ec --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/database/errors.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export class DatabaseTransactionError extends Error { + constructor( + message: string, + public readonly operation: string, + public readonly cause?: Error, + ) { + super(message); + this.name = 'DatabaseTransactionError'; + } +} + +export class DeadlockError extends DatabaseTransactionError { + constructor(operation: string, cause?: Error) { + super('Transaction deadlock detected', operation, cause); + this.name = 'DeadlockError'; + } +} + +export class ConstraintViolationError extends DatabaseTransactionError { + constructor( + message: string, + operation: string, + public readonly constraintName?: string, + cause?: Error, + ) { + super(message, operation, cause); + this.name = 'ConstraintViolationError'; + } +} + +export class TransientDatabaseError extends DatabaseTransactionError { + constructor(operation: string, cause?: Error) { + super('Transient database error - retry possible', operation, cause); + this.name = 'TransientDatabaseError'; + } +} + +export class OpenChoreoIncrementalIngestionError extends Error { + constructor( + message: string, + public readonly code: string, + public readonly cause?: Error, + ) { + super(message); + this.name = 'OpenChoreoIncrementalIngestionError'; + } +} diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/database/migrations.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/database/migrations.ts new file mode 100644 index 000000000..ccbb446b7 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/database/migrations.ts @@ -0,0 +1,35 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Database migrations for incremental ingestion. + * Applies schema changes for ingestion tables. + */ +import { resolvePackagePath } from '@backstage/backend-plugin-api'; +import { Knex } from 'knex'; +import { DB_MIGRATIONS_TABLE } from './tables'; + +export async function applyDatabaseMigrations(knex: Knex): Promise { + const migrationsDir = resolvePackagePath( + '@openchoreo/backstage-plugin-catalog-backend-module-openchoreo-incremental', + 'migrations', + ); + + await knex.migrate.latest({ + directory: migrationsDir, + tableName: DB_MIGRATIONS_TABLE, + }); +} diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/database/tables.test.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/database/tables.test.ts new file mode 100644 index 000000000..efcdbe74e --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/database/tables.test.ts @@ -0,0 +1,246 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Test suite for the database table contracts. + * Cross-checks every table and column declared by src/database/tables.ts + * against the CREATE TABLE / ALTER TABLE SQL in the migrations, and verifies + * that the database manager only references tables that the migrations + * actually create. + */ +import { readFileSync } from 'fs'; +import * as path from 'path'; +import { + DB_MIGRATIONS_TABLE, + IngestionRecord, + IngestionUpsert, + MarkRecord, + MarkRecordInsert, +} from './tables'; + +const MIGRATIONS_DIR = path.resolve(__dirname, '../../migrations'); +const initMigration = readFileSync( + path.join(MIGRATIONS_DIR, '20221116073152_init.js'), + 'utf8', +); +const expandMigration = readFileSync( + path.join(MIGRATIONS_DIR, '20240110000003_expand_last_error_field.js'), + 'utf8', +); +const migrationsSource = readFileSync( + path.resolve(__dirname, 'migrations.ts'), + 'utf8', +); +const managerSource = readFileSync( + path.resolve(__dirname, 'OpenChoreoIncrementalIngestionDatabaseManager.ts'), + 'utf8', +); + +const MIGRATION_TABLES = [ + 'ingestions', + 'ingestion_marks', + 'ingestion_mark_entities', +]; + +/** + * Extracts the source block of one `createTable('name', ...)` call, ending + * at the next top-level `knex.schema` statement. + */ +function extractCreateTableBlock(source: string, tableName: string): string { + const start = source.indexOf(`createTable('${tableName}'`); + if (start === -1) { + throw new Error(`No createTable call found for '${tableName}'`); + } + const end = source.indexOf('await knex.schema', start + 1); + return source.slice(start, end === -1 ? undefined : end); +} + +/** Matches knex column builders like `.uuid('id')` or `.string('ref')`. */ +const COLUMN_REGEX = + /\.(?:uuid|string|integer|biginteger|timestamp|json|jsonb|text|boolean|float|double|date|datetime)\(\s*'([^']+)'/g; + +function parseColumns(block: string): string[] { + return Array.from(block.matchAll(COLUMN_REGEX), match => match[1]); +} + +function sorted(values: string[]): string[] { + return [...values].sort(); +} + +describe('database tables', () => { + it('reserves a dedicated migrations table name', () => { + expect(DB_MIGRATIONS_TABLE).toBe('incremental_ingestion__knex_migrations'); + // The migrations bookkeeping table must not collide with any of the + // ingestion data tables. + expect(MIGRATION_TABLES).not.toContain(DB_MIGRATIONS_TABLE); + }); + + it('creates exactly the three ingestion tables in the init migration', () => { + const created = Array.from( + initMigration.matchAll(/createTable\('([^']+)'/g), + match => match[1], + ); + expect(sorted(created)).toEqual(sorted(MIGRATION_TABLES)); + }); + + it.each(MIGRATION_TABLES)( + 'declares columns in the init migration for %s', + tableName => { + // Every column referenced by the migration is well-formed and unique. + const columns = parseColumns( + extractCreateTableBlock(initMigration, tableName), + ); + expect(new Set(columns).size).toEqual(columns.length); + }, + ); + + it('matches the ingestions columns against the IngestionRecord contract', () => { + const columns = parseColumns( + extractCreateTableBlock(initMigration, 'ingestions'), + ); + expect(sorted(columns)).toEqual([ + 'attempts', + 'completion_ticket', + 'created_at', + 'id', + 'ingestion_completed_at', + 'last_error', + 'next_action', + 'next_action_at', + 'provider_name', + 'rest_completed_at', + 'status', + ]); + + // The IngestionRecord interface must expose exactly those columns. + // Constructing a fully populated record and reflecting on its keys + // fails to compile whenever the interface and the expectation diverge. + const record: IngestionRecord = { + id: 'some-id', + provider_name: 'myProvider', + status: 'bursting', + next_action: 'ingest', + next_action_at: new Date(), + last_error: null, + attempts: 0, + created_at: '2023-01-01T00:00:00.000Z', + ingestion_completed_at: null, + rest_completed_at: null, + completion_ticket: 'open', + }; + expect(sorted(Object.keys(record))).toEqual(sorted(columns)); + + // IngestionUpsert covers the writable subset (everything but the + // generated created_at column). + const upsert: IngestionUpsert = { + id: record.id, + provider_name: record.provider_name, + status: record.status, + next_action: record.next_action, + next_action_at: record.next_action_at, + last_error: record.last_error, + attempts: record.attempts, + ingestion_completed_at: record.ingestion_completed_at, + rest_completed_at: record.rest_completed_at, + completion_ticket: record.completion_ticket, + }; + expect(sorted(Object.keys(upsert))).toEqual( + sorted(columns.filter(column => column !== 'created_at')), + ); + }); + + it('matches the ingestion_marks columns against the MarkRecord contract', () => { + const columns = parseColumns( + extractCreateTableBlock(initMigration, 'ingestion_marks'), + ); + expect(sorted(columns)).toEqual([ + 'created_at', + 'cursor', + 'id', + 'ingestion_id', + 'sequence', + ]); + + const record: MarkRecord = { + id: 'some-id', + ingestion_id: 'some-ingestion-id', + sequence: 1, + cursor: { page: 2 }, + created_at: '2023-01-01T00:00:00.000Z', + }; + expect(sorted(Object.keys(record))).toEqual(sorted(columns)); + + // Mark inserts carry the same data minus the generated created_at. + const insert: MarkRecordInsert = { + record: { + id: record.id, + ingestion_id: record.ingestion_id, + sequence: record.sequence, + cursor: record.cursor, + }, + }; + expect(sorted(Object.keys(insert.record))).toEqual( + sorted(columns.filter(column => column !== 'created_at')), + ); + }); + + it('matches the ingestion_mark_entities columns against its contract', () => { + const columns = parseColumns( + extractCreateTableBlock(initMigration, 'ingestion_mark_entities'), + ); + expect(sorted(columns)).toEqual(['id', 'ingestion_mark_id', 'ref']); + }); + + it('expands only the last_error column of the ingestions table', () => { + const upBlock = expandMigration.slice( + expandMigration.indexOf('exports.up'), + expandMigration.indexOf('exports.down'), + ); + const altered = Array.from( + upBlock.matchAll(/alterTable\('([^']+)'/g), + match => match[1], + ); + expect(altered).toEqual(['ingestions']); + expect(expandMigration).not.toMatch(/createTable\(/); + + const alteredColumns = parseColumns(upBlock); + expect(alteredColumns).toEqual(['last_error']); + expect(upBlock).toMatch(/table\.text\('last_error'\)\.alter\(\)/); + }); + + it('has the migrations applier bookkeep under the reserved table', () => { + expect(migrationsSource).toContain( + "import { DB_MIGRATIONS_TABLE } from './tables'", + ); + expect(migrationsSource).toContain('tableName: DB_MIGRATIONS_TABLE'); + }); + + it('has the manager reference only tables declared by the migrations', () => { + // Table references appear as tx('t'), tx('t'), join('t', ...) and + // purgeTable('t') calls in the manager source. + const referenced = new Set( + Array.from( + managerSource.matchAll( + /(?:purgeTable|tx|join)(?:<[^>]*>)?\(\s*'([a-zA-Z_]+)'/g, + ), + match => match[1], + ), + ); + + expect(sorted([...referenced])).toEqual(sorted(MIGRATION_TABLES)); + expect(referenced.has(DB_MIGRATIONS_TABLE)).toBe(false); + }); +}); diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/database/tables.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/database/tables.ts new file mode 100644 index 000000000..266318afb --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/database/tables.ts @@ -0,0 +1,123 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Type definitions for incremental ingestion database tables. + * Defines interfaces for ingestion records, marks, and related data structures. + */ + +export const DB_MIGRATIONS_TABLE = 'incremental_ingestion__knex_migrations'; + +/** + * The shape of data inserted into or updated in the `ingestions` table. + */ +export interface IngestionUpsert { + /** + * The ingestion record id. + */ + id?: string; + /** + * The next action the incremental entity provider will take. + */ + next_action: + | 'rest' + | 'ingest' + | 'backoff' + | 'cancel' + | 'nothing (done)' + | 'nothing (canceled)'; + /** + * Current status of the incremental entity provider. + */ + status: + | 'complete' + | 'bursting' + | 'resting' + | 'canceling' + | 'interstitial' + | 'backing off'; + /** + * The name of the incremental entity provider being updated. + */ + provider_name: string; + /** + * Date/time stamp for when the next action will trigger. + */ + next_action_at?: Date; + /** + * A record of the last error generated by the incremental entity provider. + */ + last_error?: string | null; + /** + * The number of attempts the provider has attempted during the current cycle. + */ + attempts?: number; + /** + * Date/time stamp for the completion of ingestion. + */ + ingestion_completed_at?: Date | string | null; + /** + * Date/time stamp for the end of the rest cycle before the next ingestion. + */ + rest_completed_at?: Date | string | null; + /** + * A record of the finalized status of the ingestion record. Values are either 'open' or a uuid. + */ + completion_ticket: string; +} + +/** + * This interface is for updating an existing ingestion record. + */ +export interface IngestionRecordUpdate { + ingestionId: string; + update: Partial; +} + +/** + * The expected response from the `ingestion_marks` table. + */ +export interface MarkRecord { + id: string; + sequence: number; + ingestion_id: string; + cursor: unknown; + created_at: string; +} + +/** + * The expected response from the `ingestions` table. + */ +export interface IngestionRecord extends IngestionUpsert { + id: string; + next_action_at: Date; + /** + * The date/time the ingestion record was created. + */ + created_at: string; +} + +/** + * This interface supplies all the values for adding an ingestion mark. + */ +export interface MarkRecordInsert { + record: { + id: string; + ingestion_id: string; + cursor: unknown; + sequence: number; + }; +} diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/engine/OpenChoreoIncrementalIngestionEngine.test.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/engine/OpenChoreoIncrementalIngestionEngine.test.ts new file mode 100644 index 000000000..0fc60a790 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/engine/OpenChoreoIncrementalIngestionEngine.test.ts @@ -0,0 +1,397 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Test suite for OpenChoreoIncrementalIngestionEngine. + * Verifies state-machine transitions (rest, ingest, backoff, cancel), + * burst pacing, mark()/delta semantics including removal thresholds, and + * event handling, all against mocked database manager, provider and + * catalog connection. + */ +import { mockServices } from '@backstage/backend-test-utils'; +import type { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import type { DeferredEntity } from '@backstage/plugin-catalog-node'; +import type { EventParams } from '@backstage/plugin-events-node'; +import type { Duration } from 'luxon'; +import type { OpenChoreoIncrementalIngestionDatabaseManager } from '../database/OpenChoreoIncrementalIngestionDatabaseManager'; +import type { IterationEngineOptions } from '../types'; +import { OpenChoreoIncrementalIngestionEngine } from './OpenChoreoIncrementalIngestionEngine'; + +function createMockManager() { + return { + getCurrentIngestionRecord: jest.fn(), + createProviderIngestionRecord: jest.fn(), + setProviderComplete: jest.fn(), + clearFinishedIngestions: jest.fn(), + setProviderBursting: jest.fn(), + setProviderResting: jest.fn(), + setProviderInterstitial: jest.fn(), + setProviderCanceling: jest.fn(), + setProviderCanceled: jest.fn(), + setProviderBackoff: jest.fn(), + setProviderIngesting: jest.fn(), + getLastMark: jest.fn(), + getFirstMark: jest.fn(), + createMark: jest.fn(), + createMarkEntities: jest.fn(), + getEntityCountsByKind: jest.fn(), + computeRemoved: jest.fn(), + updateIngestionRecordById: jest.fn(), + deleteEntityRecordsByRef: jest.fn(), + }; +} + +function createMockProvider() { + return { + getProviderName: jest.fn(() => 'test-provider'), + next: jest.fn(), + around: jest.fn(async (burst: (context: unknown) => Promise) => { + await burst({ marker: 'context' }); + }), + eventHandler: undefined as + | { onEvent: jest.Mock; supportsEventTopics: () => string[] } + | undefined, + }; +} + +/** A row of the `ingestions` table as the engine reads it. */ +function makeRecord(overrides: Record = {}) { + return { + id: 'ing-1', + provider_name: 'test-provider', + status: 'bursting', + next_action: 'ingest', + attempts: 0, + next_action_at: new Date(Date.now() + 60_000), + ...overrides, + }; +} + +function makeDeferred(kind: string, name: string): DeferredEntity { + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind, + metadata: { namespace: 'default', name }, + }, + }; +} + +/** + * The engine normalizes deferred entities on their way into a delta, + * guaranteeing a (possibly empty) annotations object on each entity. + */ +function asApplied(deferred: DeferredEntity): DeferredEntity { + return { + ...deferred, + entity: { + ...deferred.entity, + metadata: { ...deferred.entity.metadata, annotations: {} }, + }, + }; +} + +function createHarness(overrides: Partial = {}) { + const manager = createMockManager(); + const provider = createMockProvider(); + const connection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const options: IterationEngineOptions = { + logger: mockServices.logger.mock(), + connection, + manager: + manager as unknown as OpenChoreoIncrementalIngestionDatabaseManager, + provider: provider as unknown as IterationEngineOptions['provider'], + restLength: { minutes: 1 }, + burstLength: { seconds: 30 }, + ready: Promise.resolve(), + backoff: [{ minutes: 1 }], + ...overrides, + }; + const engine = new OpenChoreoIncrementalIngestionEngine(options); + return { engine, manager, provider, connection }; +} + +describe('OpenChoreoIncrementalIngestionEngine', () => { + const signal = () => new AbortController().signal; + + it('starts a new ingestion cycle when the rest period is past due', async () => { + const { engine, manager, provider } = createHarness(); + manager.getCurrentIngestionRecord.mockResolvedValue( + makeRecord({ + status: 'resting', + next_action: 'rest', + next_action_at: new Date(Date.now() - 60_000), + }), + ); + + await engine.handleNextAction(signal()); + + expect(manager.setProviderComplete).toHaveBeenCalledWith('ing-1'); + expect(manager.clearFinishedIngestions).toHaveBeenCalledWith( + 'test-provider', + ); + expect(provider.next).not.toHaveBeenCalled(); + }); + + it('keeps resting while the rest period is not due', async () => { + const { engine, manager } = createHarness(); + manager.getCurrentIngestionRecord.mockResolvedValue( + makeRecord({ + status: 'resting', + next_action: 'rest', + next_action_at: new Date(Date.now() + 60_000), + }), + ); + + await engine.handleNextAction(signal()); + + expect(manager.setProviderComplete).not.toHaveBeenCalled(); + expect(manager.clearFinishedIngestions).not.toHaveBeenCalled(); + }); + + it('transitions to resting when the burst finishes with done', async () => { + const { engine, manager, provider } = createHarness(); + manager.getCurrentIngestionRecord.mockResolvedValue( + makeRecord({ next_action: 'ingest' }), + ); + manager.getLastMark.mockResolvedValue(undefined); + manager.getEntityCountsByKind.mockResolvedValue({ total: 0 }); + manager.computeRemoved.mockResolvedValue({ total: 0, removed: [] }); + provider.next.mockResolvedValue({ done: true }); + + await engine.handleNextAction(signal()); + + expect(manager.setProviderBursting).toHaveBeenCalledWith('ing-1'); + expect(manager.setProviderResting).toHaveBeenCalledTimes(1); + expect(manager.setProviderResting.mock.calls[0][0]).toBe('ing-1'); + expect( + (manager.setProviderResting.mock.calls[0][1] as Duration).as( + 'milliseconds', + ), + ).toBe(60_000); + expect(manager.setProviderInterstitial).not.toHaveBeenCalled(); + }); + + it('transitions to interstitial when the burst ends unfinished', async () => { + const { engine, manager, provider } = createHarness(); + manager.getCurrentIngestionRecord.mockResolvedValue( + makeRecord({ next_action: 'ingest' }), + ); + manager.getLastMark.mockResolvedValue(undefined); + const controller = new AbortController(); + provider.next.mockImplementationOnce(async () => { + controller.abort(); + return { + done: false, + entities: [makeDeferred('Component', 'svc-1')], + cursor: { page: 1 }, + }; + }); + + await engine.handleNextAction(controller.signal); + + expect(provider.next).toHaveBeenCalledTimes(1); + expect(manager.createMark).toHaveBeenCalledTimes(1); + expect(manager.setProviderInterstitial).toHaveBeenCalledWith('ing-1'); + expect(manager.setProviderResting).not.toHaveBeenCalled(); + }); + + it('cuts a never-done burst short once burstLength elapses', async () => { + const { engine, manager, provider } = createHarness({ + burstLength: { milliseconds: 1 }, + }); + manager.getCurrentIngestionRecord.mockResolvedValue( + makeRecord({ next_action: 'ingest' }), + ); + manager.getLastMark.mockResolvedValue(undefined); + provider.next.mockImplementation(async () => { + await new Promise(resolve => setTimeout(resolve, 5)); + return { done: false, entities: [], cursor: { page: 'more' } }; + }); + + await engine.handleNextAction(signal()); + + // The provider never finishes, so only the burst length can have + // stopped it — after exactly one batch. + expect(provider.next).toHaveBeenCalledTimes(1); + expect(manager.createMark).toHaveBeenCalledTimes(1); + expect(manager.setProviderInterstitial).toHaveBeenCalledWith('ing-1'); + expect(manager.setProviderResting).not.toHaveBeenCalled(); + }); + + it('backs off when the provider throws during a burst', async () => { + const { engine, manager, provider } = createHarness(); + manager.getCurrentIngestionRecord.mockResolvedValue( + makeRecord({ next_action: 'ingest' }), + ); + manager.getLastMark.mockResolvedValue(undefined); + manager.getEntityCountsByKind.mockResolvedValue({ total: 0 }); + provider.next.mockRejectedValue(new Error('boom')); + + await engine.handleNextAction(signal()); + + expect(manager.setProviderBackoff).toHaveBeenCalledTimes(1); + expect(manager.setProviderBackoff.mock.calls[0][0]).toBe('ing-1'); + expect(manager.setProviderBackoff.mock.calls[0][1]).toBe(0); + expect((manager.setProviderBackoff.mock.calls[0][2] as Error).message).toBe( + 'boom', + ); + expect(manager.setProviderBackoff.mock.calls[0][3]).toBe(60_000); + expect(manager.createMark).not.toHaveBeenCalled(); + expect(manager.setProviderInterstitial).not.toHaveBeenCalled(); + expect(manager.setProviderResting).not.toHaveBeenCalled(); + }); + + it("cancels on a 'CANCEL' error instead of backing off", async () => { + const { engine, manager, provider } = createHarness(); + manager.getCurrentIngestionRecord.mockResolvedValue( + makeRecord({ next_action: 'ingest' }), + ); + manager.getLastMark.mockResolvedValue(undefined); + provider.next.mockRejectedValue(new Error('CANCEL')); + + await engine.handleNextAction(signal()); + + expect(manager.setProviderCanceling).toHaveBeenCalledWith( + 'ing-1', + 'CANCEL', + ); + expect(manager.setProviderBackoff).not.toHaveBeenCalled(); + }); + + it('applies a dependency-sorted delta with removed refs on the final mark', async () => { + const { engine, manager, connection } = createHarness(); + manager.getEntityCountsByKind.mockResolvedValue({ + total: 2, + Domain: 1, + Component: 1, + }); + manager.computeRemoved.mockResolvedValue({ + total: 2, + removed: [{ entityRef: 'component:default/gone' }], + }); + + const domainEntity = makeDeferred('Domain', 'ns-1'); + const componentEntity = makeDeferred('Component', 'svc-1'); + await engine.mark({ + id: 'ing-1', + sequence: 3, + entities: [componentEntity, domainEntity], + done: true, + cursor: { phase: 'components' }, + }); + + expect(manager.createMark).toHaveBeenCalledWith({ + record: { + id: expect.any(String), + ingestion_id: 'ing-1', + cursor: { phase: 'components' }, + sequence: 3, + }, + }); + expect(manager.createMarkEntities).toHaveBeenCalledTimes(1); + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [asApplied(domainEntity), asApplied(componentEntity)], + removed: [{ entityRef: 'component:default/gone' }], + }); + }); + + it('blocks removals above the configured percentage and records last_error', async () => { + const { engine, manager, connection } = createHarness({ + rejectRemovalsAbovePercentage: 10, + }); + manager.getEntityCountsByKind.mockResolvedValue({ + total: 10, + Component: 10, + }); + manager.computeRemoved.mockResolvedValue({ + total: 10, + removed: [ + { entityRef: 'component:default/a' }, + { entityRef: 'component:default/b' }, + ], + }); + + const added = makeDeferred('Component', 'svc-1'); + await engine.mark({ + id: 'ing-1', + sequence: 0, + entities: [added], + done: true, + }); + + // 2 of 10 entities (20%) would be removed, above the 10% threshold. + expect(manager.updateIngestionRecordById).toHaveBeenCalledWith({ + ingestionId: 'ing-1', + update: { last_error: expect.stringContaining('REMOVAL_THRESHOLD') }, + }); + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [asApplied(added)], + removed: [], + }); + }); + + it('ignores events whose topic is not supported', async () => { + const { engine, provider, connection } = createHarness(); + const onEvent = jest.fn(); + provider.eventHandler = { + onEvent, + supportsEventTopics: () => ['openchoreo.test'], + }; + + await engine.onEvent({ topic: 'other.topic' } as EventParams); + + expect(onEvent).not.toHaveBeenCalled(); + expect(connection.applyMutation).not.toHaveBeenCalled(); + }); + + it('persists and applies delta results from supported events', async () => { + const { engine, manager, provider, connection } = createHarness(); + const delta = { + type: 'delta' as const, + added: [makeDeferred('Component', 'evt-1')], + removed: [{ entityRef: 'component:default/gone' }], + }; + provider.eventHandler = { + onEvent: jest.fn().mockResolvedValue(delta), + supportsEventTopics: () => ['openchoreo.test'], + }; + manager.getCurrentIngestionRecord.mockResolvedValue( + makeRecord({ status: 'bursting' }), + ); + manager.getFirstMark.mockResolvedValue({ + id: 'mark-1', + sequence: 0, + cursor: null, + }); + + await engine.onEvent({ topic: 'openchoreo.test' } as EventParams); + + expect(manager.createMarkEntities).toHaveBeenCalledWith( + 'mark-1', + delta.added, + ); + expect(manager.deleteEntityRecordsByRef).toHaveBeenCalledWith( + delta.removed, + ); + expect(connection.applyMutation).toHaveBeenCalledWith(delta); + }); +}); diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/engine/OpenChoreoIncrementalIngestionEngine.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/engine/OpenChoreoIncrementalIngestionEngine.ts new file mode 100644 index 000000000..3bcacded7 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/engine/OpenChoreoIncrementalIngestionEngine.ts @@ -0,0 +1,564 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This class implements the incremental ingestion engine for OpenChoreo. + * It manages burst-based processing of entities using cursor-based pagination + * to ensure efficient memory usage and resumable ingestion for large datasets. + * Key features include state management, error handling with backoff, and event-driven updates. + */ + +import type { DeferredEntity } from '@backstage/plugin-catalog-node'; +import { Gauge, metrics } from '@opentelemetry/api'; +import { IterationEngine, IterationEngineOptions } from '../types'; +import { OpenChoreoIncrementalIngestionDatabaseManager } from '../database/OpenChoreoIncrementalIngestionDatabaseManager'; +import { performance } from 'perf_hooks'; +import { Duration } from 'luxon'; +import { v4 } from 'uuid'; +import { stringifyError } from '@backstage/errors'; +import type { EventParams } from '@backstage/plugin-events-node'; +import type { HumanDuration } from '@backstage/types'; + +const ERROR_MESSAGE_MAX_LENGTH = 700; +const MILLISECONDS_TO_SECONDS_DIVISOR = 1000; + +export class OpenChoreoIncrementalIngestionEngine implements IterationEngine { + private readonly restLength: Duration; + private readonly burstLength: Duration; + private readonly backoff: HumanDuration[]; + private readonly lastStarted: Gauge; + private readonly lastCompleted: Gauge; + + private manager: OpenChoreoIncrementalIngestionDatabaseManager; + + constructor(private options: IterationEngineOptions) { + const meter = metrics.getMeter('default'); + + this.manager = options.manager; + this.restLength = Duration.fromObject(options.restLength); + this.burstLength = Duration.fromObject(options.burstLength); + this.backoff = options.backoff ?? [ + { minutes: 1 }, + { minutes: 5 }, + { minutes: 30 }, + { hours: 3 }, + ]; + + this.lastStarted = meter.createGauge( + 'catalog_incremental.ingestions.started', + { + description: + 'Epoch timestamp seconds when the ingestion was last started', + unit: 'seconds', + }, + ); + this.lastCompleted = meter.createGauge( + 'catalog_incremental.ingestions.completed', + { + description: + 'Epoch timestamp seconds when the ingestion was last completed', + unit: 'seconds', + }, + ); + } + + async taskFn(signal: AbortSignal) { + try { + this.options.logger.debug('Begin tick'); + await this.handleNextAction(signal); + } catch (error) { + this.options.logger.error(`${error}`); + throw error; + } finally { + this.options.logger.debug('End tick'); + } + } + + async handleNextAction(signal: AbortSignal) { + await this.options.ready; + + const result = await this.getCurrentAction(); + if (result) { + const { ingestionId, nextActionAt, nextAction, attempts } = result; + + switch (nextAction) { + case 'rest': + if (Date.now() > nextActionAt) { + this.options.logger.info( + `incremental-engine: Ingestion ${ingestionId} rest period complete. Starting new ingestion`, + ); + + await this.manager.setProviderComplete(ingestionId); + await this.manager.clearFinishedIngestions( + this.options.provider.getProviderName(), + ); + + this.lastStarted.record( + Date.now() / MILLISECONDS_TO_SECONDS_DIVISOR, + { + providerName: this.options.provider.getProviderName(), + }, + ); + } else { + this.options.logger.debug( + `incremental-engine: Ingestion '${ingestionId}' rest period continuing`, + ); + } + break; + case 'ingest': + try { + await this.manager.setProviderBursting(ingestionId); + const done = await this.ingestOneBurst(ingestionId, signal); + if (done) { + this.options.logger.info( + `incremental-engine: Ingestion '${ingestionId}' complete, transitioning to rest period of ${this.restLength.toHuman()}`, + ); + this.lastCompleted.record( + Date.now() / MILLISECONDS_TO_SECONDS_DIVISOR, + { + providerName: this.options.provider.getProviderName(), + status: 'completed', + }, + ); + await this.manager.setProviderResting( + ingestionId, + this.restLength, + ); + } else { + await this.manager.setProviderInterstitial(ingestionId); + this.options.logger.debug( + `incremental-engine: Ingestion '${ingestionId}' continuing`, + ); + } + } catch (error) { + if ( + (error as Error).message && + (error as Error).message === 'CANCEL' + ) { + this.options.logger.info( + `incremental-engine: Ingestion '${ingestionId}' canceled`, + ); + await this.manager.setProviderCanceling( + ingestionId, + (error as Error).message, + ); + } else { + const currentBackoff = Duration.fromObject( + this.backoff[Math.min(this.backoff.length - 1, attempts)], + ); + + const backoffLength = currentBackoff.as('milliseconds'); + this.options.logger.error( + `incremental-engine: Ingestion '${ingestionId}' failed`, + error as Error, + ); + + // Log partial progress before backing off + try { + const entityCounts = await this.manager.getEntityCountsByKind( + ingestionId, + ); + + // Build dynamic summary of entity types + const entityEntries = Object.entries(entityCounts) + .filter(([key]) => key !== 'total') + .sort(([, a], [, b]) => b - a) // Sort by count descending + .slice(0, 10); // Limit to top 10 + + const entityTypesSummary = entityEntries + .map(([kind, count]) => { + // Proper pluralization: avoid double 's' for kinds already ending in 's' + const plural = kind.endsWith('s') ? kind : `${kind}s`; + return `${count} ${plural}`; + }) + .join(', '); + + const totalTypes = Object.keys(entityCounts).length - 1; // minus 'total' + const truncated = totalTypes > 10; + + const message = `incremental-engine: Ingestion '${ingestionId}': Partial progress before failure - ${ + entityCounts.total + } entities ingested so far (${entityTypesSummary}${ + truncated ? ` +${totalTypes - 10} more types` : '' + })`; + + this.options.logger.info(message); + } catch (countError) { + this.options.logger.debug( + `incremental-engine: Ingestion '${ingestionId}': Could not retrieve partial entity counts: ${ + (countError as Error).message + }`, + ); + } + + const truncatedError = stringifyError(error).substring( + 0, + ERROR_MESSAGE_MAX_LENGTH, + ); + this.options.logger.error( + `incremental-engine: Ingestion '${ingestionId}' threw an error during ingestion burst. Ingestion will backoff for ${currentBackoff.toHuman()} (${truncatedError})`, + ); + this.lastCompleted.record( + Date.now() / MILLISECONDS_TO_SECONDS_DIVISOR, + { + providerName: this.options.provider.getProviderName(), + status: 'failed', + }, + ); + + await this.manager.setProviderBackoff( + ingestionId, + attempts, + error as Error, + backoffLength, + ); + } + } + break; + case 'backoff': + if (Date.now() > nextActionAt) { + this.options.logger.info( + `incremental-engine: Ingestion '${ingestionId}' backoff complete, will attempt to resume`, + ); + await this.manager.setProviderIngesting(ingestionId); + } else { + this.options.logger.debug( + `incremental-engine: Ingestion '${ingestionId}' backoff continuing`, + ); + } + break; + case 'cancel': + this.options.logger.info( + `incremental-engine: Ingestion '${ingestionId}' canceling, will restart`, + ); + await this.manager.setProviderCanceled(ingestionId); + break; + default: + this.options.logger.error( + `incremental-engine: Ingestion '${ingestionId}' received unknown action '${nextAction}'`, + ); + } + } else { + this.options.logger.error( + `incremental-engine: Engine tried to create duplicate ingestion record for provider '${this.options.provider.getProviderName()}'.`, + ); + } + } + + async getCurrentAction() { + const providerName = this.options.provider.getProviderName(); + const record = await this.manager.getCurrentIngestionRecord(providerName); + if (record) { + this.options.logger.debug( + `incremental-engine: Ingestion record found: '${record.id}'`, + ); + return { + ingestionId: record.id, + nextAction: record.next_action as 'rest' | 'ingest' | 'backoff', + attempts: record.attempts as number, + nextActionAt: record.next_action_at.valueOf() as number, + }; + } + const result = await this.manager.createProviderIngestionRecord( + providerName, + ); + if (result) { + this.options.logger.info( + `incremental-engine: Ingestion record created: '${result.ingestionId}'`, + ); + } + return result; + } + + async ingestOneBurst(id: string, signal: AbortSignal) { + const lastMark = await this.manager.getLastMark(id); + + const cursor = lastMark ? lastMark.cursor : undefined; + let sequence = lastMark ? lastMark.sequence + 1 : 0; + + const start = performance.now(); + let count = 0; + let done = false; + this.options.logger.info( + `incremental-engine: Ingestion '${id}' burst initiated`, + ); + + await this.options.provider.around(async (context: unknown) => { + let next = await this.options.provider.next(context, cursor); + count++; + for (;;) { + done = next.done; + await this.mark({ + id, + sequence, + entities: next?.entities, + done: next.done, + cursor: next?.cursor, + }); + if (signal.aborted || next.done) { + break; + } else if ( + performance.now() - start > + this.burstLength.as('milliseconds') + ) { + this.options.logger.info( + `incremental-engine: Ingestion '${id}' burst ending after ${this.burstLength.toHuman()}.`, + ); + break; + } else { + next = await this.options.provider.next(context, next.cursor); + count++; + sequence++; + } + } + }); + + this.options.logger.info( + `incremental-engine: Ingestion '${id}' burst complete. (${count} batches in ${Math.round( + performance.now() - start, + )}ms).`, + ); + return done; + } + + async mark(options: { + id: string; + sequence: number; + entities?: DeferredEntity[]; + done: boolean; + cursor?: unknown; + }) { + const { id, sequence, entities, done, cursor } = options; + this.options.logger.debug( + `incremental-engine: Ingestion '${id}': MARK ${ + entities ? entities.length : 0 + } entities, cursor: ${ + cursor ? JSON.stringify(cursor) : 'none' + }, done: ${done}`, + ); + const markId = v4(); + + await this.manager.createMark({ + record: { + id: markId, + ingestion_id: id, + cursor, + sequence, + }, + }); + + if (entities && entities.length > 0) { + await this.manager.createMarkEntities(markId, entities); + } + + const added = + entities?.map(deferred => ({ + ...deferred, + entity: { + ...deferred.entity, + metadata: { + ...deferred.entity.metadata, + annotations: { + ...deferred.entity.metadata.annotations, + }, + }, + }, + })) ?? []; + + const sortedAdded = this.sortEntitiesByDependencyOrder(added); + + const removed: { entityRef: string }[] = []; + + if (done) { + this.options.logger.info( + `incremental-engine: Ingestion '${id}': Final page reached, calculating removed entities`, + ); + + try { + const entityCounts = await this.manager.getEntityCountsByKind(id); + + // Build dynamic summary of entity types + const entityEntries = Object.entries(entityCounts) + .filter(([key]) => key !== 'total') + .sort(([, a], [, b]) => b - a) // Sort by count descending + .slice(0, 10); // Limit to top 10 + + const entityTypesSummary = entityEntries + .map(([kind, count]) => { + // Proper pluralization: avoid double 's' for kinds already ending in 's' + const plural = kind.endsWith('s') ? kind : `${kind}s`; + return `${count} ${plural}`; + }) + .join(', '); + + const totalTypes = Object.keys(entityCounts).length - 1; // minus 'total' + const truncated = totalTypes > 10; + + const message = `incremental-engine: Ingestion '${id}': Successfully processed ${ + entityCounts.total + } entities (${entityTypesSummary}${ + truncated ? ` +${totalTypes - 10} more types` : '' + })`; + + this.options.logger.info(message); + } catch (error) { + const errorMessage = error as Error; + this.options.logger.warn( + `incremental-engine: Ingestion '${id}': Could not calculate entity counts: ${errorMessage.message} (Type: ${errorMessage.constructor.name})`, + { + ingestionId: id, + errorType: errorMessage.constructor.name, + errorMessage: errorMessage.message, + stack: errorMessage.stack?.substring(0, 1000), // Truncate stack for logging + }, + ); + } + + const result = await this.manager.computeRemoved( + this.options.provider.getProviderName(), + id, + ); + + const { total } = result; + + let doRemoval = true; + if (this.options.rejectEmptySourceCollections) { + if (total === 0) { + this.options.logger.error( + `incremental-engine: Ingestion '${id}': Rejecting empty entity collection!`, + ); + doRemoval = false; + } + } + + if (this.options.rejectRemovalsAbovePercentage) { + // If the total entities upserted in this ingestion is 0, then + // 100% of entities are stale and marked for removal. + const percentRemoved = + total > 0 ? (result.removed.length / total) * 100 : 100; + if (percentRemoved <= this.options.rejectRemovalsAbovePercentage) { + this.options.logger.info( + `incremental-engine: Ingestion '${id}': Removing ${result.removed.length} entities that have no matching assets`, + ); + } else { + const notice = `Attempted to remove ${percentRemoved}% of matching entities!`; + this.options.logger.error( + `incremental-engine: Ingestion '${id}': ${notice}`, + ); + await this.manager.updateIngestionRecordById({ + ingestionId: id, + update: { + last_error: `REMOVAL_THRESHOLD exceeded on ingestion mark ${markId}: ${notice}`, + }, + }); + doRemoval = false; + } + } + if (doRemoval) { + for (const entityRef of result.removed) { + removed.push(entityRef); + } + } + } + + await this.options.connection.applyMutation({ + type: 'delta', + added: sortedAdded, + removed, + }); + } + + private sortEntitiesByDependencyOrder( + entities: DeferredEntity[], + ): DeferredEntity[] { + const kindOrder = new Map([ + ['Domain', 0], + ['System', 1], + ['Component', 2], + ['API', 3], + ]); + + return entities.slice().sort((a, b) => { + const orderA = kindOrder.get(a.entity.kind) ?? 999; + const orderB = kindOrder.get(b.entity.kind) ?? 999; + return orderA - orderB; + }); + } + + async onEvent(params: EventParams): Promise { + const { topic } = params; + if (!this.supportsEventTopics().includes(topic)) { + return; + } + + const { logger, provider, connection } = this.options; + const providerName = provider.getProviderName(); + logger.debug(`incremental-engine: ${providerName} received ${topic} event`); + + if (!provider.eventHandler) { + return; + } + + const result = await provider.eventHandler.onEvent(params); + + if (result.type === 'delta') { + if (result.added.length > 0) { + const ingestionRecord = await this.manager.getCurrentIngestionRecord( + providerName, + ); + + if (!ingestionRecord) { + logger.debug( + `incremental-engine: ${providerName} skipping delta addition because incremental ingestion is restarting.`, + ); + } else { + const mark = + ingestionRecord.status === 'resting' + ? await this.manager.getLastMark(ingestionRecord.id) + : await this.manager.getFirstMark(ingestionRecord.id); + + if (!mark) { + throw new Error( + `Cannot apply delta, page records are missing! Please re-run incremental ingestion for ${providerName}.`, + ); + } + await this.manager.createMarkEntities(mark.id, result.added); + } + } + + if (result.removed.length > 0) { + await this.manager.deleteEntityRecordsByRef(result.removed); + } + + await connection.applyMutation(result); + logger.debug( + `incremental-engine: ${providerName} processed delta from '${topic}' event`, + ); + } else { + logger.debug( + `incremental-engine: ${providerName} ignored event from topic '${topic}'`, + ); + } + } + + supportsEventTopics(): string[] { + const { provider } = this.options; + const topics = provider.eventHandler + ? provider.eventHandler.supportsEventTopics() + : []; + return topics; + } +} diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/index.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/index.ts new file mode 100644 index 000000000..7c4dd0b6a --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/index.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Provides efficient incremental ingestion of entities into the catalog for OpenChoreo. + * + * This module enables scalable entity processing using cursor-based pagination, + * burst-based ingestion cycles, and resumable state management to handle large + * datasets without memory constraints. It supports event-driven updates and + * automatic cleanup of stale entities. + * + * @packageDocumentation + */ + +export { catalogModuleOpenchoreoIncremental as default } from './module'; +export { catalogModuleOpenchoreoIncremental } from './module'; +export { catalogModuleOpenchoreoIncrementalProvider } from './module/index'; +export { + catalogModuleOpenchoreoImmediateCatalogIncremental, + openchoreoImmediateCatalogIncrementalServiceFactory, +} from './openchoreoImmediateCatalogIncremental'; +export { + openchoreoIncrementalProvidersExtensionPoint, + type OpenChoreoIncrementalProviderExtensionPoint, +} from './module/index'; +export { + type EntityIteratorResult, + type IncrementalEntityEventResult, + type IncrementalEntityProvider, + type IncrementalEntityProviderOptions, +} from './types'; diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/module.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/module.ts new file mode 100644 index 000000000..fca7f8547 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/module.ts @@ -0,0 +1,42 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Module definition for OpenChoreo incremental ingestion. + * Exports the main catalog module for incremental entity processing. + */ + +import { createBackendFeatureLoader } from '@backstage/backend-plugin-api'; +import catalogModuleOpenchoreoIncrementalEntityProvider, { + catalogModuleOpenchoreoIncrementalProvider, +} from './module/index'; + +/** + * The full incremental ingestion composition, installed with a single + * `backend.add(import(...))`. + * + * Yields the wrapper module (extension point + admin router + provider + * wrapping) first — it registers the `openchoreoIncrementalProvidersExtensionPoint` + * the provider module consumes — then the OpenChoreo provider module that + * registers `OpenChoreoIncrementalEntityProvider` through it. Both are inert + * unless `openchoreo.features.incrementalIngestion.enabled` is true. + */ +export const catalogModuleOpenchoreoIncremental = createBackendFeatureLoader({ + *loader() { + yield catalogModuleOpenchoreoIncrementalEntityProvider; + yield catalogModuleOpenchoreoIncrementalProvider; + }, +}); diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/module/WrapperProviders.test.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/module/WrapperProviders.test.ts new file mode 100644 index 000000000..7d8420959 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/module/WrapperProviders.test.ts @@ -0,0 +1,304 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Test suite for WrapperProviders. + * Verifies provider wrapping, the shared one-shot migrations gate, scheduler + * task registration, event subscription for topic-supporting providers, and + * the ready signal semantics. The migrations applier is injected as a + * counting fake over a dummy knex object, so no real database is needed. + */ +import type { SchedulerService } from '@backstage/backend-plugin-api'; +import { mockServices } from '@backstage/backend-test-utils'; +import type { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import type { EventsService } from '@backstage/plugin-events-node'; +import type { Knex } from 'knex'; +import type { + IncrementalEntityProvider, + IncrementalEntityProviderOptions, +} from '../types'; +import { WrapperProviders } from './WrapperProviders'; + +// WrapperProviders caches its shared migrations promise at module scope, so +// an isolated module registry is used per test to keep the "runs exactly +// once" assertions deterministic regardless of test ordering. +function freshWrapperProviders(): typeof WrapperProviders { + let fresh: typeof WrapperProviders | undefined; + jest.isolateModules(() => { + fresh = require('./WrapperProviders').WrapperProviders; + }); + return fresh!; +} + +/** + * The only thing read off the client before connect completes is the knex + * dialect name, used by the database manager for batch sizing. + */ +const dummyClient = { + client: { config: { client: 'sqlite3' } }, +} as unknown as Knex; + +function makeProvider( + name: string, + eventHandler?: IncrementalEntityProvider['eventHandler'], +): IncrementalEntityProvider { + return { + getProviderName: () => name, + around: async burst => { + await burst(undefined); + }, + next: async () => ({ done: true }), + ...(eventHandler ? { eventHandler } : {}), + }; +} + +const providerOptions: IncrementalEntityProviderOptions = { + burstInterval: { seconds: 30 }, + burstLength: { seconds: 30 }, + restLength: { minutes: 30 }, +}; + +function makeHarness() { + const scheduler = { scheduleTask: jest.fn() }; + const events = { subscribe: jest.fn() }; + return { scheduler, events }; +} + +describe('WrapperProviders', () => { + it('wrap() returns an entity provider named after the wrapped provider', async () => { + const FreshWrapperProviders = freshWrapperProviders(); + const { scheduler, events } = makeHarness(); + + const providers = new FreshWrapperProviders({ + config: mockServices.rootConfig.mock(), + logger: mockServices.logger.mock(), + client: dummyClient, + scheduler: scheduler as unknown as SchedulerService, + events: events as unknown as EventsService, + }); + + const wrapped = providers.wrap(makeProvider('provider-x'), providerOptions); + + expect(typeof wrapped.getProviderName).toBe('function'); + expect(typeof wrapped.connect).toBe('function'); + expect(wrapped.getProviderName()).toBe('provider-x'); + }); + + it('connect() awaits the shared migrations before scheduling the task', async () => { + const FreshWrapperProviders = freshWrapperProviders(); + const { scheduler, events } = makeHarness(); + + const applyDatabaseMigrations = jest.fn(); + let releaseMigrations!: () => void; + const migrationsGate = new Promise(resolve => { + releaseMigrations = resolve; + }); + applyDatabaseMigrations.mockImplementation(async () => { + await migrationsGate; + }); + + const providers = new FreshWrapperProviders({ + config: mockServices.rootConfig.mock(), + logger: mockServices.logger.mock(), + client: dummyClient, + scheduler: scheduler as unknown as SchedulerService, + applyDatabaseMigrations, + events: events as unknown as EventsService, + }); + + const connection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const connectPromise = providers + .wrap(makeProvider('provider-1'), providerOptions) + .connect(connection); + await new Promise(resolve => setImmediate(resolve)); + + // The migrations gate is still closed: the applier ran once with the + // shared client, but nothing downstream has happened yet. + expect(applyDatabaseMigrations).toHaveBeenCalledTimes(1); + expect(applyDatabaseMigrations).toHaveBeenCalledWith(dummyClient); + expect(scheduler.scheduleTask).not.toHaveBeenCalled(); + expect(events.subscribe).not.toHaveBeenCalled(); + + releaseMigrations(); + await connectPromise; + + expect(scheduler.scheduleTask).toHaveBeenCalledTimes(1); + const task = (scheduler.scheduleTask as jest.Mock).mock.calls[0][0]; + expect(task.id).toBe('provider-1'); + expect(task.fn).toEqual(expect.any(Function)); + expect(task.frequency.as('milliseconds')).toBe(30_000); + // The scheduled timeout includes the burst length plus a safety margin. + expect(task.timeout.as('milliseconds')).toBe(90_000); + + // The provider reports no event topics, so no subscription happens. + expect(events.subscribe).not.toHaveBeenCalled(); + }); + + it('clamps too-short burst intervals up to the scheduler minimum', async () => { + const FreshWrapperProviders = freshWrapperProviders(); + const { scheduler, events } = makeHarness(); + + const providers = new FreshWrapperProviders({ + config: mockServices.rootConfig.mock(), + logger: mockServices.logger.mock(), + client: dummyClient, + scheduler: scheduler as unknown as SchedulerService, + applyDatabaseMigrations: jest.fn(async () => {}), + events: events as unknown as EventsService, + }); + + await providers + .wrap( + makeProvider('provider-fast'), + // 1 second is below the 5 second scheduler minimum. + { + ...providerOptions, + burstInterval: { seconds: 1 }, + burstLength: { seconds: 1 }, + }, + ) + .connect({ applyMutation: jest.fn(), refresh: jest.fn() }); + + const task = (scheduler.scheduleTask as jest.Mock).mock.calls[0][0]; + expect(task.frequency.as('milliseconds')).toBe(5_000); + expect(task.timeout.as('milliseconds')).toBe(61_000); + }); + + it('connects two providers in parallel while running the migrations exactly once', async () => { + const FreshWrapperProviders = freshWrapperProviders(); + const { scheduler, events } = makeHarness(); + + const applyDatabaseMigrations = jest.fn(); + let releaseMigrations!: () => void; + const migrationsGate = new Promise(resolve => { + releaseMigrations = resolve; + }); + applyDatabaseMigrations.mockImplementation(async () => { + await migrationsGate; + }); + + const providers = new FreshWrapperProviders({ + config: mockServices.rootConfig.mock(), + logger: mockServices.logger.mock(), + client: dummyClient, + scheduler: scheduler as unknown as SchedulerService, + applyDatabaseMigrations, + events: events as unknown as EventsService, + }); + + const connection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const connectA = providers + .wrap(makeProvider('provider-a'), providerOptions) + .connect(connection); + const connectB = providers + .wrap(makeProvider('provider-b'), providerOptions) + .connect(connection); + await new Promise(resolve => setImmediate(resolve)); + + // Both providers are parked on the same gated migration run. + expect(applyDatabaseMigrations).toHaveBeenCalledTimes(1); + expect(scheduler.scheduleTask).not.toHaveBeenCalled(); + + releaseMigrations(); + await Promise.all([connectA, connectB]); + + expect(applyDatabaseMigrations).toHaveBeenCalledTimes(1); + const scheduledIds = (scheduler.scheduleTask as jest.Mock).mock.calls.map( + call => call[0].id, + ); + expect(scheduledIds).toEqual(['provider-a', 'provider-b']); + }); + + it('subscribes to events only when the provider reports topics', async () => { + const FreshWrapperProviders = freshWrapperProviders(); + const { scheduler, events } = makeHarness(); + + const providers = new FreshWrapperProviders({ + config: mockServices.rootConfig.mock(), + logger: mockServices.logger.mock(), + client: dummyClient, + scheduler: scheduler as unknown as SchedulerService, + applyDatabaseMigrations: jest.fn(async () => {}), + events: events as unknown as EventsService, + }); + + const topics = ['openchoreo.namespace', 'openchoreo.project']; + await providers + .wrap( + makeProvider('provider-events', { + onEvent: jest.fn(), + supportsEventTopics: () => topics, + }), + providerOptions, + ) + .connect({ applyMutation: jest.fn(), refresh: jest.fn() }); + + expect(events.subscribe).toHaveBeenCalledTimes(1); + expect(events.subscribe).toHaveBeenCalledWith({ + topics, + id: 'catalog-backend-module-incremental-ingestion:provider-events', + onEvent: expect.any(Function), + }); + }); + + it('resolves the ready signal only after every wrapped provider connected', async () => { + const FreshWrapperProviders = freshWrapperProviders(); + const { scheduler, events } = makeHarness(); + + const providers = new FreshWrapperProviders({ + config: mockServices.rootConfig.mock(), + logger: mockServices.logger.mock(), + client: dummyClient, + scheduler: scheduler as unknown as SchedulerService, + applyDatabaseMigrations: jest.fn(async () => {}), + events: events as unknown as EventsService, + }); + + const connection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const wrapped1 = providers.wrap( + makeProvider('provider-1'), + providerOptions, + ); + const wrapped2 = providers.wrap( + makeProvider('provider-2'), + providerOptions, + ); + + let ready = false; + providers.waitForReady().then(() => { + ready = true; + }); + + await wrapped1.connect(connection); + expect(ready).toBe(false); + + await wrapped2.connect(connection); + expect(ready).toBe(true); + await expect(providers.waitForReady()).resolves.toBeUndefined(); + }); +}); diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/module/WrapperProviders.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/module/WrapperProviders.ts new file mode 100644 index 000000000..4fa115f83 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/module/WrapperProviders.ts @@ -0,0 +1,196 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + LoggerService, + RootConfigService, + SchedulerService, +} from '@backstage/backend-plugin-api'; +import { stringifyError } from '@backstage/errors'; +import { + EntityProvider, + EntityProviderConnection, +} from '@backstage/plugin-catalog-node'; +import { createDeferred } from '@backstage/types'; +import express from 'express'; +import { Knex } from 'knex'; +import { Duration } from 'luxon'; +import { OpenChoreoIncrementalIngestionDatabaseManager } from '../database/OpenChoreoIncrementalIngestionDatabaseManager'; +import { applyDatabaseMigrations } from '../database/migrations'; +import { OpenChoreoIncrementalIngestionEngine } from '../engine/OpenChoreoIncrementalIngestionEngine'; +import { IncrementalProviderRouter } from '../router/routes'; +import { + IncrementalEntityProvider, + IncrementalEntityProviderOptions, +} from '../types'; +import { EventsService } from '@backstage/plugin-events-node'; + +const MINIMUM_SCHEDULER_INTERVAL_MS = 5000; +const BURST_LENGTH_MARGIN_MINUTES = 1; + +// Singleton promise for running the database migrations. +// Module-level so that concurrent WrapperProviders instances (and repeated +// connect calls) all await the same single migration run, preventing +// duplicated concurrent migrations. The applier chosen by the first caller — +// including a test-injected `applyDatabaseMigrations` override — is cached. +let migrationsPromise: Promise | undefined; + +/** + * WrapperProviders class for managing incremental entity providers. + * Handles initialization, database migrations, scheduling, and event subscriptions + * for providers that support burst-based, resumable entity ingestion. + */ + +/** + * Helps in the creation of the catalog entity providers that wrap the + * incremental ones. + */ +export class WrapperProviders { + private numberOfProvidersToConnect = 0; + private readonly readySignal = createDeferred(); + + constructor( + private readonly options: { + config: RootConfigService; + logger: LoggerService; + client: Knex; + scheduler: SchedulerService; + applyDatabaseMigrations?: typeof applyDatabaseMigrations; + events: EventsService; + }, + ) {} + + wrap( + provider: IncrementalEntityProvider, + options: IncrementalEntityProviderOptions, + ): EntityProvider { + this.numberOfProvidersToConnect += 1; + return { + getProviderName: () => provider.getProviderName(), + connect: async connection => { + try { + await this.startProvider(provider, options, connection); + } finally { + this.numberOfProvidersToConnect -= 1; + if (this.numberOfProvidersToConnect === 0) { + this.readySignal.resolve(); + } + } + }, + }; + } + + adminRouter(): express.Router { + return new IncrementalProviderRouter( + new OpenChoreoIncrementalIngestionDatabaseManager({ + client: this.options.client, + logger: this.options.logger, + }), + this.options.logger, + ).createRouter(); + } + + /** + * Waits for all wrapped providers to complete their initial connection. + * This is useful for tests or initialization code that needs to ensure + * all providers are ready before proceeding. + */ + waitForReady(): Promise { + return this.readySignal; + } + + private async startProvider( + provider: IncrementalEntityProvider, + providerOptions: IncrementalEntityProviderOptions, + connection: EntityProviderConnection, + ) { + const logger = this.options.logger.child({ + entityProvider: provider.getProviderName(), + }); + + try { + if (!migrationsPromise) { + migrationsPromise = Promise.resolve().then(async () => { + const apply = + this.options.applyDatabaseMigrations ?? applyDatabaseMigrations; + await apply(this.options.client); + }); + } + + await migrationsPromise; + + const { burstInterval, burstLength, restLength } = providerOptions; + + logger.info(`Connecting`); + + const manager = new OpenChoreoIncrementalIngestionDatabaseManager({ + client: this.options.client, + logger, + }); + const engine = new OpenChoreoIncrementalIngestionEngine({ + ...providerOptions, + ready: this.readySignal, + manager, + logger, + provider, + restLength, + connection, + }); + + let frequency = Duration.isDuration(burstInterval) + ? burstInterval + : Duration.fromObject(burstInterval); + if (frequency.as('milliseconds') < MINIMUM_SCHEDULER_INTERVAL_MS) { + frequency = Duration.fromMillis(MINIMUM_SCHEDULER_INTERVAL_MS); + } + + let length = Duration.isDuration(burstLength) + ? burstLength + : Duration.fromObject(burstLength); + length = length.plus( + Duration.fromObject({ minutes: BURST_LENGTH_MARGIN_MINUTES }), + ); + + await this.options.scheduler.scheduleTask({ + id: provider.getProviderName(), + fn: engine.taskFn.bind(engine), + frequency, + timeout: length, + }); + + const topics = engine.supportsEventTopics(); + if (topics.length > 0) { + logger.info( + `Provider ${provider.getProviderName()} subscribing to events for topics: ${topics.join( + ',', + )}`, + ); + await this.options.events.subscribe({ + topics, + id: `catalog-backend-module-incremental-ingestion:${provider.getProviderName()}`, + onEvent: evt => engine.onEvent(evt), + }); + } + } catch (error) { + logger.warn( + `Failed to initialize incremental ingestion provider ${provider.getProviderName()}, ${stringifyError( + error, + )}`, + ); + throw error; + } + } +} diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/module/catalogModuleIncrementalIngestionEntityProvider.test.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/module/catalogModuleIncrementalIngestionEntityProvider.test.ts new file mode 100644 index 000000000..eed8d7d87 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/module/catalogModuleIncrementalIngestionEntityProvider.test.ts @@ -0,0 +1,147 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Test suite for catalogModuleOpenchoreoIncrementalEntityProvider and + * catalogModuleOpenchoreoIncrementalProvider. + * Drives both backend modules through startTestBackend with a recording + * catalogProcessingExtensionPoint double. Incremental ingestion is opt-in: + * without the openchoreo.features.incrementalIngestion.enabled flag no + * provider is added and no migrations run; with the flag the wrapped + * OpenChoreoIncrementalEntityProvider is added, and driving its connect() + * engages the real migrations on the injected SQLite client and registers + * the scheduler task. + */ +import { mockServices, startTestBackend } from '@backstage/backend-test-utils'; +import { TestDatabases } from '@backstage/backend-test-utils'; +import type { SchedulerService } from '@backstage/backend-plugin-api'; +import { + coreServices, + createServiceFactory, +} from '@backstage/backend-plugin-api'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { catalogModuleOpenchoreoIncrementalProvider } from './openchoreoIncrementalProviderModule'; +import { catalogModuleOpenchoreoIncrementalEntityProvider } from './catalogModuleIncrementalIngestionEntityProvider'; +import { DB_MIGRATIONS_TABLE } from '../database/tables'; + +jest.setTimeout(60_000); + +const EXPECTED_MIGRATION_NAMES = [ + '20221116073152_init.js', + '20240110000001_add_performance_indexes.js', + '20240110000003_expand_last_error_field.js', +]; + +/** A recording scheduler factory exposing scheduleTask to the test. */ +function makeSchedulerFactory() { + const scheduleTask = jest.fn(); + const factory = createServiceFactory({ + service: coreServices.scheduler, + deps: {}, + factory: async () => ({ scheduleTask } as unknown as SchedulerService), + }); + return { scheduleTask, factory }; +} + +describe('catalogModuleOpenchoreoIncrementalEntityProvider', () => { + // SQLITE_3 runs everywhere with no docker dependency; the PostgreSQL + // variants of the migration are exercised in deployment environments. + const databases = TestDatabases.create({ + ids: ['SQLITE_3'], + }); + + it('stays idle without the incrementalIngestion.enabled flag', async () => { + const knex = await databases.init('SQLITE_3'); + const addEntityProvider = jest.fn(); + const scheduler = makeSchedulerFactory(); + + const backend = await startTestBackend({ + extensionPoints: [ + [catalogProcessingExtensionPoint, { addEntityProvider }], + ], + features: [ + mockServices.database.factory({ knex }), + scheduler.factory, + catalogModuleOpenchoreoIncrementalEntityProvider, + catalogModuleOpenchoreoIncrementalProvider, + ], + }); + + try { + expect(addEntityProvider).not.toHaveBeenCalled(); + expect(scheduler.scheduleTask).not.toHaveBeenCalled(); + + // The module never touched the database: no migrations were applied. + await expect(knex(DB_MIGRATIONS_TABLE).select('name')).rejects.toThrow(); + } finally { + await backend.stop(); + } + }); + + it('adds the wrapped provider and engages migrations and scheduling when the flag is true', async () => { + const knex = await databases.init('SQLITE_3'); + const addEntityProvider = jest.fn(); + const scheduler = makeSchedulerFactory(); + + const backend = await startTestBackend({ + extensionPoints: [ + [catalogProcessingExtensionPoint, { addEntityProvider }], + ], + features: [ + mockServices.rootConfig.factory({ + data: { + openchoreo: { + baseUrl: 'http://localhost:8080', + features: { incrementalIngestion: { enabled: true } }, + }, + }, + }), + mockServices.database.factory({ knex }), + scheduler.factory, + catalogModuleOpenchoreoIncrementalEntityProvider, + catalogModuleOpenchoreoIncrementalProvider, + ], + }); + + try { + expect(addEntityProvider).toHaveBeenCalledTimes(1); + const wrapped = addEntityProvider.mock.calls[0][0]; + expect(wrapped.getProviderName()).toBe( + 'OpenChoreoIncrementalEntityProvider', + ); + + // Drive the wrapped provider the way the catalog engine would. The + // wrapper is not injectable at the module layer, so engagement is + // asserted through the real migrations applied to the injected client + // and through the registered scheduler task. + await wrapped.connect({ applyMutation: jest.fn(), refresh: jest.fn() }); + + const names = (await knex(DB_MIGRATIONS_TABLE).select('name')).map( + (row: { name: string }) => row.name, + ); + expect(names.sort()).toEqual(EXPECTED_MIGRATION_NAMES); + + expect(scheduler.scheduleTask).toHaveBeenCalledTimes(1); + expect(scheduler.scheduleTask).toHaveBeenCalledWith( + expect.objectContaining({ + id: 'OpenChoreoIncrementalEntityProvider', + }), + ); + } finally { + await backend.stop(); + } + }); +}); diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/module/catalogModuleIncrementalIngestionEntityProvider.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/module/catalogModuleIncrementalIngestionEntityProvider.ts new file mode 100644 index 000000000..ecb1f7642 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/module/catalogModuleIncrementalIngestionEntityProvider.ts @@ -0,0 +1,162 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Main module for OpenChoreo incremental ingestion entity provider. + * Defines the extension point and backend module for registering and managing incremental providers. + */ + +import { + coreServices, + createBackendModule, + createExtensionPoint, +} from '@backstage/backend-plugin-api'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { WrapperProviders } from './WrapperProviders'; +import { eventsServiceRef } from '@backstage/plugin-events-node'; +import { + IncrementalEntityProvider, + IncrementalEntityProviderOptions, +} from '../types'; + +/** + * @public + * Interface for {@link openchoreoIncrementalProvidersExtensionPoint}. + */ +export interface OpenChoreoIncrementalProviderExtensionPoint { + /** Adds a new incremental entity provider */ + addProvider(config: { + options: IncrementalEntityProviderOptions; + provider: IncrementalEntityProvider; + }): void; +} + +/** + * @public + * + * Extension point for registering OpenChoreo incremental ingestion providers. + * The `catalogModuleOpenchoreoIncrementalEntityProvider` must be installed for these providers to work. + * + * @example + * + * ```ts +backend.add(createBackendModule({ + pluginId: 'catalog', + moduleId: 'my-openchoreo-incremental-provider', + register(env) { + env.registerInit({ + deps: { + extension: openchoreoIncrementalProvidersExtensionPoint, + }, + async init({ extension }) { + extension.addProvider({ + options: { + burstInterval:, + burstLength:, + restLength: , + }, + provider: { + next(context, cursor) { + }, + }, + }); + }, + }); +})) + * ``` +**/ +export const openchoreoIncrementalProvidersExtensionPoint = + createExtensionPoint({ + id: 'catalog.openchoreoIncrementalProvider.providers', + }); + +/** + * Registers the incremental entity provider with the catalog processing extension point for OpenChoreo. + * + * @public + */ +export const catalogModuleOpenchoreoIncrementalEntityProvider = + createBackendModule({ + pluginId: 'catalog', + moduleId: 'openchoreo-incremental-entity-provider', + register(env) { + const addedProviders = new Array<{ + provider: IncrementalEntityProvider; + options: IncrementalEntityProviderOptions; + }>(); + + env.registerExtensionPoint(openchoreoIncrementalProvidersExtensionPoint, { + addProvider({ options, provider }) { + addedProviders.push({ options, provider }); + }, + }); + + env.registerInit({ + deps: { + catalog: catalogProcessingExtensionPoint, + config: coreServices.rootConfig, + database: coreServices.database, + httpRouter: coreServices.httpRouter, + logger: coreServices.logger, + scheduler: coreServices.scheduler, + events: eventsServiceRef, + }, + async init({ + catalog, + config, + database, + httpRouter, + logger, + scheduler, + events, + }) { + // Incremental ingestion is opt-in: unless + // openchoreo.features.incrementalIngestion.enabled is true the + // module stays completely idle — no admin router, no wrapped + // providers (and therefore no migrations or scheduled tasks) — + // and the scheduled full-sync OpenChoreoEntityProvider from the + // sibling catalog module keeps running. + const incrementalEnabled = + config.getOptionalBoolean( + 'openchoreo.features.incrementalIngestion.enabled', + ) ?? false; + if (!incrementalEnabled) { + logger.info( + 'OpenChoreo incremental ingestion disabled (openchoreo.features.incrementalIngestion.enabled not set or false); module idle', + ); + return; + } + + const client = await database.getClient(); + + const providers = new WrapperProviders({ + config, + logger, + client, + scheduler, + events, + }); + + for (const entry of addedProviders) { + const wrapped = providers.wrap(entry.provider, entry.options); + catalog.addEntityProvider(wrapped); + } + + httpRouter.use(providers.adminRouter()); + }, + }); + }, + }); diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/module/index.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/module/index.ts new file mode 100644 index 000000000..684b6b12e --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/module/index.ts @@ -0,0 +1,27 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Module index for OpenChoreo incremental ingestion. + * Exports the main components for the incremental provider module. + */ + +export { + catalogModuleOpenchoreoIncrementalEntityProvider as default, + openchoreoIncrementalProvidersExtensionPoint, + type OpenChoreoIncrementalProviderExtensionPoint, +} from './catalogModuleIncrementalIngestionEntityProvider'; +export { catalogModuleOpenchoreoIncrementalProvider } from './openchoreoIncrementalProviderModule'; diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/module/openchoreoIncrementalProviderModule.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/module/openchoreoIncrementalProviderModule.ts new file mode 100644 index 000000000..4318b4eae --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/module/openchoreoIncrementalProviderModule.ts @@ -0,0 +1,99 @@ +/** + * Backend module for OpenChoreo incremental provider. + * Registers the OpenChoreoIncrementalEntityProvider with the extension point, + * configuring it with burst and rest intervals from the application config. + */ + +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { openchoreoIncrementalProvidersExtensionPoint } from './catalogModuleIncrementalIngestionEntityProvider'; +import { OpenChoreoIncrementalEntityProvider } from '../providers/OpenChoreoIncrementalEntityProvider'; + +export const catalogModuleOpenchoreoIncrementalProvider = createBackendModule({ + pluginId: 'catalog', + moduleId: 'openchoreo-incremental-provider', + register(env) { + env.registerInit({ + deps: { + extension: openchoreoIncrementalProvidersExtensionPoint, + config: coreServices.rootConfig, + logger: coreServices.logger, + }, + async init({ extension, config, logger }) { + // Mirrors the gate in catalogModuleIncrementalIngestionEntityProvider: + // when incremental ingestion is disabled the wrapper module stays + // idle (and logs the single "module idle" line), so skip offering + // the provider too — nothing would consume it anyway. + const incrementalEnabled = + config.getOptionalBoolean( + 'openchoreo.features.incrementalIngestion.enabled', + ) ?? false; + if (!incrementalEnabled) { + return; + } + + const provider = new OpenChoreoIncrementalEntityProvider({ + config, + logger, + }); + + extension.addProvider({ + provider, + options: { + // The interval between bursts of processing activity + burstInterval: { + seconds: Math.max( + 1, + config.getOptionalNumber( + 'openchoreo.incremental.burstInterval', + ) || 30, + ), + }, + // The duration of each burst of processing activity + burstLength: { + seconds: Math.max( + 1, + config.getOptionalNumber( + 'openchoreo.incremental.burstLength', + ) || 10, + ), + }, + // The duration of rest periods between bursts + restLength: { + minutes: Math.max( + 1, + config.getOptionalNumber('openchoreo.incremental.restLength') || + 30, + ), + }, + // Backoff intervals for retry attempts (configurable array of durations in seconds) + backoff: (() => { + const backoffConfig = config.getOptional( + 'openchoreo.incremental.backoff', + ); + if ( + Array.isArray(backoffConfig) && + backoffConfig.every( + (item): item is number => + typeof item === 'number' && item > 0, + ) + ) { + return backoffConfig.map((seconds: number) => ({ + seconds: Math.max(1, seconds), + })); + } + return [ + { seconds: 30 }, + { minutes: 1 }, + { minutes: 5 }, + { minutes: 30 }, + ]; + })(), + }, + }); + }, + }); + }, +}); diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/openchoreoImmediateCatalogIncremental.test.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/openchoreoImmediateCatalogIncremental.test.ts new file mode 100644 index 000000000..73403d16a --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/openchoreoImmediateCatalogIncremental.test.ts @@ -0,0 +1,155 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Test suite for openchoreoImmediateCatalogIncremental. + * Drives the backend module and the immediate-catalog service factory + * through startTestBackend with a recording catalogProcessingExtensionPoint + * double. The module and the factory share one ScaffolderEntityProvider + * instance: the provider registered with the catalog engine is the same one + * that backs the insertEntity/removeEntity service, and its delta mutations + * carry the location key of the main incremental provider so full syncs keep + * managing the same bucket of entities. + */ +import { startTestBackend } from '@backstage/backend-test-utils'; +import { createBackendModule } from '@backstage/backend-plugin-api'; +import type { Entity } from '@backstage/catalog-model'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import type { + EntityProvider, + EntityProviderConnection, +} from '@backstage/plugin-catalog-node'; +import { + immediateCatalogServiceRef, + type ImmediateCatalogService, +} from '@openchoreo/backstage-plugin-catalog-backend-module'; +import { + catalogModuleOpenchoreoImmediateCatalogIncremental, + openchoreoImmediateCatalogIncrementalServiceFactory, +} from './openchoreoImmediateCatalogIncremental'; + +jest.setTimeout(60_000); + +function makeEntity(name: string): Entity { + return { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { namespace: 'default', name }, + }; +} + +/** Builds a throwaway catalog module that captures the plugin-scoped service. */ +function makeServiceCapture(target: { service?: ImmediateCatalogService }) { + return createBackendModule({ + pluginId: 'catalog', + moduleId: 'capture-immediate-catalog-service', + register(env) { + env.registerInit({ + deps: { service: immediateCatalogServiceRef }, + async init({ service }) { + target.service = service; + }, + }); + }, + }); +} + +describe('openchoreoImmediateCatalogIncremental', () => { + it('registers a scaffolder entity provider with the catalog engine', async () => { + const addEntityProvider = jest.fn(); + + const backend = await startTestBackend({ + extensionPoints: [ + [catalogProcessingExtensionPoint, { addEntityProvider }], + ], + features: [catalogModuleOpenchoreoImmediateCatalogIncremental], + }); + + try { + expect(addEntityProvider).toHaveBeenCalledTimes(1); + const provider = addEntityProvider.mock.calls[0][0] as EntityProvider; + expect(provider.getProviderName()).toBe('ScaffolderEntityProvider'); + // The provider stays unconnected here; the shared instance is driven + // through the service in the test below. + } finally { + await backend.stop(); + } + }); + + it('routes immediate insert and remove mutations through the shared provider', async () => { + const addEntityProvider = jest.fn(); + const captured: { service?: ImmediateCatalogService } = {}; + + const backend = await startTestBackend({ + extensionPoints: [ + [catalogProcessingExtensionPoint, { addEntityProvider }], + ], + features: [ + catalogModuleOpenchoreoImmediateCatalogIncremental, + openchoreoImmediateCatalogIncrementalServiceFactory, + makeServiceCapture(captured), + ], + }); + + try { + expect(captured.service).toBeDefined(); + expect(captured.service!.insertEntity).toBeInstanceOf(Function); + expect(captured.service!.removeEntity).toBeInstanceOf(Function); + + const provider = addEntityProvider.mock.calls[0][0] as EntityProvider; + const applyMutation = jest.fn(); + const connection: EntityProviderConnection = { + applyMutation, + refresh: jest.fn(), + }; + + // Before the catalog engine connects the provider, mutations fail. + await expect( + captured.service!.insertEntity(makeEntity('too-early')), + ).rejects.toThrow(/not connected/i); + + await provider.connect(connection); + + await captured.service!.insertEntity(makeEntity('hello')); + expect(applyMutation).toHaveBeenCalledTimes(1); + expect(applyMutation).toHaveBeenNthCalledWith(1, { + type: 'delta', + added: [ + { + entity: makeEntity('hello'), + locationKey: 'provider:OpenChoreoIncrementalEntityProvider', + }, + ], + removed: [], + }); + + await captured.service!.removeEntity('component:default/hello'); + expect(applyMutation).toHaveBeenCalledTimes(2); + expect(applyMutation).toHaveBeenNthCalledWith(2, { + type: 'delta', + added: [], + removed: [ + { + entityRef: 'component:default/hello', + locationKey: 'provider:OpenChoreoIncrementalEntityProvider', + }, + ], + }); + } finally { + await backend.stop(); + } + }); +}); diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/openchoreoImmediateCatalogIncremental.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/openchoreoImmediateCatalogIncremental.ts new file mode 100644 index 000000000..c09bcc367 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/openchoreoImmediateCatalogIncremental.ts @@ -0,0 +1,71 @@ +import { + coreServices, + createBackendModule, + createServiceFactory, +} from '@backstage/backend-plugin-api'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { + immediateCatalogServiceRef, + ScaffolderEntityProvider, + type ImmediateCatalogService, +} from '@openchoreo/backstage-plugin-catalog-backend-module'; + +let scaffolderProviderInstance: ScaffolderEntityProvider | undefined; + +const MAIN_INCREMENTAL_PROVIDER_NAME = 'OpenChoreoIncrementalEntityProvider'; + +/** + * Adds a catalog entity provider that supports immediate delta mutations. + * + * This is intended to be used together with the OpenChoreo incremental ingestion provider, + * without enabling the legacy scheduled OpenChoreo provider. + */ +export const catalogModuleOpenchoreoImmediateCatalogIncremental = + createBackendModule({ + pluginId: 'catalog', + moduleId: 'openchoreo-immediate-catalog-incremental', + register(env) { + env.registerInit({ + deps: { + catalog: catalogProcessingExtensionPoint, + logger: coreServices.logger, + }, + async init({ catalog, logger }) { + if (!scaffolderProviderInstance) { + scaffolderProviderInstance = new ScaffolderEntityProvider( + logger, + MAIN_INCREMENTAL_PROVIDER_NAME, + ); + } + + catalog.addEntityProvider(scaffolderProviderInstance); + }, + }); + }, + }); + +/** + * Provides the `openchoreo.immediate-catalog` service used by OpenChoreo scaffolder actions. + */ +export const openchoreoImmediateCatalogIncrementalServiceFactory = + createServiceFactory({ + service: immediateCatalogServiceRef, + deps: { + logger: coreServices.logger, + }, + async factory({ logger }): Promise { + if (!scaffolderProviderInstance) { + scaffolderProviderInstance = new ScaffolderEntityProvider( + logger, + MAIN_INCREMENTAL_PROVIDER_NAME, + ); + } + + return { + insertEntity: async entity => + scaffolderProviderInstance!.insertEntity(entity), + removeEntity: async entityRef => + scaffolderProviderInstance!.removeEntity(entityRef), + }; + }, + }); diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/providers/OpenChoreoIncrementalEntityProvider.test.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/providers/OpenChoreoIncrementalEntityProvider.test.ts new file mode 100644 index 000000000..25922650c --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/providers/OpenChoreoIncrementalEntityProvider.test.ts @@ -0,0 +1,476 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Test suite for OpenChoreoIncrementalEntityProvider. + * Verifies cursor-based phase traversal (namespaces -> projects -> + * components), exact cursor chains, restart resumability, and failure + * handling of the resumable incremental ingestion provider. + */ +import { ConfigReader } from '@backstage/config'; +import { mockServices } from '@backstage/backend-test-utils'; +import { createOpenChoreoApiClient } from '@openchoreo/openchoreo-client-node'; +import type * as mockApi from '../testUtils/mockApi'; +import { + createMockClient, + mockComponent, + mockNamespace, + mockProject, +} from '../testUtils/mockApi'; +import { + OpenChoreoIncrementalEntityProvider, + type OpenChoreoCursor, +} from './OpenChoreoIncrementalEntityProvider'; + +jest.mock('@openchoreo/openchoreo-client-node', () => + ( + jest.requireActual('../testUtils/mockApi') as typeof mockApi + ).mockOpenChoreoClientNodeModule(), +); +jest.mock('@openchoreo/backstage-plugin-catalog-backend-module', () => + ( + jest.requireActual('../testUtils/mockApi') as typeof mockApi + ).mockCatalogBackendModule(), +); + +const NAMESPACES_PATH = '/api/v1/namespaces'; +const PROJECTS_PATH = '/api/v1/namespaces/{namespaceName}/projects'; +const COMPONENTS_PATH = '/api/v1/namespaces/{namespaceName}/components'; + +function domain(name: string) { + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Domain', + metadata: { name }, + }, + }; +} + +function system(name: string) { + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'System', + metadata: { name }, + }, + }; +} + +function component(name: string) { + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { name }, + }, + }; +} + +describe('OpenChoreoIncrementalEntityProvider', () => { + const logger = mockServices.logger.mock(); + const context = { baseUrl: 'http://localhost:8080', logger }; + + const createProvider = (chunkSize = 10) => + new OpenChoreoIncrementalEntityProvider({ + config: new ConfigReader({ + openchoreo: { + baseUrl: 'http://localhost:8080', + incremental: { chunkSize }, + }, + }), + logger, + }); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('returns its provider name', () => { + expect(createProvider().getProviderName()).toBe( + 'OpenChoreoIncrementalEntityProvider', + ); + }); + + it('around() supplies the burst with baseUrl, logger and token', async () => { + const burst = jest.fn(); + await createProvider().around(burst); + expect(burst).toHaveBeenCalledWith({ + baseUrl: 'http://localhost:8080', + logger, + token: undefined, + }); + }); + + it('emits a Domain per namespace on the initial page and seeds the cursor', async () => { + const client = createMockClient({ + [NAMESPACES_PATH]: [ + { items: [mockNamespace('ns-1'), mockNamespace('ns-2')] }, + ], + }); + + const result = await createProvider().next(context); + + expect(result).toEqual({ + done: false, + entities: [domain('ns-1'), domain('ns-2')], + cursor: { + phase: 'projects', + namespaceApiCursor: undefined, + namespaceQueue: ['ns-1', 'ns-2'], + currentIndex: 0, + }, + }); + expect(client.calls).toEqual([ + { path: NAMESPACES_PATH, params: { query: { limit: 10 } } }, + ]); + expect(createOpenChoreoApiClient).toHaveBeenCalledWith( + expect.objectContaining({ baseUrl: 'http://localhost:8080', logger }), + ); + }); + + it('continues namespaces through namespaceApiCursor and moves to projects when the list ends', async () => { + const client = createMockClient({ + [NAMESPACES_PATH]: [ + { items: [mockNamespace('ns-1')], nextCursor: 'ns-c1' }, + { items: [mockNamespace('ns-2')] }, + ], + }); + const provider = createProvider(); + + const first = await provider.next(context); + expect(first.done).toBe(false); + expect(first.cursor).toEqual({ + phase: 'namespaces', + namespaceApiCursor: 'ns-c1', + namespaceQueue: ['ns-1'], + currentIndex: 0, + }); + + const second = await provider.next(context, first.cursor); + expect(second).toEqual({ + done: false, + entities: [domain('ns-2')], + cursor: { + phase: 'projects', + namespaceApiCursor: undefined, + namespaceQueue: ['ns-1', 'ns-2'], + currentIndex: 0, + }, + }); + expect(client.calls[1]).toEqual({ + path: NAMESPACES_PATH, + params: { query: { limit: 10, cursor: 'ns-c1' } }, + }); + }); + + it('walks an exact three-page namespace cursor chain', async () => { + const client = createMockClient({ + [NAMESPACES_PATH]: [ + { items: [mockNamespace('ns-a')], nextCursor: 'ns-c1' }, + { items: [mockNamespace('ns-b')], nextCursor: 'ns-c2' }, + { items: [mockNamespace('ns-c')] }, + ], + }); + const provider = createProvider(); + + const first = await provider.next(context); + const second = await provider.next(context, first.cursor); + const third = await provider.next(context, second.cursor); + + expect(client.calls.map(call => call.params?.query)).toEqual([ + { limit: 10 }, + { limit: 10, cursor: 'ns-c1' }, + { limit: 10, cursor: 'ns-c2' }, + ]); + expect(first.cursor).toEqual({ + phase: 'namespaces', + namespaceApiCursor: 'ns-c1', + namespaceQueue: ['ns-a'], + currentIndex: 0, + }); + expect(second.cursor).toEqual({ + phase: 'namespaces', + namespaceApiCursor: 'ns-c2', + namespaceQueue: ['ns-a', 'ns-b'], + currentIndex: 0, + }); + expect(third).toEqual({ + done: false, + entities: [domain('ns-c')], + cursor: { + phase: 'projects', + namespaceApiCursor: undefined, + namespaceQueue: ['ns-a', 'ns-b', 'ns-c'], + currentIndex: 0, + }, + }); + }); + + it('pages projects per namespace, advances currentIndex and transitions to components', async () => { + const client = createMockClient({ + ['/api/v1/namespaces/ns-1/projects']: [ + { items: [mockProject('proj-1a')], nextCursor: 'pr-ns1-c1' }, + { items: [mockProject('proj-1b')] }, + ], + ['/api/v1/namespaces/ns-2/projects']: [ + { items: [mockProject('proj-2a')] }, + ], + }); + const provider = createProvider(); + const cursor: OpenChoreoCursor = { + phase: 'projects', + namespaceQueue: ['ns-1', 'ns-2'], + currentIndex: 0, + }; + + const first = await provider.next(context, cursor); + expect(first).toEqual({ + done: false, + entities: [system('proj-1a')], + cursor: { + ...cursor, + projectApiCursor: 'pr-ns1-c1', + currentNamespace: 'ns-1', + }, + }); + + const second = await provider.next(context, first.cursor); + expect(second).toEqual({ + done: false, + entities: [system('proj-1b')], + cursor: { + ...cursor, + projectApiCursor: undefined, + currentIndex: 1, + currentNamespace: 'ns-1', + }, + }); + + const third = await provider.next(context, second.cursor); + expect(third).toEqual({ + done: false, + entities: [system('proj-2a')], + cursor: { + ...cursor, + projectApiCursor: undefined, + currentIndex: 2, + currentNamespace: 'ns-2', + }, + }); + + // Queue exhausted: pure transition call, no extra GET. + const fourth = await provider.next(context, third.cursor); + expect(fourth).toEqual({ + done: false, + entities: [], + cursor: { + phase: 'components', + namespaceQueue: ['ns-1', 'ns-2'], + currentIndex: 0, + projectApiCursor: undefined, + currentNamespace: 'ns-2', + }, + }); + expect(client.calls).toEqual([ + { + path: PROJECTS_PATH, + params: { path: { namespaceName: 'ns-1' }, query: { limit: 10 } }, + }, + { + path: PROJECTS_PATH, + params: { + path: { namespaceName: 'ns-1' }, + query: { limit: 10, cursor: 'pr-ns1-c1' }, + }, + }, + { + path: PROJECTS_PATH, + params: { path: { namespaceName: 'ns-2' }, query: { limit: 10 } }, + }, + ]); + }); + + it('fetches components flat per namespace without a project filter and finishes only after the last page', async () => { + const client = createMockClient({ + ['/api/v1/namespaces/ns-1/components']: [ + { items: [mockComponent('cmp-1')], nextCursor: 'cmp-ns1-c1' }, + { items: [mockComponent('cmp-2')] }, + ], + ['/api/v1/namespaces/ns-2/components']: [ + { items: [mockComponent('cmp-3')] }, + ], + }); + const provider = createProvider(); + const cursor: OpenChoreoCursor = { + phase: 'components', + namespaceQueue: ['ns-1', 'ns-2'], + currentIndex: 0, + }; + + const first = await provider.next(context, cursor); + expect(first.done).toBe(false); + expect(first.entities).toEqual([component('cmp-1')]); + // The component list is flat: no project filter is sent. + expect(client.calls[0]).toEqual({ + path: COMPONENTS_PATH, + params: { path: { namespaceName: 'ns-1' }, query: { limit: 10 } }, + }); + expect(client.calls[0].params?.query).not.toHaveProperty('projectName'); + expect(first.cursor).toMatchObject({ + componentApiCursor: 'cmp-ns1-c1', + currentIndex: 0, + currentNamespace: 'ns-1', + }); + + const second = await provider.next(context, first.cursor); + expect(second.done).toBe(false); + expect(second.entities).toEqual([component('cmp-2')]); + expect(second.cursor).toMatchObject({ + componentApiCursor: undefined, + currentIndex: 1, + }); + + const third = await provider.next(context, second.cursor); + expect(third.done).toBe(false); + expect(third.entities).toEqual([component('cmp-3')]); + expect(third.cursor).toMatchObject({ currentIndex: 2 }); + + // Only the call past the last namespace's last page reports done. + const fourth = await provider.next(context, third.cursor); + expect(fourth).toEqual({ done: true }); + expect(client.calls).toHaveLength(3); + }); + + it('resumes a JSON round-tripped cursor in a brand new provider instance', async () => { + const client = createMockClient({ + [NAMESPACES_PATH]: [ + { items: [mockNamespace('ns-a')], nextCursor: 'ns-c1' }, + { items: [mockNamespace('ns-b')], nextCursor: 'ns-c2' }, + { items: [mockNamespace('ns-c')] }, + ], + }); + const firstProvider = createProvider(); + + const first = await firstProvider.next(context); + const second = await firstProvider.next(context, first.cursor); + + // Simulate persisting the cursor to the ingestion database and + // restarting the process: a fresh provider instance receives the + // cursor after a JSON round-trip. + const restartedCursor = JSON.parse( + JSON.stringify(second.cursor), + ) as OpenChoreoCursor; + const restartedProvider = createProvider(); + const third = await restartedProvider.next(context, restartedCursor); + + expect(third).toEqual({ + done: false, + entities: [domain('ns-c')], + cursor: { + phase: 'projects', + namespaceApiCursor: undefined, + namespaceQueue: ['ns-a', 'ns-b', 'ns-c'], + currentIndex: 0, + }, + }); + expect(client.calls[2]).toEqual({ + path: NAMESPACES_PATH, + params: { query: { limit: 10, cursor: 'ns-c2' } }, + }); + }); + + it('rejects a stuck namespace cursor that echoes back unchanged', async () => { + createMockClient({ + [NAMESPACES_PATH]: [ + { items: [mockNamespace('ns-1')], nextCursor: 'stuck-ns-cursor' }, + { items: [mockNamespace('ns-2')], nextCursor: 'stuck-ns-cursor' }, + ], + }); + const provider = createProvider(); + + const first = await provider.next(context); + await expect(provider.next(context, first.cursor)).rejects.toThrow( + "same pagination cursor for namespaces twice ('stuck-ns-cursor')", + ); + }); + + it('rejects a stuck project cursor, naming the namespace', async () => { + createMockClient({ + ['/api/v1/namespaces/ns-1/projects']: [ + { items: [mockProject('proj-1')], nextCursor: 'stuck-pr-cursor' }, + { items: [mockProject('proj-1-echo')], nextCursor: 'stuck-pr-cursor' }, + ], + }); + const provider = createProvider(); + + await expect( + provider.next(context, { + phase: 'projects', + namespaceQueue: ['ns-1'], + currentIndex: 0, + projectApiCursor: 'stuck-pr-cursor', + }), + ).rejects.toThrow( + "same pagination cursor for projects in namespace ns-1 twice ('stuck-pr-cursor')", + ); + }); + + it('rejects with the response status when the namespaces fetch fails', async () => { + createMockClient({ + [NAMESPACES_PATH]: [ + { status: 500, error: { message: 'upstream exploded' } }, + ], + }); + + await expect(createProvider().next(context)).rejects.toThrow( + 'Failed to fetch namespaces: 500 HTTP 500 - upstream exploded', + ); + }); + + it('rejects with the response status when a projects fetch fails', async () => { + createMockClient({ + ['/api/v1/namespaces/ns-1/projects']: [ + { status: 404, error: { message: 'namespace not found' } }, + ], + }); + + await expect( + createProvider().next(context, { + phase: 'projects', + namespaceQueue: ['ns-1'], + currentIndex: 0, + }), + ).rejects.toThrow( + 'Failed to fetch projects for namespace ns-1: 404 HTTP 404 - namespace not found', + ); + }); + + it('caps a configured chunkSize of 500 to the API maximum of 100', async () => { + const client = createMockClient({ + [NAMESPACES_PATH]: [{ items: [mockNamespace('ns-1')] }], + }); + + await createProvider(500).next(context); + + expect(client.calls[0].params?.query?.limit).toBe(100); + expect(logger.debug).toHaveBeenCalledWith( + expect.stringContaining( + 'Configured chunkSize 500 exceeds API max; capping to 100', + ), + ); + }); +}); diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/providers/OpenChoreoIncrementalEntityProvider.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/providers/OpenChoreoIncrementalEntityProvider.ts new file mode 100644 index 000000000..5b73ff5b8 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/providers/OpenChoreoIncrementalEntityProvider.ts @@ -0,0 +1,534 @@ +import type { Config } from '@backstage/config'; +import type { LoggerService } from '@backstage/backend-plugin-api'; +import { Entity } from '@backstage/catalog-model'; +import { + createOpenChoreoApiClient, + getCreatedAt, + getDeletionTimestamp, + getDescription, + getDisplayName, + getName, + getNamespace, + getUid, + type OpenChoreoComponents, +} from '@openchoreo/openchoreo-client-node'; +import { ComponentTypeUtils } from '@openchoreo/backstage-plugin-common'; +import { + translateNamespaceToDomainEntity, + translateProjectToEntity, +} from '@openchoreo/backstage-plugin-catalog-backend-module'; +import type { EntityIteratorResult, IncrementalEntityProvider } from '../types'; +import { ComponentBatchProcessor } from './componentBatchProcessor'; + +// New-API resource shapes returned by the typed client +type NewNamespace = OpenChoreoComponents['schemas']['Namespace']; +type NewProject = OpenChoreoComponents['schemas']['Project']; + +/** The typed OpenChoreo API client returned by createOpenChoreoApiClient. */ +export type OpenChoreoApiClient = ReturnType; + +/** + * Incremental entity provider for OpenChoreo. + * Processes entities in phases (namespaces, projects, components) using + * cursor-based pagination to enable efficient, resumable ingestion of large + * datasets. + * + * ## Iterator Semantics + * - `done: false` = Continue iteration, more batches available + * - `done: true` = Iteration complete, no more data to process + * + * **Important**: `done: false` means overall iteration continues, NOT that + * the current resource has more items. When a resource is exhausted, we + * return `done: false` and advance to the next resource. Only when ALL + * phases complete do we return `done: true`. + * + * One page of one phase is fetched per `next()` call; the cursor is + * persisted between bursts by the ingestion engine, which makes the + * traversal resumable across process restarts. + */ + +interface CursorTraversalCursor { + phase: 'namespaces' | 'projects' | 'components'; + namespaceApiCursor?: string; + projectApiCursor?: string; + componentApiCursor?: string; + namespaceQueue: string[]; + currentIndex: number; + currentNamespace?: string; +} + +export type OpenChoreoCursor = CursorTraversalCursor; + +// Context for API client and shared state +export interface OpenChoreoContext { + baseUrl: string; + logger: LoggerService; + token?: string; +} + +/** Extracts a human-readable message from an openapi-fetch error object. */ +function extractErrorMessage(error: unknown): string { + return typeof error === 'object' && error !== null && 'message' in error + ? (error as { message: string }).message + : JSON.stringify(error); +} + +/** + * Incremental entity provider for OpenChoreo that processes entities in + * phases using cursor-based pagination for efficient, resumable ingestion of + * large datasets. Processes namespaces, projects, and components in sequence + * with memory-efficient chunking. Supports progressive traversal through + * large catalogs without requiring full data loading. + */ +export class OpenChoreoIncrementalEntityProvider + implements IncrementalEntityProvider +{ + // The OpenAPI schema caps page size at 100 (LimitParam maximum) + private static readonly API_MAX_PAGE_LIMIT = 100; + + private readonly config: Config; + private readonly logger: LoggerService; + private readonly baseUrl: string; + private readonly token?: string; + private readonly chunkSize: number; + private readonly defaultOwner: string; + private readonly componentTypeUtils: ComponentTypeUtils; + private readonly batchProcessor: ComponentBatchProcessor; + + /** + * Creates a new instance of the incremental entity provider + * @param options - Backstage config and logger for OpenChoreo settings + */ + constructor(readonly options: { config: Config; logger: LoggerService }) { + this.config = options.config; + this.logger = options.logger; + this.baseUrl = this.config.getString('openchoreo.baseUrl'); + this.token = this.config.getOptionalString('openchoreo.token'); + + const configuredChunkSize = + this.config.getOptionalNumber('openchoreo.incremental.chunkSize') || 100; + this.chunkSize = Math.min( + configuredChunkSize, + OpenChoreoIncrementalEntityProvider.API_MAX_PAGE_LIMIT, + ); + if (this.chunkSize < configuredChunkSize) { + this.logger.debug( + `Configured chunkSize ${configuredChunkSize} exceeds API max; capping to ${this.chunkSize}`, + ); + } + + // Default owner for built-in Backstage entities (Domain, System, + // Component). These kinds require an owner per Backstage schema + // validation. Qualified with the 'default' namespace so the owner + // resolves correctly for entities in non-default namespaces. + const ownerName = + this.config.getOptionalString('openchoreo.defaultOwner') || + 'openchoreo-users'; + this.defaultOwner = `group:default/${ownerName}`; + + this.componentTypeUtils = ComponentTypeUtils.fromConfig(this.config); + this.batchProcessor = new ComponentBatchProcessor({ + locationKey: `provider:${this.getProviderName()}`, + defaultOwner: this.defaultOwner, + componentTypeUtils: this.componentTypeUtils, + }); + } + + getProviderName(): string { + return 'OpenChoreoIncrementalEntityProvider'; + } + + /** + * Sets up the provider context for a burst of processing. + * @param burst - Function to execute with the prepared context + */ + async around( + burst: (context: OpenChoreoContext) => Promise, + ): Promise { + const context: OpenChoreoContext = { + baseUrl: this.baseUrl, + logger: this.logger, + token: this.token, + }; + + await burst(context); + } + + /** + * Processes the next batch of entities using cursor-based pagination. + * Exactly one API page is fetched per call; the returned cursor captures + * the full traversal state so the next call can resume seamlessly. + * @param context - Provider context with baseUrl and logger + * @param cursor - Current traversal state for resumable processing + * @returns Iterator result with entities and next cursor state + */ + async next( + context: OpenChoreoContext, + cursor?: OpenChoreoCursor, + ): Promise> { + const client = createOpenChoreoApiClient({ + baseUrl: context.baseUrl, + token: context.token, + logger: context.logger, + }); + + if (!cursor) { + return this.processInitialPage(client); + } + + switch (cursor.phase) { + case 'namespaces': + return this.processNamespacesPhase(client, cursor); + case 'projects': + return this.processProjectsPhase(client, cursor); + case 'components': + return this.processComponentsPhase(client, context, cursor); + default: + return { done: true }; // Unknown phase = complete iteration + } + } + + // ===================== Phase implementations ===================== // + + /** + * Fetches the first page of namespaces and seeds the traversal cursor. + */ + private async processInitialPage( + client: OpenChoreoApiClient, + ): Promise> { + const res = await client.GET('/api/v1/namespaces', { + params: { query: { limit: this.chunkSize } }, + }); + if (res.error || !res.data) { + throw new Error( + `Failed to fetch namespaces: ${res.response.status} ${ + res.response.statusText + } - ${extractErrorMessage(res.error)}`, + ); + } + + const items = res.data.items ?? []; + const entities: Entity[] = items.map(ns => this.translateNamespace(ns)); + const namespaceQueue = items + .map(ns => getName(ns)) + .filter((name): name is string => Boolean(name)); + const nextCursor = res.data.pagination?.nextCursor; + + const initial: CursorTraversalCursor = { + phase: nextCursor ? 'namespaces' : 'projects', + namespaceApiCursor: nextCursor, + namespaceQueue, + currentIndex: 0, + }; + + return { + done: false, + entities: entities.map(entity => ({ entity })), + cursor: initial, + }; + } + + /** + * Phase 'namespaces': pages through the namespace list, emitting Domain + * entities and appending names to the queue. Transitions to the projects + * phase once the namespace list is exhausted. + */ + private async processNamespacesPhase( + client: OpenChoreoApiClient, + cursor: CursorTraversalCursor, + ): Promise> { + if (!cursor.namespaceApiCursor) { + // No more namespace pages — transition to the projects phase. + // Note: done:false = continue iteration, not current resource has + // more items. + return { + done: false, + entities: [], + cursor: { + ...cursor, + phase: 'projects', + currentIndex: 0, + }, + }; + } + + const res = await client.GET('/api/v1/namespaces', { + params: { + query: { + limit: this.chunkSize, + cursor: cursor.namespaceApiCursor, + }, + }, + }); + if (res.error || !res.data) { + throw new Error( + `Failed to fetch namespaces: ${res.response.status} ${ + res.response.statusText + } - ${extractErrorMessage(res.error)}`, + ); + } + + const nextCursor = res.data.pagination?.nextCursor; + this.assertCursorAdvances( + cursor.namespaceApiCursor, + nextCursor, + 'namespaces', + ); + + const items = res.data.items ?? []; + const entities: Entity[] = items.map(ns => this.translateNamespace(ns)); + const newNames = items + .map(ns => getName(ns)) + .filter((name): name is string => Boolean(name)); + + return { + done: false, + entities: entities.map(entity => ({ entity })), + cursor: { + ...cursor, + namespaceApiCursor: nextCursor, + namespaceQueue: cursor.namespaceQueue.concat(newNames), + phase: nextCursor ? 'namespaces' : 'projects', + currentIndex: 0, + }, + }; + } + + /** + * Phase 'projects': pages through the project list of each queued + * namespace in turn, emitting System entities. Transitions to the + * components phase once every namespace's projects are exhausted. + */ + private async processProjectsPhase( + client: OpenChoreoApiClient, + cursor: CursorTraversalCursor, + ): Promise> { + if (cursor.currentIndex >= cursor.namespaceQueue.length) { + // All namespaces' projects processed — transition to components. + // Note: done:false = continue iteration, not current resource has + // more items. + return { + done: false, + entities: [], + cursor: { + ...cursor, + phase: 'components', + currentIndex: 0, + projectApiCursor: undefined, + }, + }; + } + + const namespaceName = cursor.namespaceQueue[cursor.currentIndex]; + + const res = await client.GET( + '/api/v1/namespaces/{namespaceName}/projects', + { + params: { + path: { namespaceName }, + query: { + limit: this.chunkSize, + ...(cursor.projectApiCursor && { cursor: cursor.projectApiCursor }), + }, + }, + }, + ); + if (res.error || !res.data) { + throw new Error( + `Failed to fetch projects for namespace ${namespaceName}: ${ + res.response.status + } ${res.response.statusText} - ${extractErrorMessage(res.error)}`, + ); + } + + const nextCursor = res.data.pagination?.nextCursor; + this.assertCursorAdvances( + cursor.projectApiCursor, + nextCursor, + `projects in namespace ${namespaceName}`, + ); + + const items = res.data.items ?? []; + const entities: Entity[] = items.map(project => + this.translateProject(project, namespaceName), + ); + + if (!nextCursor) { + // Finished this namespace's projects — move to the next namespace. + // Note: done:false = continue iteration, not current resource has + // more items. + return { + done: false, + entities: entities.map(entity => ({ entity })), + cursor: { + ...cursor, + projectApiCursor: undefined, + currentIndex: cursor.currentIndex + 1, + currentNamespace: namespaceName, + }, + }; + } + + return { + done: false, + entities: entities.map(entity => ({ entity })), + cursor: { + ...cursor, + projectApiCursor: nextCursor, + currentNamespace: namespaceName, + }, + }; + } + + /** + * Phase 'components': pages through the FLAT component list of each + * queued namespace (no project filter), delegating translation to the + * batch processor. The owning project of each component is read from its + * spec (`spec.owner.projectName`). This is the only phase that can + * return `done: true`. + */ + private async processComponentsPhase( + client: OpenChoreoApiClient, + context: OpenChoreoContext, + cursor: CursorTraversalCursor, + ): Promise> { + if (cursor.currentIndex >= cursor.namespaceQueue.length) { + return { done: true }; // Iteration complete — no more data + } + + const namespaceName = cursor.namespaceQueue[cursor.currentIndex]; + + const res = await client.GET( + '/api/v1/namespaces/{namespaceName}/components', + { + params: { + path: { namespaceName }, + query: { + limit: this.chunkSize, + ...(cursor.componentApiCursor && { + cursor: cursor.componentApiCursor, + }), + }, + }, + }, + ); + if (res.error || !res.data) { + throw new Error( + `Failed to fetch components for namespace ${namespaceName}: ${ + res.response.status + } ${res.response.statusText} - ${extractErrorMessage(res.error)}`, + ); + } + + const nextCursor = res.data.pagination?.nextCursor; + this.assertCursorAdvances( + cursor.componentApiCursor, + nextCursor, + `components in namespace ${namespaceName}`, + ); + + const items = res.data.items ?? []; + const entities = await this.batchProcessor.translateComponentsWithApisBatch( + client, + items, + namespaceName, + { logger: context.logger, config: this.config }, + ); + + if (!nextCursor) { + // Finished this namespace's components — move to the next namespace. + // Note: done:false = continue iteration, not current resource has + // more items. + return { + done: false, + entities: entities.map(entity => ({ entity })), + cursor: { + ...cursor, + componentApiCursor: undefined, + currentIndex: cursor.currentIndex + 1, + currentNamespace: namespaceName, + }, + }; + } + + return { + done: false, + entities: entities.map(entity => ({ entity })), + cursor: { + ...cursor, + componentApiCursor: nextCursor, + currentNamespace: namespaceName, + }, + }; + } + + // ===================== Translation helpers ===================== // + + /** + * Translates a new-API Namespace into a Backstage Domain entity using the + * shared translator from the non-incremental module. + */ + private translateNamespace(namespace: NewNamespace): Entity { + return translateNamespaceToDomainEntity( + { + name: getName(namespace)!, + displayName: getDisplayName(namespace), + description: getDescription(namespace), + createdAt: getCreatedAt(namespace), + status: namespace.status?.phase, + }, + { + locationKey: this.getProviderName(), + defaultOwner: this.defaultOwner, + }, + ); + } + + /** + * Translates a new-API Project into a Backstage System entity using the + * shared translator from the non-incremental module. + */ + private translateProject(project: NewProject, namespaceName: string): Entity { + return translateProjectToEntity( + { + name: getName(project)!, + displayName: getDisplayName(project), + description: getDescription(project), + namespaceName: getNamespace(project) ?? namespaceName, + uid: getUid(project), + deletionTimestamp: getDeletionTimestamp(project), + deploymentPipelineRef: project.spec?.deploymentPipelineRef?.name, + projectTypeName: project.spec?.type?.name, + projectTypeKind: project.spec?.type?.kind, + }, + namespaceName, + { + locationKey: this.getProviderName(), + defaultOwner: this.defaultOwner, + }, + ); + } + + // ===================== Guards ===================== // + + /** + * Guards against a server returning the exact cursor that was just sent, + * which would loop the traversal forever. Throwing here lets the + * ingestion engine back off and retry the burst. + */ + private assertCursorAdvances( + sentCursor: string | undefined, + receivedCursor: string | undefined, + label: string, + ): void { + if (receivedCursor && sentCursor && receivedCursor === sentCursor) { + throw new Error( + `OpenChoreo API returned the same pagination cursor for ${label} twice ` + + `('${receivedCursor.substring( + 0, + 50, + )}'); aborting to avoid an infinite loop`, + ); + } + } +} diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/providers/componentBatchProcessor.test.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/providers/componentBatchProcessor.test.ts new file mode 100644 index 000000000..9a73ed6d8 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/providers/componentBatchProcessor.test.ts @@ -0,0 +1,165 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Test suite for ComponentBatchProcessor. + * Verifies batch translation of flat per-namespace components through the + * shared translator of the non-incremental sibling module, including the + * project attribution rules (spec.owner.projectName with namespace + * fallback) and nameless-item skipping. + */ +import { ConfigReader } from '@backstage/config'; +import { mockServices } from '@backstage/backend-test-utils'; +import { ComponentTypeUtils } from '@openchoreo/backstage-plugin-common'; +import { translateComponentToEntity } from '@openchoreo/backstage-plugin-catalog-backend-module'; +import type * as mockApi from '../testUtils/mockApi'; +import { createMockClient, mockComponent } from '../testUtils/mockApi'; +import { ComponentBatchProcessor } from './componentBatchProcessor'; + +jest.mock('@openchoreo/backstage-plugin-catalog-backend-module', () => + ( + jest.requireActual('../testUtils/mockApi') as typeof mockApi + ).mockCatalogBackendModule(), +); + +describe('ComponentBatchProcessor', () => { + const logger = mockServices.logger.mock(); + const processor = new ComponentBatchProcessor({ + locationKey: 'provider:OpenChoreoIncrementalEntityProvider', + defaultOwner: 'group:default/openchoreo-users', + componentTypeUtils: ComponentTypeUtils.createDefault(), + }); + const context = { logger, config: new ConfigReader({}) }; + + const run = ( + components: Parameters< + ComponentBatchProcessor['translateComponentsWithApisBatch'] + >[1], + ) => + processor.translateComponentsWithApisBatch( + createMockClient({}) as never, + components, + 'ns-1', + context, + ); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('translates every component through the shared translator', async () => { + const entities = await run([ + mockComponent('svc-ready'), + mockComponent('svc-not-ready', { + status: { + conditions: [ + { + type: 'Ready', + status: 'False', + reason: 'RenderingFailed', + }, + ], + }, + }), + ]); + + expect(entities).toEqual([ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { name: 'svc-ready' }, + }, + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { name: 'svc-not-ready' }, + }, + ]); + expect(translateComponentToEntity).toHaveBeenCalledTimes(2); + expect(translateComponentToEntity).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'svc-ready', + type: 'deployment/service', + status: 'Ready', + }), + 'ns-1', + 'svc-ready-project', + expect.objectContaining({ + defaultOwner: 'group:default/openchoreo-users', + componentTypeUtils: expect.any(ComponentTypeUtils), + locationKey: 'provider:OpenChoreoIncrementalEntityProvider', + }), + ); + expect(translateComponentToEntity).toHaveBeenCalledWith( + expect.objectContaining({ name: 'svc-not-ready', status: 'Not Ready' }), + 'ns-1', + 'svc-not-ready-project', + expect.anything(), + ); + }); + + it('attributes the component to spec.owner.projectName', async () => { + await run([ + mockComponent('svc', { spec: { owner: { projectName: 'proj-x' } } }), + ]); + + expect(translateComponentToEntity).toHaveBeenCalledWith( + expect.objectContaining({ name: 'svc' }), + 'ns-1', + 'proj-x', + expect.anything(), + ); + }); + + it('falls back to the namespace when spec.owner.projectName is absent', async () => { + const entities = await run([ + mockComponent('orphan', { spec: { owner: {} } }), + ]); + + expect(entities).toEqual([ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { name: 'orphan' }, + }, + ]); + expect(translateComponentToEntity).toHaveBeenCalledWith( + expect.objectContaining({ name: 'orphan' }), + 'ns-1', + 'ns-1', + expect.anything(), + ); + expect(logger.debug).toHaveBeenCalledWith( + expect.stringContaining( + 'Component orphan in namespace ns-1 has no project reference', + ), + ); + }); + + it('skips components without a name', async () => { + const entities = await run([ + mockComponent('anonymous', { metadata: { name: undefined } }), + ]); + + expect(entities).toEqual([]); + expect(translateComponentToEntity).not.toHaveBeenCalled(); + expect(logger.debug).toHaveBeenCalledWith( + expect.stringContaining( + 'Skipping component without a name in namespace ns-1', + ), + ); + }); +}); diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/providers/componentBatchProcessor.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/providers/componentBatchProcessor.ts new file mode 100644 index 000000000..0e82b8c91 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/providers/componentBatchProcessor.ts @@ -0,0 +1,140 @@ +// Batch translation of OpenChoreo components into Backstage entities. +// +// Components arrive FLAT per namespace (the new API lists components at +// /api/v1/namespaces/{namespaceName}/components with no project nesting); +// the owning project is read from each component's spec +// (`spec.owner.projectName`). +// +// NOTE: The original implementation enriched "Service" components via +// detail fetches to derive API entities (providesApis/consumesApis). +// Deriving API entities requires helpers (e.g. +// createApiEntitiesFromNewWorkload) that are still internal to the +// non-incremental sibling module, so API-entity derivation is deferred +// until the sibling exports them. This processor does a plain per-item +// translation with no additional API calls. + +import type { Entity } from '@backstage/catalog-model'; +import type { Config } from '@backstage/config'; +import type { LoggerService } from '@backstage/backend-plugin-api'; +import { + createOpenChoreoApiClient, + getCreatedAt, + getDeletionTimestamp, + getDescription, + getDisplayName, + getName, + getUid, + isReady, + type OpenChoreoComponents, +} from '@openchoreo/openchoreo-client-node'; +import { + ComponentTypeUtils, + type ComponentResponse, +} from '@openchoreo/backstage-plugin-common'; +import { translateComponentToEntity } from '@openchoreo/backstage-plugin-catalog-backend-module'; + +type NewComponent = OpenChoreoComponents['schemas']['Component']; + +/** The typed OpenChoreo API client returned by createOpenChoreoApiClient. */ +type OpenChoreoApiClient = ReturnType; + +/** Shared translation configuration supplied by the provider. */ +export interface ComponentBatchProcessorOptions { + /** Location key (already `provider:`-prefixed) stamped on entities. */ + locationKey: string; + /** Default owner ref used as `spec.owner`. */ + defaultOwner: string; + /** Runtime component-type utilities built from config. */ + componentTypeUtils: ComponentTypeUtils; +} + +/** + * Processes a batch of components from a single API page and translates + * them into Backstage Component entities. + */ +export class ComponentBatchProcessor { + constructor(private readonly options: ComponentBatchProcessorOptions) {} + + /** + * Translates a page of components into Component entities. + * + * @param _client - API client (currently unused; detail fetches are + * deferred — see the file-level note) + * @param components - Components from one list page + * @param namespaceName - Namespace the components belong to + * @param context - Provider context for logging + * @returns Array of translated entities + */ + async translateComponentsWithApisBatch( + _client: OpenChoreoApiClient, + components: NewComponent[], + namespaceName: string, + context: { logger: LoggerService; config: Config }, + ): Promise { + const entities: Entity[] = []; + + for (const component of components) { + const componentName = getName(component); + if (!componentName) { + context.logger.debug( + `Skipping component without a name in namespace ${namespaceName}`, + ); + continue; + } + + // The owning project lives on the component spec + // (ComponentSpec.owner.projectName). Fall back to the namespace name + // so the entity still gets a valid spec.system value. + const projectName = component.spec?.owner?.projectName; + if (!projectName) { + context.logger.debug( + `Component ${componentName} in namespace ${namespaceName} has no project reference; attributing to the namespace`, + ); + } + const effectiveProjectName = projectName ?? namespaceName; + + const componentTypeRef = component.spec?.componentType; + const componentType = + typeof componentTypeRef === 'string' + ? componentTypeRef + : componentTypeRef?.name ?? ''; + + entities.push( + translateComponentToEntity( + { + name: componentName, + displayName: getDisplayName(component), + uid: getUid(component), + type: componentType, + componentType: + typeof componentTypeRef === 'object' && componentTypeRef + ? { + kind: componentTypeRef.kind, + name: componentTypeRef.name, + } + : undefined, + status: isReady(component) ? 'Ready' : 'Not Ready', + createdAt: getCreatedAt(component), + description: getDescription(component), + deletionTimestamp: getDeletionTimestamp(component), + componentWorkflow: component.spec?.workflow + ? { + name: component.spec.workflow.name ?? '', + parameters: component.spec.workflow.parameters, + } + : undefined, + } as ComponentResponse, + namespaceName, + effectiveProjectName, + { + defaultOwner: this.options.defaultOwner, + componentTypeUtils: this.options.componentTypeUtils, + locationKey: this.options.locationKey, + }, + ), + ); + } + + return entities; + } +} diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/router/routes.test.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/router/routes.test.ts new file mode 100644 index 000000000..789f524fd --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/router/routes.test.ts @@ -0,0 +1,468 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Test suite for the incremental provider admin router. + * Drives the produced express router through supertest with a fully stubbed + * database manager, covering success payloads, unknown-provider 404s, and + * the error-handling path. + */ +import { mockErrorHandler } from '@backstage/backend-test-utils'; +import type { LoggerService } from '@backstage/backend-plugin-api'; +import express from 'express'; +import request from 'supertest'; +import type { OpenChoreoIncrementalIngestionDatabaseManager } from '../database/OpenChoreoIncrementalIngestionDatabaseManager'; +import type { IngestionRecord } from '../database/tables'; +import { IncrementalProviderRouter } from './routes'; + +function createMockManager() { + return { + healthcheck: jest.fn(), + cleanupProviders: jest.fn(), + getCurrentIngestionRecord: jest.fn(), + listProviders: jest.fn(), + triggerNextProviderAction: jest.fn(), + setProviderComplete: jest.fn(), + setProviderCanceling: jest.fn(), + updateByName: jest.fn(), + purgeAndResetProvider: jest.fn(), + getAllMarks: jest.fn(), + clearFinishedIngestions: jest.fn(), + }; +} + +function makeLogger() { + return { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + child: jest.fn().mockReturnThis(), + } as unknown as LoggerService; +} + +function makeRecord(overrides: Partial = {}): IngestionRecord { + return { + id: 'ingestion-1', + provider_name: 'myProvider', + status: 'bursting', + next_action: 'ingest', + next_action_at: new Date('2023-06-01T12:00:00.000Z'), + last_error: 'previous burst failed', + attempts: 1, + created_at: '2023-06-01T11:00:00.000Z', + ingestion_completed_at: null, + rest_completed_at: null, + completion_ticket: 'open', + ...overrides, + }; +} + +describe('IncrementalProviderRouter', () => { + let app: express.Express; + let manager: ReturnType; + let logger: LoggerService; + + beforeEach(() => { + manager = createMockManager(); + logger = makeLogger(); + const router = new IncrementalProviderRouter( + manager as unknown as OpenChoreoIncrementalIngestionDatabaseManager, + logger, + ).createRouter(); + app = express(); + app.use(router); + app.use(mockErrorHandler()); + }); + + describe('GET /incremental/health', () => { + it('reports healthy when there are no duplicate active ingestions', async () => { + manager.healthcheck.mockResolvedValue([ + { id: 'ingestion-1', provider_name: 'myProvider' }, + ]); + + const res = await request(app).get('/incremental/health'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ success: true, data: { healthy: true } }); + expect(manager.healthcheck).toHaveBeenCalledTimes(1); + }); + + it('reports the duplicated providers when duplicates exist', async () => { + manager.healthcheck.mockResolvedValue([ + { id: 'ingestion-1', provider_name: 'myProvider' }, + { id: 'ingestion-2', provider_name: 'myProvider' }, + ]); + + const res = await request(app).get('/incremental/health'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + success: false, + data: { healthy: false, duplicateIngestions: ['myProvider'] }, + error: 'Duplicate ingestions detected', + }); + }); + + it('surfaces manager failures as 500 responses', async () => { + manager.healthcheck.mockRejectedValue(new Error('db down')); + + const res = await request(app).get('/incremental/health'); + + expect(res.status).toBe(500); + }); + }); + + describe('POST /incremental/cleanup', () => { + it('returns the cleanup results', async () => { + manager.cleanupProviders.mockResolvedValue({ + ingestionsDeleted: 3, + ingestionMarksDeleted: 7, + markEntitiesDeleted: 42, + }); + + const res = await request(app).post('/incremental/cleanup'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + success: true, + data: { + ingestionsDeleted: 3, + ingestionMarksDeleted: 7, + markEntitiesDeleted: 42, + }, + }); + expect(manager.cleanupProviders).toHaveBeenCalledTimes(1); + }); + }); + + describe('GET /incremental/providers', () => { + it('lists all known providers', async () => { + manager.listProviders.mockResolvedValue(['myProvider', 'otherProvider']); + + const res = await request(app).get('/incremental/providers'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + success: true, + data: { providers: ['myProvider', 'otherProvider'] }, + }); + }); + }); + + describe('GET /incremental/providers/:provider', () => { + it('returns the current status for a provider with an open ingestion', async () => { + manager.getCurrentIngestionRecord.mockResolvedValue(makeRecord()); + + const res = await request(app).get('/incremental/providers/myProvider'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + success: true, + data: { + status: { + current_action: 'bursting', + next_action_at: '2023-06-01T12:00:00.000Z', + }, + last_error: 'previous burst failed', + }, + }); + }); + + it('reports a rest-complete provider that has no open ingestion', async () => { + manager.getCurrentIngestionRecord.mockResolvedValue(undefined); + manager.listProviders.mockResolvedValue(['myProvider']); + + const res = await request(app).get('/incremental/providers/myProvider'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + success: true, + data: { + status: { current_action: 'rest complete, waiting to start' }, + }, + }); + }); + + it('returns 404 for an unknown provider', async () => { + manager.getCurrentIngestionRecord.mockResolvedValue(undefined); + manager.listProviders.mockResolvedValue(['myProvider']); + + const res = await request(app).get('/incremental/providers/nope'); + + expect(res.status).toBe(404); + expect(res.body).toEqual({ + success: false, + error: "Provider 'nope' not found", + }); + expect(logger.error).toHaveBeenCalledWith( + 'nope - No ingestion record found in the database!', + ); + }); + }); + + describe('POST /incremental/providers/:provider/trigger', () => { + it('triggers the next action for a provider with an open ingestion', async () => { + manager.getCurrentIngestionRecord.mockResolvedValue(makeRecord()); + + const res = await request(app).post( + '/incremental/providers/myProvider/trigger', + ); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + success: true, + data: { message: 'myProvider: Next action triggered.' }, + }); + expect(manager.triggerNextProviderAction).toHaveBeenCalledWith( + 'myProvider', + ); + }); + + it('declines to trigger a restarting provider', async () => { + manager.getCurrentIngestionRecord.mockResolvedValue(undefined); + manager.listProviders.mockResolvedValue(['myProvider']); + + const res = await request(app).post( + '/incremental/providers/myProvider/trigger', + ); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + success: true, + data: { + message: 'Unable to trigger next action (provider is restarting)', + }, + }); + expect(manager.triggerNextProviderAction).not.toHaveBeenCalled(); + }); + + it('returns 404 for an unknown provider', async () => { + manager.getCurrentIngestionRecord.mockResolvedValue(undefined); + manager.listProviders.mockResolvedValue([]); + + const res = await request(app).post( + '/incremental/providers/nope/trigger', + ); + + expect(res.status).toBe(404); + expect(res.body.error).toBe("Provider 'nope' not found"); + }); + }); + + describe('POST /incremental/providers/:provider/start', () => { + it('completes a resting ingestion to start the next cycle', async () => { + manager.getCurrentIngestionRecord.mockResolvedValue( + makeRecord({ status: 'resting' }), + ); + + const res = await request(app).post( + '/incremental/providers/myProvider/start', + ); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + success: true, + data: { message: 'myProvider: Next cycle triggered.' }, + }); + expect(manager.setProviderComplete).toHaveBeenCalledWith('ingestion-1'); + expect(manager.setProviderCanceling).not.toHaveBeenCalled(); + }); + + it('cancels a non-resting ingestion to start the next cycle', async () => { + manager.getCurrentIngestionRecord.mockResolvedValue(makeRecord()); + + const res = await request(app).post( + '/incremental/providers/myProvider/start', + ); + + expect(res.status).toBe(200); + expect(manager.setProviderCanceling).toHaveBeenCalledWith('ingestion-1'); + expect(manager.setProviderComplete).not.toHaveBeenCalled(); + }); + + it('returns 404 for an unknown provider', async () => { + manager.getCurrentIngestionRecord.mockResolvedValue(undefined); + manager.listProviders.mockResolvedValue([]); + + const res = await request(app).post('/incremental/providers/nope/start'); + + expect(res.status).toBe(404); + }); + }); + + describe('POST /incremental/providers/:provider/cancel', () => { + it('cancels the open ingestion and schedules a cooldown', async () => { + manager.getCurrentIngestionRecord.mockResolvedValue(makeRecord()); + + const res = await request(app).post( + '/incremental/providers/myProvider/cancel', + ); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + success: true, + data: { message: 'myProvider: Current ingestion canceled.' }, + }); + expect(manager.updateByName).toHaveBeenCalledTimes(1); + const [provider, update] = manager.updateByName.mock.calls[0]; + expect(provider).toBe('myProvider'); + expect(update.next_action).toBe('nothing (done)'); + expect(update.status).toBe('resting'); + expect(update.ingestion_completed_at).toEqual(expect.any(Date)); + // The cancel cooldown is 24 hours from now. + expect( + (update.next_action_at as Date).getTime() - Date.now(), + ).toBeGreaterThan(23 * 60 * 60 * 1000); + expect( + (update.next_action_at as Date).getTime() - Date.now(), + ).toBeLessThanOrEqual(24 * 60 * 60 * 1000); + }); + + it('asks for patience while a restarting provider has no record', async () => { + manager.getCurrentIngestionRecord.mockResolvedValue(undefined); + manager.listProviders.mockResolvedValue(['myProvider']); + + const res = await request(app).post( + '/incremental/providers/myProvider/cancel', + ); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + success: true, + data: { + message: 'Provider is currently restarting, please wait.', + }, + }); + expect(manager.updateByName).not.toHaveBeenCalled(); + }); + }); + + describe('DELETE /incremental/providers/:provider', () => { + it('purges and resets the provider', async () => { + manager.purgeAndResetProvider.mockResolvedValue({ + provider: 'myProvider', + ingestionsDeleted: 1, + marksDeleted: 2, + markEntitiesDeleted: 3, + }); + + const res = await request(app).delete( + '/incremental/providers/myProvider', + ); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + success: true, + data: { + provider: 'myProvider', + ingestionsDeleted: 1, + marksDeleted: 2, + markEntitiesDeleted: 3, + }, + }); + expect(manager.purgeAndResetProvider).toHaveBeenCalledWith('myProvider'); + }); + }); + + describe('GET /incremental/providers/:provider/marks', () => { + it('returns all marks of the open ingestion', async () => { + manager.getCurrentIngestionRecord.mockResolvedValue(makeRecord()); + manager.getAllMarks.mockResolvedValue([ + { id: 'mark-1', sequence: 2, cursor: { page: 2 }, created_at: 't2' }, + ]); + + const res = await request(app).get( + '/incremental/providers/myProvider/marks', + ); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + success: true, + data: { + records: [ + { + id: 'mark-1', + sequence: 2, + cursor: { page: 2 }, + created_at: 't2', + }, + ], + }, + }); + expect(manager.getAllMarks).toHaveBeenCalledWith('ingestion-1'); + }); + + it('reports a restarting provider without marks', async () => { + manager.getCurrentIngestionRecord.mockResolvedValue(undefined); + manager.listProviders.mockResolvedValue(['myProvider']); + + const res = await request(app).get( + '/incremental/providers/myProvider/marks', + ); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + success: true, + data: { message: 'No records yet (provider is restarting)' }, + }); + }); + + it('returns 404 for an unknown provider', async () => { + manager.getCurrentIngestionRecord.mockResolvedValue(undefined); + manager.listProviders.mockResolvedValue([]); + + const res = await request(app).get('/incremental/providers/nope/marks'); + + expect(res.status).toBe(404); + expect(res.body.error).toBe("Provider 'nope' not found"); + }); + }); + + describe('DELETE /incremental/providers/:provider/marks', () => { + it('clears finished ingestions and reports the deletions', async () => { + manager.clearFinishedIngestions.mockResolvedValue({ + deletions: { + markEntitiesDeleted: 4, + marksDeleted: 2, + ingestionsDeleted: 1, + }, + }); + + const res = await request(app).delete( + '/incremental/providers/myProvider/marks', + ); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + success: true, + data: { + message: "Expired marks for provider 'myProvider' removed.", + deletions: { + deletions: { + markEntitiesDeleted: 4, + marksDeleted: 2, + ingestionsDeleted: 1, + }, + }, + }, + }); + expect(manager.clearFinishedIngestions).toHaveBeenCalledWith( + 'myProvider', + ); + }); + }); +}); diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/router/routes.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/router/routes.ts new file mode 100644 index 000000000..8087d59f2 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/router/routes.ts @@ -0,0 +1,272 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Router for incremental provider management endpoints. + * Provides REST API endpoints for monitoring and controlling incremental ingestion processes. + */ + +import express from 'express'; +import Router from 'express-promise-router'; +import { OpenChoreoIncrementalIngestionDatabaseManager } from '../database/OpenChoreoIncrementalIngestionDatabaseManager'; +import { LoggerService } from '@backstage/backend-plugin-api'; + +const POST_CANCEL_COOLDOWN_MS = 24 * 60 * 60 * 1000; + +export class IncrementalProviderRouter { + private manager: OpenChoreoIncrementalIngestionDatabaseManager; + private logger: LoggerService; + + constructor( + manager: OpenChoreoIncrementalIngestionDatabaseManager, + logger: LoggerService, + ) { + this.manager = manager; + this.logger = logger; + } + + createRouter(): express.Router { + const router = Router(); + router.use(express.json()); + + router.get('/incremental/health', async (_, res) => { + const records = await this.manager.healthcheck(); + const providers = records.map(record => record.provider_name); + const duplicates = [ + ...new Set(providers.filter((e, i, a) => a.indexOf(e) !== i)), + ]; + + if (duplicates.length > 0) { + res.json({ + success: false, + data: { healthy: false, duplicateIngestions: duplicates }, + error: 'Duplicate ingestions detected', + }); + } else { + res.json({ success: true, data: { healthy: true } }); + } + }); + + router.post('/incremental/cleanup', async (_, res) => { + const result = await this.manager.cleanupProviders(); + res.json({ success: true, data: result }); + }); + + router.get('/incremental/providers/:provider', async (req, res) => { + const { provider } = req.params; + const record = await this.manager.getCurrentIngestionRecord(provider); + if (record) { + res.json({ + success: true, + data: { + status: { + current_action: record.status, + next_action_at: new Date(record.next_action_at), + }, + last_error: record.last_error, + }, + }); + } else { + const providers: string[] = await this.manager.listProviders(); + if (providers.includes(provider)) { + res.json({ + success: true, + data: { + status: { + current_action: 'rest complete, waiting to start', + }, + }, + }); + } else { + this.logger.error( + `${provider} - No ingestion record found in the database!`, + ); + res.status(404).json({ + success: false, + error: `Provider '${provider}' not found`, + }); + } + } + }); + + router.post( + `/incremental/providers/:provider/trigger`, + async (req, res) => { + const { provider } = req.params; + const record = await this.manager.getCurrentIngestionRecord(provider); + if (record) { + await this.manager.triggerNextProviderAction(provider); + res.json({ + success: true, + data: { message: `${provider}: Next action triggered.` }, + }); + } else { + const providers: string[] = await this.manager.listProviders(); + if (providers.includes(provider)) { + this.logger.debug( + `${provider} - No ingestion record, provider is restarting`, + ); + res.json({ + success: true, + data: { + message: + 'Unable to trigger next action (provider is restarting)', + }, + }); + } else { + res.status(404).json({ + success: false, + error: `Provider '${provider}' not found`, + }); + } + } + }, + ); + + router.post(`/incremental/providers/:provider/start`, async (req, res) => { + const { provider } = req.params; + + const record = await this.manager.getCurrentIngestionRecord(provider); + if (record) { + const ingestionId = record.id; + if (record.status === 'resting') { + await this.manager.setProviderComplete(ingestionId); + } else { + await this.manager.setProviderCanceling(ingestionId); + } + res.json({ + success: true, + data: { message: `${provider}: Next cycle triggered.` }, + }); + } else { + const providers: string[] = await this.manager.listProviders(); + if (providers.includes(provider)) { + this.logger.debug( + `${provider} - No ingestion record, provider is already restarting`, + ); + res.json({ + success: true, + data: { message: 'Provider is already restarting' }, + }); + } else { + res.status(404).json({ + success: false, + error: `Provider '${provider}' not found`, + }); + } + } + }); + + router.get(`/incremental/providers`, async (_req, res) => { + const providers = await this.manager.listProviders(); + + res.json({ + success: true, + data: { providers }, + }); + }); + + router.post(`/incremental/providers/:provider/cancel`, async (req, res) => { + const { provider } = req.params; + const record = await this.manager.getCurrentIngestionRecord(provider); + if (record) { + const next_action_at = new Date(); + next_action_at.setTime( + next_action_at.getTime() + POST_CANCEL_COOLDOWN_MS, + ); + await this.manager.updateByName(provider, { + next_action: 'nothing (done)', + ingestion_completed_at: new Date(), + next_action_at, + status: 'resting', + }); + res.json({ + success: true, + data: { message: `${provider}: Current ingestion canceled.` }, + }); + } else { + const providers: string[] = await this.manager.listProviders(); + if (providers.includes(provider)) { + this.logger.debug( + `${provider} - No ingestion record, provider is restarting`, + ); + res.json({ + success: true, + data: { message: 'Provider is currently restarting, please wait.' }, + }); + } else { + res.status(404).json({ + success: false, + error: `Provider '${provider}' not found`, + }); + } + } + }); + + router.delete('/incremental/providers/:provider', async (req, res) => { + const { provider } = req.params; + const result = await this.manager.purgeAndResetProvider(provider); + res.json({ success: true, data: result }); + }); + + router.get(`/incremental/providers/:provider/marks`, async (req, res) => { + const { provider } = req.params; + const record = await this.manager.getCurrentIngestionRecord(provider); + if (record) { + const id = record.id; + const records = await this.manager.getAllMarks(id); + res.json({ success: true, data: { records } }); + } else { + const providers: string[] = await this.manager.listProviders(); + if (providers.includes(provider)) { + this.logger.debug( + `${provider} - No ingestion record, provider is restarting`, + ); + res.json({ + success: true, + data: { message: 'No records yet (provider is restarting)' }, + }); + } else { + this.logger.error( + `${provider} - No ingestion record found in the database!`, + ); + res.status(404).json({ + success: false, + error: `Provider '${provider}' not found`, + }); + } + } + }); + + router.delete( + `/incremental/providers/:provider/marks`, + async (req, res) => { + const { provider } = req.params; + const deletions = await this.manager.clearFinishedIngestions(provider); + + res.json({ + success: true, + data: { + message: `Expired marks for provider '${provider}' removed.`, + deletions, + }, + }); + }, + ); + + return router; + } +} diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/testUtils/mockApi.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/testUtils/mockApi.ts new file mode 100644 index 000000000..d65011c98 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/testUtils/mockApi.ts @@ -0,0 +1,312 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Shared test doubles for the incremental ingestion test suites. + * + * - `createMockClient` builds a cursor-following stand-in for the typed + * API client returned by `createOpenChoreoApiClient`. Routes map an API + * path (templated or concrete, e.g. `/api/v1/namespaces/my-ns/projects`) + * to the list of pages the "server" serves for it; `GET` walks the pages + * by cursor chain and records every (path, params) invocation. + * - `mockOpenChoreoClientNodeModule` and `mockCatalogBackendModule` + * produce `jest.mock` module factories for the client package and the + * non-incremental sibling module respectively. + * - `mockNamespace` / `mockProject` / `mockComponent` build K8s-style + * resource fixtures shaped like the new OpenChoreo API responses. + */ + +import type { OpenChoreoComponents } from '@openchoreo/openchoreo-client-node'; + +type NewNamespace = OpenChoreoComponents['schemas']['Namespace']; +type NewProject = OpenChoreoComponents['schemas']['Project']; +type NewComponent = OpenChoreoComponents['schemas']['Component']; + +/** One page of a cursor-paginated list response. */ +export interface MockPage { + items: unknown[]; + /** Cursor the server reports as the start of the next page. */ + nextCursor?: string; +} + +/** A scripted page that makes the client return an error response. */ +export interface MockErrorPage { + status: number; + statusText?: string; + error?: { message: string }; +} + +/** The scripted pages served for one route. */ +export type MockRoutePages = Array; + +/** A recorded `client.GET(path, { params })` invocation. */ +export interface MockGetCall { + path: string; + params?: { + path?: Record; + query?: Record; + }; +} + +/** A mock typed API client that walks scripted pages by cursor. */ +export interface MockApiClient { + GET: jest.Mock; + /** Every (path, params) pair the client was called with, in order. */ + calls: MockGetCall[]; +} + +/** Recursively optional overrides for the fixture builders. */ +type DeepPartial = T extends (infer U)[] + ? DeepPartial[] + : T extends object + ? { [K in keyof T]?: DeepPartial } + : T; + +function isErrorPage(page: MockPage | MockErrorPage): page is MockErrorPage { + return (page as MockErrorPage).status !== undefined; +} + +/** + * Resolves the page for a request cursor: `undefined` starts at the first + * page; a cursor equal to `pages[i].nextCursor` advances to `pages[i + 1]`. + */ +function resolvePage( + pages: MockRoutePages, + cursor: string | undefined, +): MockPage | MockErrorPage | undefined { + if (cursor === undefined) { + return pages[0]; + } + for (let i = 0; i < pages.length; i++) { + const page = pages[i]; + if (!isErrorPage(page) && page.nextCursor === cursor) { + return pages[i + 1]; + } + } + return undefined; +} + +/** + * Looks up the scripted pages for a call. Concrete path keys (with path + * params substituted, e.g. `/api/v1/namespaces/ns-1/projects`) win over + * templated keys so each namespace can serve its own page set. + */ +function resolveRoute( + routes: Record, + path: string, + params?: MockGetCall['params'], +): MockRoutePages | undefined { + if (params?.path) { + const concrete = path.replace(/\{(\w+)\}/g, (placeholder, key: string) => + params.path && key in params.path + ? String(params.path[key]) + : placeholder, + ); + if (routes[concrete]) { + return routes[concrete]; + } + } + return routes[path]; +} + +function errorResult(page: MockErrorPage) { + return { + data: undefined, + error: page.error ?? { message: `Request failed with ${page.status}` }, + response: { + ok: false, + status: page.status, + statusText: page.statusText ?? `HTTP ${page.status}`, + }, + }; +} + +/** The most recently created mock client, served by the module factory. */ +let activeClient: MockApiClient | undefined; + +/** + * Creates a cursor-following mock API client and installs it as the client + * returned by the mocked `createOpenChoreoApiClient`. + */ +export function createMockClient( + routes: Record, +): MockApiClient { + const calls: MockGetCall[] = []; + const client: MockApiClient = { + calls, + GET: jest.fn( + async ( + path: string, + options?: { params?: MockGetCall['params'] }, + ): Promise => { + const params = options?.params; + calls.push({ path, params }); + + const pages = resolveRoute(routes, path, params); + const cursor = + typeof params?.query?.cursor === 'string' + ? params.query.cursor + : undefined; + const page = pages ? resolvePage(pages, cursor) : undefined; + + if (!page) { + return errorResult({ + status: 400, + error: { + message: `No mock page for cursor '${cursor}' on ${path}`, + }, + }); + } + if (isErrorPage(page)) { + return errorResult(page); + } + return { + data: { + items: page.items, + pagination: page.nextCursor ? { nextCursor: page.nextCursor } : {}, + }, + error: undefined, + response: { ok: true, status: 200, statusText: 'OK' }, + }; + }, + ), + }; + activeClient = client; + return client; +} + +/** + * `jest.mock` factory for '@openchoreo/openchoreo-client-node'. Keeps the + * real resource helpers (getName, isReady, ...) and replaces only the + * client factory with one returning the mock installed by + * {@link createMockClient}. + */ +export function mockOpenChoreoClientNodeModule() { + return { + ...jest.requireActual>( + '@openchoreo/openchoreo-client-node', + ), + createOpenChoreoApiClient: jest.fn(() => { + if (!activeClient) { + throw new Error( + 'createMockClient() must be called before the provider runs', + ); + } + return activeClient; + }), + }; +} + +/** + * `jest.mock` factory for '@openchoreo/backstage-plugin-catalog-backend-module'. + * The translators echo the source resource name into minimal + * `{ kind, metadata: { name } }` entities so traversal tests can assert on + * kinds and names without depending on the real translation rules. + */ +export function mockCatalogBackendModule() { + return { + translateNamespaceToDomainEntity: jest.fn( + (namespace: { name: string }) => ({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'Domain', + metadata: { name: namespace.name }, + }), + ), + translateProjectToEntity: jest.fn((project: { name: string }) => ({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'System', + metadata: { name: project.name }, + })), + translateComponentToEntity: jest.fn((component: { name: string }) => ({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { name: component.name }, + })), + }; +} + +function k8sMeta(name: string) { + return { + name, + namespace: 'test-namespace', + uid: `uid-${name}`, + creationTimestamp: '2025-01-06T10:00:00Z', + labels: {}, + annotations: { + 'openchoreo.dev/display-name': name, + 'openchoreo.dev/description': `${name} description`, + }, + }; +} + +const readyCondition = { + type: 'Ready', + status: 'True', + lastTransitionTime: '2025-01-06T10:00:05Z', + reason: 'Reconciled', + message: 'Resource is ready', +}; + +/** Builds a K8s-style Namespace fixture. */ +export function mockNamespace( + name: string, + overrides: DeepPartial = {}, +): NewNamespace { + return { + ...overrides, + metadata: { ...k8sMeta(name), ...overrides.metadata }, + status: { phase: 'Active', ...overrides.status }, + } as NewNamespace; +} + +/** Builds a K8s-style Project fixture. */ +export function mockProject( + name: string, + overrides: DeepPartial = {}, +): NewProject { + return { + ...overrides, + metadata: { ...k8sMeta(name), ...overrides.metadata }, + spec: { + deploymentPipelineRef: { kind: 'DeploymentPipeline', name: 'default' }, + type: { kind: 'ProjectType', name: 'default' }, + ...overrides.spec, + }, + } as NewProject; +} + +/** + * Builds a K8s-style Component fixture. The default spec attributes the + * component to the project `${name}-project`; override + * `spec.owner.projectName` (or pass `spec.owner: {}` to exercise the + * namespace fallback). + */ +export function mockComponent( + name: string, + overrides: DeepPartial = {}, +): NewComponent { + return { + ...overrides, + metadata: { ...k8sMeta(name), ...overrides.metadata }, + spec: { + owner: { projectName: `${name}-project` }, + componentType: { kind: 'ComponentType', name: 'deployment/service' }, + autoDeploy: false, + ...overrides.spec, + }, + status: { conditions: [readyCondition], ...overrides.status }, + } as NewComponent; +} diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/types.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/types.ts new file mode 100644 index 000000000..5c85af6d4 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/types.ts @@ -0,0 +1,201 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Type definitions for incremental entity providers. + * Defines interfaces and types for burst-based, resumable entity ingestion. + */ + +import { + LoggerService, + SchedulerServiceTaskFunction, +} from '@backstage/backend-plugin-api'; +import type { + DeferredEntity, + EntityProviderConnection, +} from '@backstage/plugin-catalog-node'; +import { EventParams } from '@backstage/plugin-events-node'; +import { HumanDuration } from '@backstage/types'; +import { OpenChoreoIncrementalIngestionDatabaseManager } from './database/OpenChoreoIncrementalIngestionDatabaseManager'; + +/** + * Ingest entities into the catalog in bite-sized chunks. + * + * A Normal `EntityProvider` allows you to introduce entities into the + * processing pipeline by calling an `applyMutation()` on the full set + * of entities. However, this is not great when the number of entities + * that you have to keep track of is extremely large because it + * entails having all of them in memory at once. An + * `IncrementalEntityProvider` by contrast allows you to provide + * batches of entities in sequence so that you never need to have more + * than a few hundred in memory at a time. + * + * @public + */ +export interface IncrementalEntityProvider { + /** + * This name must be unique between all of the entity providers + * operating in the catalog. + */ + getProviderName(): string; + + /** + * Return a single page of entities from a specific point in the + * ingestion. + * + * @param context - anything needed in order to fetch a single page. + * @param cursor - a unique value identifying the page to ingest. + * @returns The entities to be ingested, as well as the cursor of + * the next page after this one. + */ + next( + context: TContext, + cursor?: TCursor, + ): Promise>; + + /** + * Do any setup and teardown necessary in order to provide the + * context for fetching pages. This should always invoke `burst` in + * order to fetch the individual pages. + * + * @param burst - a function which performs a series of iterations + */ + around(burst: (context: TContext) => Promise): Promise; + + /** + * If set, the IncrementalEntityProvider will receive and respond to + * events. + * + * This system acts as a wrapper for the Backstage events bus, and + * requires the events backend to function. It does not provide its + * own events backend. See {@link https://github.com/backstage/backstage/tree/master/plugins/events-backend}. + */ + eventHandler?: { + /** + * This method accepts an incoming event for the provider, and + * optionally maps the payload to an object containing a delta + * mutation. + * + * If a delta result is returned by this method, it will be ingested + * automatically by the provider. Alternatively, if an "ignored" result is + * returned, then it is understood that this event should not cause anything + * to be ingested. + */ + onEvent: (params: EventParams) => Promise; + + /** + * This method returns an array of topics for the IncrementalEntityProvider + * to respond to. + */ + supportsEventTopics: () => string[]; + }; +} + +/** + * An object returned by event handler to indicate whether to ignore the event + * or to apply a delta in response to the event. + * + * @public + */ +export type IncrementalEntityEventResult = + | { + type: 'ignored'; + } + | { + type: 'delta'; + added: DeferredEntity[]; + removed: { entityRef: string }[]; + }; + +/** + * Value returned by an {@link IncrementalEntityProvider} to provide a + * single page of entities to ingest. + * + * @public + */ +export type EntityIteratorResult = + | { + done: false; + entities: DeferredEntity[]; + cursor: T; + } + | { + done: true; + entities?: DeferredEntity[]; + cursor?: T; + }; + +/** @public */ +export interface IncrementalEntityProviderOptions { + /** + * Entities are ingested in bursts. This interval determines how + * much time to wait in between each burst. + */ + burstInterval: HumanDuration; + + /** + * Entities are ingested in bursts. This value determines how long + * to keep ingesting within each burst. + */ + burstLength: HumanDuration; + + /** + * After a successful ingestion, the incremental entity provider + * will rest for this period of time before starting to ingest + * again. + */ + restLength: HumanDuration; + + /** + * In the event of an error during an ingestion burst, the backoff + * determines how soon it will be retried. E.g. + * `[{ minutes: 1}, { minutes: 5}, {minutes: 30 }, { hours: 3 }]` + */ + backoff?: HumanDuration[]; + + /** + * If an error occurs at a data source that results in a large + * number of assets being inadvertently removed, it will result in + * Backstage removing all associated entities. To avoid that, set + * a percentage of entities past which removal will be disallowed. + */ + rejectRemovalsAbovePercentage?: number; + + /** + * Similar to the rejectRemovalsAbovePercentage, this option + * prevents removals in circumstances where a data source has + * improperly returned 0 assets. If set to `true`, Backstage will + * reject removals when that happens. + */ + rejectEmptySourceCollections?: boolean; +} + +export interface IterationEngine { + taskFn: SchedulerServiceTaskFunction; +} + +export interface IterationEngineOptions { + logger: LoggerService; + connection: EntityProviderConnection; + manager: OpenChoreoIncrementalIngestionDatabaseManager; + provider: IncrementalEntityProvider; + restLength: HumanDuration; + burstLength: HumanDuration; + ready: Promise; + backoff?: IncrementalEntityProviderOptions['backoff']; + rejectRemovalsAbovePercentage?: number; + rejectEmptySourceCollections?: boolean; +} diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/utils/ConfigValidator.test.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/utils/ConfigValidator.test.ts new file mode 100644 index 000000000..f8e7fb180 --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/utils/ConfigValidator.test.ts @@ -0,0 +1,223 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Test suite for ConfigValidator. + * Verifies full-config validation, per-field range enforcement, defaults for + * absent sections, and the business-rule checks layered on top of the zod + * schema. Field-level failures are surfaced through the wrapped ZodError + * cause of OpenChoreoIncrementalIngestionError. + */ +import { mockServices } from '@backstage/backend-test-utils'; +import { ConfigReader } from '@backstage/config'; +import { ZodError } from 'zod'; +import { OpenChoreoIncrementalIngestionError } from '../database/errors'; +import { ConfigValidator } from './ConfigValidator'; + +const VALID_INCREMENTAL_CONFIG = { + burstLength: 10, + burstInterval: 30, + restLength: 30, + chunkSize: 100, + rejectRemovalsAbovePercentage: 20, + rejectEmptySourceCollections: true, + maxConcurrentRequests: 5, + batchDelayMs: 100, +}; + +function makeConfig(incrementalOverrides: Record = {}) { + return new ConfigReader({ + openchoreo: { + api: { + baseUrl: 'https://api.openchoreo.example.com', + token: 'secret-token', + }, + incremental: { + ...VALID_INCREMENTAL_CONFIG, + ...incrementalOverrides, + }, + }, + }); +} + +/** Extracts the offending leaf field name from the wrapped ZodError cause. */ +function zodIssuePath(error: unknown): string { + const cause = (error as OpenChoreoIncrementalIngestionError).cause; + expect(cause).toBeInstanceOf(ZodError); + const issues = (cause as ZodError).issues; + expect(issues.length).toBeGreaterThan(0); + const path = issues[0].path; + return String(path[path.length - 1]); +} + +describe('ConfigValidator', () => { + const logger = mockServices.logger.mock(); + + it('accepts a valid full configuration', () => { + const result = ConfigValidator.validateConfig(makeConfig(), logger); + + expect(result).toEqual({ + openchoreo: { + api: { + baseUrl: 'https://api.openchoreo.example.com', + token: 'secret-token', + }, + incremental: { + burstLength: 10, + burstInterval: 30, + restLength: 30, + chunkSize: 100, + rejectRemovalsAbovePercentage: 20, + rejectEmptySourceCollections: true, + maxConcurrentRequests: 5, + batchDelayMs: 100, + }, + }, + }); + }); + + it.each([ + ['chunkSize', 101], + ['burstLength', 0], + ['burstInterval', 4], + ['restLength', 1441], + ['maxConcurrentRequests', 51], + ['batchDelayMs', 10_001], + ])( + 'rejects an out-of-range %s and names the field via the zod cause', + (field, value) => { + let thrown: unknown; + try { + ConfigValidator.validateConfig(makeConfig({ [field]: value }), logger); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(OpenChoreoIncrementalIngestionError); + const error = thrown as OpenChoreoIncrementalIngestionError; + expect(error.name).toBe('OpenChoreoIncrementalIngestionError'); + expect(error.code).toBe('CONFIG_VALIDATION_ERROR'); + expect(error.message).toContain('Configuration validation failed'); + expect(zodIssuePath(error)).toBe(field); + }, + ); + + it('rejects values the config layer cannot convert', () => { + expect(() => + ConfigValidator.validateConfig( + makeConfig({ burstLength: 'ten' }), + logger, + ), + ).toThrow( + expect.objectContaining({ + code: 'CONFIG_VALIDATION_ERROR', + message: expect.stringContaining('Unable to convert'), + }), + ); + expect(() => + ConfigValidator.validateConfig( + makeConfig({ maxConcurrentRequests: 'many' }), + logger, + ), + ).toThrow(OpenChoreoIncrementalIngestionError); + }); + + it('yields an empty openchoreo section when nothing is configured', () => { + const result = ConfigValidator.validateConfig(new ConfigReader({}), logger); + expect(result).toEqual({ openchoreo: {} }); + }); + + it('applies schema defaults when the incremental section is empty', () => { + const result = ConfigValidator.validateConfig( + new ConfigReader({ openchoreo: { incremental: {} } }), + logger, + ); + + expect(result.openchoreo.incremental).toEqual({ + burstLength: 10, + burstInterval: 30, + restLength: 30, + chunkSize: 100, + rejectEmptySourceCollections: false, + maxConcurrentRequests: 5, + batchDelayMs: 100, + }); + }); + + // Business-rule failures keep their own error code; validateConfig no + // longer re-wraps them as CONFIG_VALIDATION_ERROR (which swallowed it). + it('rejects burst lengths that are not shorter than the burst interval', () => { + expect(() => + ConfigValidator.validateConfig( + makeConfig({ burstLength: 30, burstInterval: 30 }), + logger, + ), + ).toThrow( + expect.objectContaining({ + code: 'INVALID_BURST_TIMING', + message: expect.stringContaining('burstLength'), + }), + ); + }); + + it('rejects non-http API base URLs that still parse as URLs', () => { + expect(() => + ConfigValidator.validateConfig( + new ConfigReader({ + openchoreo: { + api: { baseUrl: 'ftp://api.openchoreo.example.com' }, + // The API base URL rule is only reached once an incremental + // section is present. + incremental: {}, + }, + }), + logger, + ), + ).toThrow( + expect.objectContaining({ + code: 'INVALID_API_BASE_URL', + message: expect.stringContaining('http://'), + }), + ); + }); + + it('exposes defaults and merges user overrides', () => { + const defaults = ConfigValidator.getDefaultConfig(); + expect(defaults.openchoreo!.incremental).toMatchObject({ + burstLength: 10, + burstInterval: 30, + restLength: 30, + chunkSize: 50, + maxConcurrentRequests: 5, + }); + + const merged = ConfigValidator.mergeWithDefaults({ + openchoreo: { + incremental: { ...VALID_INCREMENTAL_CONFIG, chunkSize: 25 }, + }, + }); + expect(merged.openchoreo.incremental).toEqual({ + burstLength: 10, + burstInterval: 30, + restLength: 30, + chunkSize: 25, + rejectRemovalsAbovePercentage: 20, + rejectEmptySourceCollections: true, + maxConcurrentRequests: 5, + batchDelayMs: 100, + }); + }); +}); diff --git a/plugins/catalog-backend-module-openchoreo-incremental/src/utils/ConfigValidator.ts b/plugins/catalog-backend-module-openchoreo-incremental/src/utils/ConfigValidator.ts new file mode 100644 index 000000000..ec3fb1dcc --- /dev/null +++ b/plugins/catalog-backend-module-openchoreo-incremental/src/utils/ConfigValidator.ts @@ -0,0 +1,275 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { LoggerService } from '@backstage/backend-plugin-api'; +import { + openchoreoIncrementalConfigValidation, + OpenChoreoIncrementalConfig, +} from '../config'; +import { OpenChoreoIncrementalIngestionError } from '../database/errors'; + +/** + * Utility class for validating OpenChoreo incremental plugin configuration. + */ +export class ConfigValidator { + /** + * Validates the complete OpenChoreo configuration. + * + * @param config - The Backstage configuration object + * @param logger - Logger service for reporting validation issues + * @returns Validated configuration object + * @throws OpenChoreoIncrementalIngestionError for invalid configuration + */ + static validateConfig( + config: Config, + logger: LoggerService, + ): OpenChoreoIncrementalConfig { + try { + // Extract the raw configuration data + const rawConfig = this.extractRawConfig(config); + + // Validate using Zod schema + const validatedConfig = openchoreoIncrementalConfigValidation.parse( + rawConfig, + ) as OpenChoreoIncrementalConfig; + + // Apply additional business logic validation + this.validateBusinessRules(validatedConfig, logger); + + return validatedConfig; + } catch (error) { + if ( + error instanceof OpenChoreoIncrementalIngestionError && + error.code !== 'CONFIG_VALIDATION_ERROR' + ) { + // Business-rule failures (INVALID_BURST_TIMING, INVALID_API_BASE_URL) + // carry their own code; re-wrapping them would swallow it. + throw error; + } + + if (error instanceof Error && error.name === 'ZodError') { + const zodError = error as { + issues?: { path: (string | number)[]; message: string }[]; + }; + const errorMessages = + zodError.issues + ?.map( + issue => + `${issue.path?.join('.') || 'unknown'}: ${issue.message}`, + ) + .join(', ') || 'Unknown validation error'; + + throw new OpenChoreoIncrementalIngestionError( + `Configuration validation failed: ${errorMessages}`, + 'CONFIG_VALIDATION_ERROR', + error, + ); + } + + throw new OpenChoreoIncrementalIngestionError( + `Failed to validate configuration: ${ + error instanceof Error ? error.message : String(error) + }`, + 'CONFIG_VALIDATION_ERROR', + error instanceof Error ? error : undefined, + ); + } + } + + /** + * Extracts raw configuration data from Backstage config object. + * + * @param config - The Backstage configuration object + * @returns Raw configuration data + */ + private static extractRawConfig(config: Config): any { + // Initialize with empty openchoreo object to ensure it's always present + const rawConfig: any = { + openchoreo: {}, + }; + + // Extract OpenChoreo API configuration + if (config.has('openchoreo.api')) { + rawConfig.openchoreo = { + ...rawConfig.openchoreo, + api: { + baseUrl: config.getString('openchoreo.api.baseUrl'), + ...(config.has('openchoreo.api.token') && { + token: config.getString('openchoreo.api.token'), + }), + }, + }; + } + + // Extract OpenChoreo incremental configuration + if (config.has('openchoreo.incremental')) { + const incrementalConfig = config.getConfig('openchoreo.incremental'); + + rawConfig.openchoreo = { + ...rawConfig.openchoreo, + incremental: { + burstLength: incrementalConfig.getOptionalNumber('burstLength'), + burstInterval: incrementalConfig.getOptionalNumber('burstInterval'), + restLength: incrementalConfig.getOptionalNumber('restLength'), + chunkSize: incrementalConfig.getOptionalNumber('chunkSize'), + backoff: undefined, // TODO: Implement proper backoff array parsing + rejectRemovalsAbovePercentage: incrementalConfig.getOptionalNumber( + 'rejectRemovalsAbovePercentage', + ), + rejectEmptySourceCollections: incrementalConfig.getOptionalBoolean( + 'rejectEmptySourceCollections', + ), + maxConcurrentRequests: incrementalConfig.getOptionalNumber( + 'maxConcurrentRequests', + ), + batchDelayMs: incrementalConfig.getOptionalNumber('batchDelayMs'), + }, + }; + } + + return rawConfig; + } + + /** + * Validates additional business rules beyond schema validation. + * + * @param config - Validated configuration object + * @param logger - Logger service for warnings + */ + private static validateBusinessRules( + config: OpenChoreoIncrementalConfig, + logger: LoggerService, + ): void { + const incremental = config.openchoreo.incremental; + + if (!incremental) { + return; // No incremental config to validate + } + + // Validate timing relationships + if (incremental.burstLength >= incremental.burstInterval) { + throw new OpenChoreoIncrementalIngestionError( + `burstLength (${incremental.burstLength}s) must be less than burstInterval (${incremental.burstInterval}s) to ensure proper burst/rest cycle. Current configuration would cause overlapping or continuous bursts.`, + 'INVALID_BURST_TIMING', + ); + } + + // Validate chunk size vs burst length + const maxEntitiesPerBurst = incremental.burstLength * 10; // Rough estimate + if (incremental.chunkSize > maxEntitiesPerBurst) { + logger.warn( + `chunkSize (${incremental.chunkSize}) may be too large for burstLength (${incremental.burstLength}s). Consider reducing chunk size or increasing burst length.`, + ); + } + + // Validate backoff configuration + if (incremental.backoff && incremental.backoff.length > 0) { + if (incremental.backoff.some(delay => delay <= 0)) { + throw new OpenChoreoIncrementalIngestionError( + 'All backoff durations must be positive numbers', + 'INVALID_BACKOFF_CONFIG', + ); + } + + if (incremental.backoff.length > 10) { + logger.warn( + `Backoff array has ${incremental.backoff.length} entries, which may be excessive. Consider using fewer, longer delays.`, + ); + } + } + + // Validate removal percentage + if (incremental.rejectRemovalsAbovePercentage !== undefined) { + if ( + incremental.rejectRemovalsAbovePercentage < 0 || + incremental.rejectRemovalsAbovePercentage > 100 + ) { + throw new OpenChoreoIncrementalIngestionError( + 'rejectRemovalsAbovePercentage must be between 0 and 100', + 'INVALID_REMOVAL_THRESHOLD', + ); + } + + if (incremental.rejectRemovalsAbovePercentage > 50) { + logger.warn( + `rejectRemovalsAbovePercentage (${incremental.rejectRemovalsAbovePercentage}%) is very high. This may prevent legitimate removals.`, + ); + } + } + + // Validate API configuration + if (config.openchoreo.api) { + const { baseUrl } = config.openchoreo.api; + + if (!baseUrl.startsWith('http://') && !baseUrl.startsWith('https://')) { + throw new OpenChoreoIncrementalIngestionError( + 'openchoreo.api.baseUrl must start with http:// or https://', + 'INVALID_API_BASE_URL', + ); + } + + if (baseUrl.endsWith('/')) { + logger.warn( + 'openchoreo.api.baseUrl should not end with a slash. Trailing slash will be removed.', + ); + } + } + } + + /** + * Gets default configuration values. + * + * @returns Default configuration object + */ + static getDefaultConfig(): Partial { + return { + openchoreo: { + incremental: { + burstLength: 10, + burstInterval: 30, + restLength: 30, + chunkSize: 50, + rejectEmptySourceCollections: false, + maxConcurrentRequests: 5, + batchDelayMs: 100, + }, + }, + }; + } + + /** + * Merges user configuration with defaults. + * + * @param userConfig - User-provided configuration + * @returns Merged configuration + */ + static mergeWithDefaults( + userConfig: Partial, + ): OpenChoreoIncrementalConfig { + const defaults = this.getDefaultConfig(); + + return { + openchoreo: { + api: userConfig.openchoreo?.api || defaults.openchoreo?.api, + incremental: { + ...defaults.openchoreo!.incremental!, + ...userConfig.openchoreo?.incremental, + }, + }, + } as OpenChoreoIncrementalConfig; + } +} diff --git a/plugins/catalog-backend-module-openchoreo/src/module.ts b/plugins/catalog-backend-module-openchoreo/src/module.ts index b67478ae2..d45bdea82 100644 --- a/plugins/catalog-backend-module-openchoreo/src/module.ts +++ b/plugins/catalog-backend-module-openchoreo/src/module.ts @@ -110,6 +110,16 @@ export const catalogModuleOpenchoreo = createBackendModule({ const eventsEnabled = openchoreoConfig?.getOptionalBoolean('events.enabled') ?? true; + // Incremental ingestion (registered by the portal from + // @openchoreo/backstage-plugin-catalog-backend-module-openchoreo-incremental) + // replaces this module's scheduled full-sync provider; when it is + // enabled the scheduled provider stands down so entities are not + // double-ingested. + const incrementalIngestionEnabled = + openchoreoConfig?.getOptionalBoolean( + 'features.incrementalIngestion.enabled', + ) ?? false; + const taskRunner = scheduler.createScheduledTaskRunner({ frequency: { seconds: frequency }, timeout: { seconds: timeout }, @@ -205,18 +215,24 @@ export const catalogModuleOpenchoreo = createBackendModule({ // Register the scheduled OpenChoreo entity provider. When // events are disabled we pass `undefined` so the provider's // `connect()` skips its event subscriptions. - catalog.addEntityProvider( - new OpenChoreoEntityProvider( - taskRunner, - logger, - config, - tokenService, - eventsEnabled ? events : undefined, - catalogService, - auth, - urlReader, - ), - ); + if (incrementalIngestionEnabled) { + logger.info( + 'Incremental ingestion enabled (openchoreo.features.incrementalIngestion.enabled=true); scheduled OpenChoreoEntityProvider standing down', + ); + } else { + catalog.addEntityProvider( + new OpenChoreoEntityProvider( + taskRunner, + logger, + config, + tokenService, + eventsEnabled ? events : undefined, + catalogService, + auth, + urlReader, + ), + ); + } // Create and register the ScaffolderEntityProvider for immediate insertions // Pass 'OpenChoreoEntityProvider' so it uses the same location key bucket diff --git a/plugins/catalog-backend-module-openchoreo/src/provider/OpenChoreoEntityProvider.ts b/plugins/catalog-backend-module-openchoreo/src/provider/OpenChoreoEntityProvider.ts index 456f284aa..a06672d93 100644 --- a/plugins/catalog-backend-module-openchoreo/src/provider/OpenChoreoEntityProvider.ts +++ b/plugins/catalog-backend-module-openchoreo/src/provider/OpenChoreoEntityProvider.ts @@ -547,11 +547,12 @@ export class OpenChoreoEntityProvider implements EntityProvider { for (const ns of namespaces) { const nsName = getName(ns)!; try { - const workflowplanes = await fetchAllPages(() => + const workflowplanes = await fetchAllPages(cursor => client .GET('/api/v1/namespaces/{namespaceName}/workflowplanes', { params: { path: { namespaceName: nsName }, + query: { limit: 100, cursor }, }, }) .then(res => { @@ -587,11 +588,12 @@ export class OpenChoreoEntityProvider implements EntityProvider { const nsName = getName(ns)!; try { const observabilityplanes = - await fetchAllPages(() => + await fetchAllPages(cursor => client .GET('/api/v1/namespaces/{namespaceName}/observabilityplanes', { params: { path: { namespaceName: nsName }, + query: { limit: 100, cursor }, }, }) .then(res => { @@ -663,20 +665,25 @@ export class OpenChoreoEntityProvider implements EntityProvider { >(); try { - const pipelines = await fetchAllPages(() => - client - .GET('/api/v1/namespaces/{namespaceName}/deploymentpipelines', { - params: { - path: { namespaceName: nsName }, - }, - }) - .then(res => { - if (res.error) - throw new Error( - `Failed to fetch deployment pipelines for ${nsName}`, - ); - return res.data; - }), + const pipelines = await fetchAllPages( + cursor => + client + .GET( + '/api/v1/namespaces/{namespaceName}/deploymentpipelines', + { + params: { + path: { namespaceName: nsName }, + query: { limit: 100, cursor }, + }, + }, + ) + .then(res => { + if (res.error) + throw new Error( + `Failed to fetch deployment pipelines for ${nsName}`, + ); + return res.data; + }), ); // The DP↔Project relation pair is emitted by diff --git a/plugins/openchoreo-common/config.d.ts b/plugins/openchoreo-common/config.d.ts index ca2151fec..55958319d 100644 --- a/plugins/openchoreo-common/config.d.ts +++ b/plugins/openchoreo-common/config.d.ts @@ -116,6 +116,21 @@ export interface Config { */ enabled?: boolean; }; + + /** + * Incremental catalog ingestion. + * When enabled, the burst-based incremental provider replaces the + * scheduled full-sync entity provider (which stands down). + * @deepVisibility frontend + */ + incrementalIngestion?: { + /** + * Enable or disable incremental catalog ingestion. + * When false (default), the scheduled OpenChoreoEntityProvider runs. + * @visibility frontend + */ + enabled?: boolean; + }; }; /** diff --git a/yarn.lock b/yarn.lock index ade1497ca..6ab9af362 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9803,6 +9803,37 @@ __metadata: languageName: unknown linkType: soft +"@openchoreo/backstage-plugin-catalog-backend-module-openchoreo-incremental@workspace:^, @openchoreo/backstage-plugin-catalog-backend-module-openchoreo-incremental@workspace:plugins/catalog-backend-module-openchoreo-incremental": + version: 0.0.0-use.local + resolution: "@openchoreo/backstage-plugin-catalog-backend-module-openchoreo-incremental@workspace:plugins/catalog-backend-module-openchoreo-incremental" + dependencies: + "@backstage/backend-defaults": "npm:^0.17.1" + "@backstage/backend-plugin-api": "npm:^1.9.1" + "@backstage/backend-test-utils": "npm:^1.11.3" + "@backstage/catalog-model": "npm:^1.9.0" + "@backstage/cli": "npm:^0.36.2" + "@backstage/config": "npm:^1.3.8" + "@backstage/errors": "npm:^1.3.1" + "@backstage/plugin-catalog-node": "npm:^2.2.1" + "@backstage/plugin-events-node": "npm:^0.4.22" + "@backstage/types": "npm:^1.2.2" + "@openchoreo/backstage-plugin-catalog-backend-module": "workspace:^" + "@openchoreo/backstage-plugin-common": "workspace:^" + "@openchoreo/openchoreo-client-node": "workspace:^" + "@opentelemetry/api": "npm:^1.9.0" + "@types/express": "npm:4.17.23" + "@types/luxon": "npm:^3.4.2" + "@types/supertest": "npm:2.0.16" + express: "npm:4.21.2" + express-promise-router: "npm:^4.1.0" + knex: "npm:3.1.0" + luxon: "npm:^3.0.0" + supertest: "npm:6.3.4" + uuid: "npm:^11.0.0" + zod: "npm:^4.1.12" + languageName: unknown + linkType: soft + "@openchoreo/backstage-plugin-catalog-backend-module-openchoreo-users@workspace:^, @openchoreo/backstage-plugin-catalog-backend-module-openchoreo-users@workspace:plugins/catalog-backend-module-openchoreo-users": version: 0.0.0-use.local resolution: "@openchoreo/backstage-plugin-catalog-backend-module-openchoreo-users@workspace:plugins/catalog-backend-module-openchoreo-users" @@ -10421,6 +10452,7 @@ __metadata: "@openchoreo/backstage-plugin-auth-backend-module-openchoreo-auth": "workspace:^" "@openchoreo/backstage-plugin-backend": "workspace:^" "@openchoreo/backstage-plugin-catalog-backend-module": "workspace:^" + "@openchoreo/backstage-plugin-catalog-backend-module-openchoreo-incremental": "workspace:^" "@openchoreo/backstage-plugin-catalog-backend-module-openchoreo-users": "workspace:^" "@openchoreo/backstage-plugin-openchoreo-ci-backend": "workspace:^" "@openchoreo/backstage-plugin-openchoreo-observability-backend": "workspace:^" @@ -15590,6 +15622,13 @@ __metadata: languageName: node linkType: hard +"@types/luxon@npm:^3.4.2": + version: 3.7.5 + resolution: "@types/luxon@npm:3.7.5" + checksum: 10c0/7b851e94cd0d47784682dd3bc412960519d2c888bcca1408b6c8484742805637ed1a94797ce4e44fc925c8b20b52803035880d0122e3658c17f98838b387c46b + languageName: node + linkType: hard + "@types/luxon@npm:~3.4.0": version: 3.4.2 resolution: "@types/luxon@npm:3.4.2" @@ -38754,7 +38793,7 @@ __metadata: languageName: node linkType: hard -"zod@npm:^3.25.76 || ^4.0.0, zod@npm:^4.0.0, zod@npm:^4.4.3": +"zod@npm:^3.25.76 || ^4.0.0, zod@npm:^4.0.0, zod@npm:^4.1.12, zod@npm:^4.4.3": version: 4.4.3 resolution: "zod@npm:4.4.3" checksum: 10c0/7ea31b558e88f9faf44f31dd185e2e1cbf51fed3081787fb96cc2534749b50c0acfc6da7f0922a7353ed092dd358c7d50c28ea96c94d04af64191bd33152eca3