diff --git a/.github/workflows/ci-host.yaml b/.github/workflows/ci-host.yaml index 8fa3dd48412..a725d885034 100644 --- a/.github/workflows/ci-host.yaml +++ b/.github/workflows/ci-host.yaml @@ -583,7 +583,16 @@ jobs: # raw notFound when the host store does a direct GET) point at # the same race; match both. A second shard pass typically lands # after the race resolves. - RETRY_PATTERN='ChunkLoadError|Failed to fetch dynamically imported module|NetworkError when attempting to fetch resource|unable to fetch https://icons\.[^:]+: fetch failed|cross-realm fetch failed for https://realm-test\.[^/]+|Could not find https://realm-test\.[^/"]+' + # The `Global error: Uncaught TypeError: Failed to fetch` form is a + # test-harness startup race: base card components (workspace, + # cards-grid) fire a `_types` fetch against the mock test realm the + # moment they render, and with base modules compiled into the host + # bundle that render can land before the test-realm service worker + # is intercepting — the fetch escapes to the real network and the + # rejection surfaces as a Global error. The race is rare per shard, + # so a second pass lands; the durable fix is queuing test-realm + # fetches in the harness until realm registration completes. + RETRY_PATTERN='ChunkLoadError|Failed to fetch dynamically imported module|NetworkError when attempting to fetch resource|unable to fetch https://icons\.[^:]+: fetch failed|cross-realm fetch failed for https://realm-test\.[^/]+|Could not find https://realm-test\.[^/"]+|Global error: Uncaught TypeError: Failed to fetch' if [ $exit_code -ne 0 ] && grep -Eq "$RETRY_PATTERN" /tmp/test-output.log; then echo "" echo "::warning::Transient chunk-fetch failure detected — retrying shard ${{ matrix.shardIndex }}" diff --git a/packages/base/audio-file-def.gts b/packages/base/audio-file-def.gts index 2b693bf9a12..ec8f2d02018 100644 --- a/packages/base/audio-file-def.gts +++ b/packages/base/audio-file-def.gts @@ -1,6 +1,6 @@ import MusicIcon from '@cardstack/boxel-icons/music'; import { - BaseDefComponent, + type BaseDefComponent, Component, NumberField, contains, diff --git a/packages/base/brand-guide.gts b/packages/base/brand-guide.gts index 2e1b982d947..f0880d04fe2 100644 --- a/packages/base/brand-guide.gts +++ b/packages/base/brand-guide.gts @@ -31,7 +31,7 @@ import { buildCssVariableName, sanitizeHtmlSafe, eq, - CssVariableEntry, + type CssVariableEntry, } from '@cardstack/boxel-ui/helpers'; import { cardTypeDisplayName } from '@cardstack/runtime-common'; diff --git a/packages/base/card-api.gts b/packages/base/card-api.gts index cdbff4f60fd..d3c181974d2 100644 --- a/packages/base/card-api.gts +++ b/packages/base/card-api.gts @@ -27,8 +27,8 @@ import { baseRef, CardContextName, CardError, - CodeRef, - ToolContext, + type CodeRef, + type ToolContext, Deferred, byteStreamToUint8Array, fields, @@ -55,12 +55,12 @@ import { loadCardDocument, Loader, localId, - LocalPath, + type LocalPath, meta, primitive, realmURL, relativeTo, - SingleCardDocument, + type SingleCardDocument, uuidv4, NumberSerializer, type Format, @@ -86,9 +86,9 @@ import { FileMetaResourceType, CardResourceType, loadFileMetaDocument, - CardResource, - LooseLinkableResource, - LooseSingleResourceDocument, + type CardResource, + type LooseLinkableResource, + type LooseSingleResourceDocument, shouldTrackRuntimeModuleGraph, shouldTrackRuntimeRelationship, trackRuntimeFileDependency, @@ -208,8 +208,8 @@ import { TextInputValidator } from './text-input-validator'; import { type GetMenuItemParams, getDefaultCardMenuItems } from './menu-items'; import { getDefaultFileMenuItems } from './file-menu-items'; import { - LinkableDocument, - SingleFileMetaDocument, + type LinkableDocument, + type SingleFileMetaDocument, } from '@cardstack/runtime-common/document-types'; import type { MarkdownEmbedChooser } from '@cardstack/runtime-common/bfm-card-references'; import type { FileMetaResource } from '@cardstack/runtime-common'; @@ -4921,16 +4921,22 @@ export function resolveRef( } function myLoader(): Loader { - // we know this code is always loaded by an instance of our Loader, which sets - // import.meta.loader. + // A Loader that evaluates this module injects `import.meta.loader`. When + // the module is compiled into the host bundle instead, the platform + // evaluates it and no loader is injected; the host publishes the loader + // bundled modules should use (see Loader.setForBundledModules). - // When type-checking realm-server, tsc sees this file and thinks - // it will be transpiled to CommonJS and so it complains about this line. But - // this file is always loaded through our loader and always has access to import.meta. + // When type-checking realm-server, tsc sees this file and thinks it will + // be transpiled to CommonJS and so it complains about import.meta. // @ts-ignore - return (import.meta as any).loader; + let loader = (import.meta as any).loader ?? Loader.forBundledModules(); + if (!loader) { + throw new Error('No Loader is available to this module'); + } + return loader; } + class FallbackCardStore implements CardStore { #instances: Map = new Map(); #fileMetaInstances: Map = new Map(); diff --git a/packages/base/card-serialization.ts b/packages/base/card-serialization.ts index 3f8dd66a5ed..861990ab8e8 100644 --- a/packages/base/card-serialization.ts +++ b/packages/base/card-serialization.ts @@ -5,7 +5,6 @@ import type { CardResource, CardResourceMeta, FileMetaResource, - Loader, LooseCardResource, LooseFileMetaResource, LooseSingleCardDocument, @@ -25,6 +24,7 @@ import { isEqual, merge } from 'lodash-es'; import { assertIsSerializerName, CardResourceType, + Loader, fieldSerializer, FileMetaResourceType, getSerializer, @@ -92,14 +92,19 @@ export const deserialize = Symbol.for('cardstack-deserialize'); // --- Serialization Functions --- function myLoader(): Loader { - // we know this code is always loaded by an instance of our Loader, which sets - // import.meta.loader. + // A Loader that evaluates this module injects `import.meta.loader`. When + // the module is compiled into the host bundle instead, the platform + // evaluates it and no loader is injected; the host publishes the loader + // bundled modules should use (see Loader.setForBundledModules). - // When type-checking realm-server, tsc sees this file and thinks - // it will be transpiled to CommonJS and so it complains about this line. But - // this file is always loaded through our loader and always has access to import.meta. + // When type-checking realm-server, tsc sees this file and thinks it will + // be transpiled to CommonJS and so it complains about import.meta. // @ts-ignore - return (import.meta as any).loader; + let loader = (import.meta as any).loader ?? Loader.forBundledModules(); + if (!loader) { + throw new Error('No Loader is available to this module'); + } + return loader; } export async function cardClassFromResource( diff --git a/packages/base/cards-grid.gts b/packages/base/cards-grid.gts index a4f9c1491f6..70d350bf041 100644 --- a/packages/base/cards-grid.gts +++ b/packages/base/cards-grid.gts @@ -25,7 +25,7 @@ import { subscribeToRealm, codeRefFromInternalKey, type Query, - CardErrorJSONAPI, + type CardErrorJSONAPI, } from '@cardstack/runtime-common'; import CardsGridLayout, { diff --git a/packages/base/code-ref.gts b/packages/base/code-ref.gts index e69e18b9251..f956a9f29e7 100644 --- a/packages/base/code-ref.gts +++ b/packages/base/code-ref.gts @@ -9,6 +9,7 @@ import { CardURLContextName, fieldSerializer, CodeRefSerializer, + Loader, } from '@cardstack/runtime-common'; import { not } from '@cardstack/boxel-ui/helpers'; import { BoxelInput } from '@cardstack/boxel-ui/components'; @@ -75,7 +76,14 @@ class EditView extends Component { module = new URL(module, new URL(this.cardURL)).href; } try { - let code = (await import(module))[name]; + // Load through the Loader rather than a bare dynamic import: the + // module is a runtime realm URL, which only a Loader can resolve + // (shims, realm mappings, authenticated fetch). Inside + // loader-evaluated modules the AMD transpile rewrites `import()` + // this way implicitly; compiled-in modules must do it explicitly. + let code = (await myLoader().import>(module))[ + name + ]; if (code) { this.validationState = 'valid'; if (!opts?.checkOnly) { @@ -91,6 +99,22 @@ class EditView extends Component { ); } +function myLoader(): Loader { + // A Loader that evaluates this module injects `import.meta.loader`. When + // the module is compiled into the host bundle instead, the platform + // evaluates it and no loader is injected; the host publishes the loader + // bundled modules should use (see Loader.setForBundledModules). + + // When type-checking realm-server, tsc sees this file and thinks it will + // be transpiled to CommonJS and so it complains about import.meta. + // @ts-ignore + let loader = (import.meta as any).loader ?? Loader.forBundledModules(); + if (!loader) { + throw new Error('No Loader is available to this module'); + } + return loader; +} + export default class CodeRefField extends FieldDef { static displayName = 'CodeRef'; static icon = CodeIcon; diff --git a/packages/base/color-field/components/advanced-color-picker.gts b/packages/base/color-field/components/advanced-color-picker.gts index ac79db7b084..1c7eb427815 100644 --- a/packages/base/color-field/components/advanced-color-picker.gts +++ b/packages/base/color-field/components/advanced-color-picker.gts @@ -13,10 +13,10 @@ import type { ColorFieldConfiguration } from '../util/color-utils'; import { parseCssColor, parseCssColorSafe } from '../util/color-utils'; import { detectColorFormat, - RichColorFormat, + type RichColorFormat, hexToRgba, hsvToRgb, - RGBA, + type RGBA, rgbaToFormatString, rgbaToHexString, rgbaToHsl, diff --git a/packages/base/color-field/components/color-wheel-picker.gts b/packages/base/color-field/components/color-wheel-picker.gts index 4e17071a2bf..87139e990fd 100644 --- a/packages/base/color-field/components/color-wheel-picker.gts +++ b/packages/base/color-field/components/color-wheel-picker.gts @@ -13,7 +13,7 @@ import type { import { detectColorFormat, hslToRgb, - RGBA, + type RGBA, rgbaToFormatString, rgbaToHsv, } from '@cardstack/boxel-ui/helpers'; diff --git a/packages/base/color-field/components/slider-picker.gts b/packages/base/color-field/components/slider-picker.gts index 252e18d46cd..f20686bb8b1 100644 --- a/packages/base/color-field/components/slider-picker.gts +++ b/packages/base/color-field/components/slider-picker.gts @@ -10,8 +10,8 @@ import type Owner from '@ember/owner'; import type { ColorFieldSignature } from '../util/color-field-signature'; import { parseCssColor, - SliderColorFormat, - SliderVariantConfiguration, + type SliderColorFormat, + type SliderVariantConfiguration, } from '../util/color-utils'; import { detectColorFormat, diff --git a/packages/base/color.gts b/packages/base/color.gts index 3d1977d5004..b2d71ebd528 100644 --- a/packages/base/color.gts +++ b/packages/base/color.gts @@ -1,5 +1,5 @@ -import { Component } from '@cardstack/base/card-api'; -import StringField from '@cardstack/base/string'; +import { Component } from './card-api'; +import StringField from './string'; import { Swatch } from '@cardstack/boxel-ui/components'; import { markdownEscape } from '@cardstack/boxel-ui/helpers'; import PaletteIcon from '@cardstack/boxel-icons/palette'; diff --git a/packages/base/contains-many-component.gts b/packages/base/contains-many-component.gts index 3e045e9ef1e..f054be40681 100644 --- a/packages/base/contains-many-component.gts +++ b/packages/base/contains-many-component.gts @@ -447,12 +447,18 @@ export function getContainsManyComponent({ } function myLoader(): Loader { - // we know this code is always loaded by an instance of our Loader, which sets - // import.meta.loader. + // A Loader that evaluates this module injects `import.meta.loader`. When + // the module is compiled into the host bundle instead, the platform + // evaluates it and no loader is injected; the host publishes the loader + // bundled modules should use (see Loader.setForBundledModules). - // When type-checking realm-server, tsc sees this file and thinks - // it will be transpiled to CommonJS and so it complains about this line. But - // this file is always loaded through our loader and always has access to import.meta. + // When type-checking realm-server, tsc sees this file and thinks it will + // be transpiled to CommonJS and so it complains about import.meta. // @ts-ignore - return (import.meta as any).loader; + let loader = (import.meta as any).loader ?? Loader.forBundledModules(); + if (!loader) { + throw new Error('No Loader is available to this module'); + } + return loader; } + diff --git a/packages/base/csv-file-def.gts b/packages/base/csv-file-def.gts index 2c9a2732e57..32219592b09 100644 --- a/packages/base/csv-file-def.gts +++ b/packages/base/csv-file-def.gts @@ -2,7 +2,7 @@ import { byteStreamToUint8Array } from '@cardstack/runtime-common'; import { htmlSafe } from '@ember/template'; import CsvIcon from '@cardstack/boxel-icons/csv'; import { - BaseDefComponent, + type BaseDefComponent, Component, StringField, contains, diff --git a/packages/base/field-component.gts b/packages/base/field-component.gts index c558492cc66..6615d6611ea 100644 --- a/packages/base/field-component.gts +++ b/packages/base/field-component.gts @@ -8,10 +8,10 @@ import { type BaseDefComponent, type BaseDefConstructor, type Theme, - CardContext, + type CardContext, formats, - FieldFormats, - CardCrudFunctions, + type FieldFormats, + type CardCrudFunctions, } from './card-api'; import { isCard, isCompoundField } from './field-support'; import { @@ -23,7 +23,7 @@ import { isCardInstance, type CodeRef, type Permissions, - ResolvedCodeRef, + type ResolvedCodeRef, CardCrudFunctionsContextName, } from '@cardstack/runtime-common'; import type { ComponentLike } from '@glint/template'; diff --git a/packages/base/json-file-def.gts b/packages/base/json-file-def.gts index 05e59eb6e02..c868220abd8 100644 --- a/packages/base/json-file-def.gts +++ b/packages/base/json-file-def.gts @@ -2,7 +2,7 @@ import { byteStreamToUint8Array } from '@cardstack/runtime-common'; import { htmlSafe } from '@ember/template'; import JsonIcon from '@cardstack/boxel-icons/json'; import { - BaseDefComponent, + type BaseDefComponent, Component, StringField, contains, diff --git a/packages/base/links-to-editor.gts b/packages/base/links-to-editor.gts index 2c22b582412..d633c916217 100644 --- a/packages/base/links-to-editor.gts +++ b/packages/base/links-to-editor.gts @@ -14,7 +14,7 @@ import { type Field, type CardContext, type LinkableDefConstructor, - CreateCardFn, + type CreateCardFn, isFileDef, } from './card-api'; import { @@ -282,12 +282,18 @@ export class LinksToEditor extends GlimmerComponent { } function myLoader(): Loader { - // we know this code is always loaded by an instance of our Loader, which sets - // import.meta.loader. + // A Loader that evaluates this module injects `import.meta.loader`. When + // the module is compiled into the host bundle instead, the platform + // evaluates it and no loader is injected; the host publishes the loader + // bundled modules should use (see Loader.setForBundledModules). - // When type-checking realm-server, tsc sees this file and thinks - // it will be transpiled to CommonJS and so it complains about this line. But - // this file is always loaded through our loader and always has access to import.meta. + // When type-checking realm-server, tsc sees this file and thinks it will + // be transpiled to CommonJS and so it complains about import.meta. // @ts-ignore - return (import.meta as any).loader; + let loader = (import.meta as any).loader ?? Loader.forBundledModules(); + if (!loader) { + throw new Error('No Loader is available to this module'); + } + return loader; } + diff --git a/packages/base/links-to-many-component.gts b/packages/base/links-to-many-component.gts index d742d03eabd..9d10b215fa5 100644 --- a/packages/base/links-to-many-component.gts +++ b/packages/base/links-to-many-component.gts @@ -11,8 +11,8 @@ import { type FieldDef, type Format, type LinkableDefConstructor, - CreateCardFn, - CardCrudFunctions, + type CreateCardFn, + type CardCrudFunctions, isFileDef, brokenLinkFormat, } from './card-api'; @@ -22,7 +22,7 @@ import { } from './field-support'; import { rawArrayValues } from './watched-array'; import { - BoxComponentSignature, + type BoxComponentSignature, CardCrudFunctionsConsumer, DefaultFormatsConsumer, PermissionsConsumer, @@ -49,7 +49,7 @@ import { type ResolvedCodeRef, uuidv4, CardCrudFunctionsContextName, - CardErrorJSONAPI, + type CardErrorJSONAPI, cardTypeName, } from '@cardstack/runtime-common'; import { @@ -825,12 +825,18 @@ export function getLinksToManyComponent({ } function myLoader(): Loader { - // we know this code is always loaded by an instance of our Loader, which sets - // import.meta.loader. + // A Loader that evaluates this module injects `import.meta.loader`. When + // the module is compiled into the host bundle instead, the platform + // evaluates it and no loader is injected; the host publishes the loader + // bundled modules should use (see Loader.setForBundledModules). - // When type-checking realm-server, tsc sees this file and thinks - // it will be transpiled to CommonJS and so it complains about this line. But - // this file is always loaded through our loader and always has access to import.meta. + // When type-checking realm-server, tsc sees this file and thinks it will + // be transpiled to CommonJS and so it complains about import.meta. // @ts-ignore - return (import.meta as any).loader; + let loader = (import.meta as any).loader ?? Loader.forBundledModules(); + if (!loader) { + throw new Error('No Loader is available to this module'); + } + return loader; } + diff --git a/packages/base/markdown-file-def.gts b/packages/base/markdown-file-def.gts index 64036c376e3..970b386d365 100644 --- a/packages/base/markdown-file-def.gts +++ b/packages/base/markdown-file-def.gts @@ -11,7 +11,7 @@ import { } from '@cardstack/runtime-common'; import MarkdownIcon from '@cardstack/boxel-icons/align-box-left-middle'; import { - BaseDefComponent, + type BaseDefComponent, CardDef, Component, StringField, diff --git a/packages/base/package.json b/packages/base/package.json index 542b669c508..1c72b6ce083 100644 --- a/packages/base/package.json +++ b/packages/base/package.json @@ -15,6 +15,7 @@ "@types/lodash-es": "catalog:", "awesome-phonenumber": "catalog:", "concurrently": "catalog:", + "date-fns": "catalog:", "ember-cli-htmlbars": "^6.3.0", "ember-concurrency": "catalog:", "ember-css-url": "^1.0.0", @@ -28,7 +29,7 @@ "yaml": "catalog:" }, "peerDependencies": { - "ember-provide-consume-context": "^0.7.0", + "ember-provide-consume-context": "^0.8.0", "ember-source": "catalog:", "lodash-es": "catalog:" }, diff --git a/packages/base/skill-frontmatter-field.gts b/packages/base/skill-frontmatter-field.gts index 4bbc93cc293..f7300267102 100644 --- a/packages/base/skill-frontmatter-field.gts +++ b/packages/base/skill-frontmatter-field.gts @@ -3,7 +3,7 @@ import { codeRefWithAbsoluteIdentifier, getClass, rri, - type Loader, + Loader, type ResolvedCodeRef, type ToolContext, type ToolSchemaError, @@ -267,12 +267,18 @@ async function generateToolDefinitions( } function myLoader(): Loader { - // we know this code is always loaded by an instance of our Loader, which - // sets import.meta.loader. + // A Loader that evaluates this module injects `import.meta.loader`. When + // the module is compiled into the host bundle instead, the platform + // evaluates it and no loader is injected; the host publishes the loader + // bundled modules should use (see Loader.setForBundledModules). - // When type-checking realm-server, tsc sees this file and thinks it will be - // transpiled to CommonJS and so it complains about this line. But this file - // is always loaded through our loader and always has access to import.meta. + // When type-checking realm-server, tsc sees this file and thinks it will + // be transpiled to CommonJS and so it complains about import.meta. // @ts-ignore - return (import.meta as any).loader; + let loader = (import.meta as any).loader ?? Loader.forBundledModules(); + if (!loader) { + throw new Error('No Loader is available to this module'); + } + return loader; } + diff --git a/packages/base/spec.gts b/packages/base/spec.gts index 35d649fedd6..7867776ed0e 100644 --- a/packages/base/spec.gts +++ b/packages/base/spec.gts @@ -1109,12 +1109,18 @@ function getIcon(specType: string) { } function myLoader(): Loader { - // we know this code is always loaded by an instance of our Loader, which sets - // import.meta.loader. + // A Loader that evaluates this module injects `import.meta.loader`. When + // the module is compiled into the host bundle instead, the platform + // evaluates it and no loader is injected; the host publishes the loader + // bundled modules should use (see Loader.setForBundledModules). - // When type-checking realm-server, tsc sees this file and thinks - // it will be transpiled to CommonJS and so it complains about this line. But - // this file is always loaded through our loader and always has access to import.meta. + // When type-checking realm-server, tsc sees this file and thinks it will + // be transpiled to CommonJS and so it complains about import.meta. // @ts-ignore - return (import.meta as any).loader; + let loader = (import.meta as any).loader ?? Loader.forBundledModules(); + if (!loader) { + throw new Error('No Loader is available to this module'); + } + return loader; } + diff --git a/packages/base/text-file-def.gts b/packages/base/text-file-def.gts index c76f9994466..09d1c34d0c7 100644 --- a/packages/base/text-file-def.gts +++ b/packages/base/text-file-def.gts @@ -1,7 +1,7 @@ import { byteStreamToUint8Array } from '@cardstack/runtime-common'; import TextFileIcon from '@cardstack/boxel-icons/file-text'; import { - BaseDefComponent, + type BaseDefComponent, Component, StringField, contains, diff --git a/packages/base/ts-file-def.gts b/packages/base/ts-file-def.gts index 9a2fc1f1652..8f7ac6d3ed2 100644 --- a/packages/base/ts-file-def.gts +++ b/packages/base/ts-file-def.gts @@ -2,7 +2,7 @@ import { byteStreamToUint8Array } from '@cardstack/runtime-common'; import { htmlSafe } from '@ember/template'; import FileCodeIcon from '@cardstack/boxel-icons/file-code'; import { - BaseDefComponent, + type BaseDefComponent, Component, StringField, contains, diff --git a/packages/base/workspace.gts b/packages/base/workspace.gts index c8801314fcd..486ac19f750 100644 --- a/packages/base/workspace.gts +++ b/packages/base/workspace.gts @@ -38,6 +38,7 @@ import { codeRef, specRef, baseCardRef, + baseRealm, baseRealmRRI, isCardInstance, SupportedMimeType, @@ -46,7 +47,7 @@ import { type Query, type Filter, type CodeRef, - CardErrorJSONAPI, + type CardErrorJSONAPI, } from '@cardstack/runtime-common'; import CardsGridLayout, { @@ -75,12 +76,11 @@ import { MarkdownDef } from './markdown-file-def'; // realm README import type { RealmEventContent } from './matrix-event'; import { Spec } from './spec'; -// This file is always loaded through the Boxel loader, which supplies -// `import.meta`. When type-checking, tsc sees the file as CommonJS output and -// rejects the meta-property, so suppress it — the same pattern used elsewhere -// in packages/base. -// @ts-ignore -const here: string = (import.meta as any).url; +// This module's canonical URL, the base for sibling code refs below. +// `import.meta.url` can't provide it in every evaluation environment (a +// bundler reports the compiled chunk's URL, not the realm module's), so +// state it directly. +const here: string = new URL('./workspace', baseRealm.url).href; const [, StripView, GridView] = VIEW_OPTIONS; diff --git a/packages/host/app/lib/bundled-base-modules.d.ts b/packages/host/app/lib/bundled-base-modules.d.ts new file mode 100644 index 00000000000..54536dfaeed --- /dev/null +++ b/packages/host/app/lib/bundled-base-modules.d.ts @@ -0,0 +1,2 @@ +declare const BASE_MODULES: Record>; +export default BASE_MODULES; diff --git a/packages/host/app/lib/bundled-base-modules.js b/packages/host/app/lib/bundled-base-modules.js new file mode 100644 index 00000000000..35d8a4570bf --- /dev/null +++ b/packages/host/app/lib/bundled-base-modules.js @@ -0,0 +1,14 @@ +// The base realm's modules, compiled into the host bundle. Lives in an +// untyped .js module because `import.meta.glob` is a vite build-time +// construct that ember-tsc (module: nodenext, CJS-flavored app files) +// rejects; the .d.ts sibling carries the type. +const BASE_MODULES = import.meta.glob( + [ + '../../../base/**/*.{gts,ts}', + '!../../../base/node_modules/**', + '!../../../base/**/*.d.ts', + ], + { eager: true }, +); + +export default BASE_MODULES; diff --git a/packages/host/app/lib/bundled-base.ts b/packages/host/app/lib/bundled-base.ts new file mode 100644 index 00000000000..9715d4e896c --- /dev/null +++ b/packages/host/app/lib/bundled-base.ts @@ -0,0 +1,73 @@ +// Bundles the @cardstack/base realm's modules into the host build and +// registers them as loader shims, so importing a base module (whether as +// `@cardstack/base/card-api` or its resolved base-realm URL) resolves to +// the compiled-in module instead of a network fetch of realm-server- +// transpiled source. This trades two properties of the fetched path for +// speed: +// +// - Base modules become singletons shared by every loader generation. +// A loader reset (test isolation, code-change flush) no longer +// re-evaluates base module state; whatever module-level state card-api +// and friends hold persists across resets. +// - The running base realm's *source* is no longer what executes in the +// host: editing base code in a realm (or reindexing it) has no effect +// until the host is rebuilt. This includes indexing — the prerender +// renders with the host dist, so index output reflects the bundled base, +// not the realm-served source. +// +// The eager glob in bundled-base-modules.js compiles every base module +// into the host's initial bundle through the same vite/embroider pipeline +// as host app code. + +import type { VirtualNetwork } from '@cardstack/runtime-common'; + +import BASE_MODULES from './bundled-base-modules'; + +const GLOB_PREFIX = '../../../base/'; + +// Registers on the virtual network (not per loader) so that every loader +// sharing the network serves the bundled modules — including loaders +// constructed outside the loader service (test-realm adapters, in-browser +// indexing). Must run after the network's `@cardstack/base/` realm mapping +// is registered, since shim identifiers resolve at registration time. +// +// Known gap: only the RRI-resolved identifier form is registered. An +// import that names a base module by its canonical +// `https://cardstack.com/base/` URL misses the shim (resolveImport passes +// URL-form identifiers through unchanged; URL→URL mapping happens at the +// network's fetch boundary, which shim lookup precedes) and falls through +// to a network fetch that evaluates a second copy of the module. +// Registering the canonical form as a second shim entry is NOT the fix: +// loaders capture export identities under whichever identifier form they +// fetched, so dual registration makes a def's identified module URL +// depend on import order. The durable fix is normalizing the identifier +// through the network's URL mappings in the loader's module-fetch path so +// both forms converge on one module state. +// Shim registration order doubles as identity-capture order (loaders +// replay the network's shim inventory dependency-first — see the loader's +// captureVirtualNetworkShimIdentities). The def classes that serialization +// identifies are declared in card-api (several modules re-export them: +// file-api, markdown, image-file-def) and cards-grid (re-exported by +// index), so those two register ahead of the alphabetical remainder. +const DECLARING_MODULES_FIRST = ['card-api', 'cards-grid']; + +export function shimBundledBase(virtualNetwork: VirtualNetwork) { + let entries = Object.entries(BASE_MODULES) + .map(([path, module]) => ({ + name: path.slice(GLOB_PREFIX.length).replace(/\.(gts|ts)$/, ''), + module, + })) + .sort((a, b) => { + let ai = DECLARING_MODULES_FIRST.indexOf(a.name); + let bi = DECLARING_MODULES_FIRST.indexOf(b.name); + if (ai !== bi) { + return (ai === -1 ? Infinity : ai) < (bi === -1 ? Infinity : bi) + ? -1 + : 1; + } + return a.name < b.name ? -1 : 1; + }); + for (let { name, module } of entries) { + virtualNetwork.shimModule(`@cardstack/base/${name}`, module); + } +} diff --git a/packages/host/app/services/loader-service.ts b/packages/host/app/services/loader-service.ts index a307ce6c8e5..90034bfcfaa 100644 --- a/packages/host/app/services/loader-service.ts +++ b/packages/host/app/services/loader-service.ts @@ -76,7 +76,9 @@ export default class LoaderService extends Service { log.debug(`resetting loader for session boundary (${reason ?? ''})`); this.clearSessionCaches(); let previous = this.loader; - this.loader = previous ? Loader.cloneLoader(previous) : this.makeInstance(); + this.loader = previous + ? this.trackCurrent(Loader.cloneLoader(previous)) + : this.makeInstance(); previous?.dispose(); } @@ -145,7 +147,7 @@ export default class LoaderService extends Service { let previous = this.loader; this.recordLoaderReplacement(previous, options?.codeChange); if (previous) { - this.loader = Loader.cloneLoader(previous); + this.loader = this.trackCurrent(Loader.cloneLoader(previous)); previous.dispose(); } else { this.loader = this.makeInstance(); @@ -196,6 +198,15 @@ export default class LoaderService extends Service { ), virtualNetwork: this.network.virtualNetwork, }); + return this.trackCurrent(loader); + } + + // Bundled base modules can't discover their loader via + // `import.meta.loader` (the platform evaluated them, not a Loader), so + // every loader that becomes this service's active one is also published + // as the loader bundled modules fall back to. + private trackCurrent(loader: Loader): Loader { + Loader.setForBundledModules(loader); return loader; } diff --git a/packages/host/app/services/network.ts b/packages/host/app/services/network.ts index ba522683b24..d96e83389c2 100644 --- a/packages/host/app/services/network.ts +++ b/packages/host/app/services/network.ts @@ -13,6 +13,7 @@ import { import config from '@cardstack/host/config/environment'; +import { shimBundledBase } from '../lib/bundled-base'; import { shimExternals } from '../lib/externals'; import { authErrorEventMiddleware } from '../utils/auth-error-guard'; @@ -74,7 +75,14 @@ export default class NetworkService extends Service { '@cardstack/base/', resolvedBaseRealmURL.href, ); + // Externals shim first: identity capture replays shims in registration + // order, and base modules re-export values whose declaring modules are + // externals (runtime-common, boxel-ui) — declaring modules must + // register ahead of their re-exporters. shimExternals(virtualNetwork); + // Base-realm modules ship inside the host bundle; any loader on this + // network serves them without fetching from the base realm. + shimBundledBase(virtualNetwork); virtualNetwork.addImportMap('@cardstack/boxel-icons/', (rest) => { return `${config.iconsURL}/@cardstack/boxel-icons/v1/icons/${rest}.js`; }); diff --git a/packages/host/tests/helpers/setup.ts b/packages/host/tests/helpers/setup.ts index 0a4e48f354c..c1eace22d6e 100644 --- a/packages/host/tests/helpers/setup.ts +++ b/packages/host/tests/helpers/setup.ts @@ -20,6 +20,8 @@ import { clearHtmlComponentCache } from '@cardstack/host/lib/html-component'; import type SessionService from '@cardstack/host/services/session'; import { AiAssistantOpen } from '@cardstack/host/utils/local-storage-keys'; +import { getTestRealmRegistry } from './test-realm-registry'; + import { cleanupMonacoEditorModels } from './index'; // Pin `yaml` into the eager test bundle. `markdown-file-def` parses frontmatter @@ -268,6 +270,25 @@ function setupFetchDebugging(hooks: NestedHooks) { startedAt: Date.now(), }); try { + // Requests to a registered test realm are answered in-page rather + // than dispatched to the network. The service-worker relay that + // otherwise serves these URLs only intercepts once the worker + // controls the page and its per-module activation has been acked; + // a card component that renders (and fetches) before that window + // closes would otherwise hit the real network and fail, since the + // test-realm host doesn't exist outside the harness. The registry + // is populated at realm construction, so this path has no such + // window. Non-fetch resources (images, workers) still rely on the + // service worker. + for (let [realmUrl, { realm }] of getTestRealmRegistry()) { + if (url.startsWith(realmUrl)) { + let response = await realm.maybeHandle(new Request(input, init)); + if (response) { + return response; + } + break; + } + } return await boundFetch(input, init); } catch (error) { let reason = formatErrorForLog(error); diff --git a/packages/host/vite.config.mjs b/packages/host/vite.config.mjs index e7ce9f4d327..becccf9d709 100644 --- a/packages/host/vite.config.mjs +++ b/packages/host/vite.config.mjs @@ -236,6 +236,11 @@ export default defineConfig(({ mode }) => ({ build: { minify: false, rolldownOptions: { + // Bundled base-realm modules may import directly from an https URL + // (e.g. currency.gts's esm.run import). Leave those imports verbatim + // in the output; the browser fetches them at chunk load, matching how + // the loader-served module behaves. + external: [/^https:\/\//], output: { keepNames: true, ...(mode === 'production' ? { minify: true } : {}), @@ -269,6 +274,15 @@ export default defineConfig(({ mode }) => ({ }, resolve: { alias: [ + // Base-realm modules (bundled via app/lib/bundled-base.ts) import + // host tools as `@cardstack/boxel-host/tools/*` (and the pre-rename + // `commands/*` spelling). At runtime the virtual network shims those + // specifiers to app/tools modules (see app/tools/index.ts); this + // alias gives the bundler the same 1:1 mapping. + { + find: /^@cardstack\/boxel-host\/(?:tools|commands)\//, + replacement: `${__dirname}/app/tools/`, + }, { find: 'path', replacement: require.resolve('path-browserify') }, { find: 'stream', replacement: require.resolve('stream-browserify') }, { find: /^util$/, replacement: require.resolve('util/') }, diff --git a/packages/runtime-common/loader.ts b/packages/runtime-common/loader.ts index 98885716a76..8268598a9d3 100644 --- a/packages/runtime-common/loader.ts +++ b/packages/runtime-common/loader.ts @@ -234,8 +234,36 @@ export class Loader { }, ) { this.fetchImplementation = fetch; - this.resolveImport = - resolveImport ?? ((moduleIdentifier) => moduleIdentifier); + let rawResolveImport = + resolveImport ?? ((moduleIdentifier: string) => moduleIdentifier); + let virtualNetwork = options?.virtualNetwork; + // Fold virtual-alias URL forms (e.g. https://cardstack.com/base/…) onto + // the real URL the network would serve them from, so both spellings of a + // module converge on one module-state entry, one shim lookup key, and one + // captured export identity. `resolveImport` itself only rewrites RRI / + // bare-package prefixes and passes full URLs through, so without this a + // virtual-alias import keys its own separate module state — and card + // serialization requires identities in real-URL form (module refs + // relativize against instance ids, which are real-form). Mapping an + // already-real URL is a no-op, so composed resolvers (cloneLoader wraps + // the parent's) stay idempotent. + this.resolveImport = virtualNetwork + ? (moduleIdentifier: string) => { + let resolved = rawResolveImport(moduleIdentifier); + // mapURL constructs a URL, so only URL-shaped identifiers can be + // folded; anything else (an unmapped prefix form, a relative + // specifier) passes through untouched. + if ( + !resolved.startsWith('http://') && + !resolved.startsWith('https://') + ) { + return resolved; + } + return ( + virtualNetwork.mapURL(resolved, 'virtual-to-real')?.href ?? resolved + ); + } + : rawResolveImport; this.retrySleep = options?.retrySleep; this.virtualNetwork = options?.virtualNetwork; // Module caches are keyed by canonical RRI form (see moduleCacheKey), whose @@ -421,6 +449,24 @@ export class Loader { return undefined; } + // Realm modules that a Loader evaluates discover their loader via + // `import.meta.loader`, which the Loader injects at eval time. Modules + // compiled into the host bundle instead (and registered as loader shims — + // see the host's bundled-base registration) are evaluated by the + // platform's module system, where `import.meta.loader` does not exist. + // The host publishes its active loader here so bundled modules can fall + // back to it; module code reads this only when `import.meta.loader` is + // absent. + static #forBundledModules: Loader | undefined; + + static setForBundledModules(loader: Loader) { + Loader.#forBundledModules = loader; + } + + static forBundledModules(): Loader | undefined { + return Loader.#forBundledModules; + } + async import( moduleIdentifier: string, dependencyTrackingContext?: RuntimeDependencyTrackingContext, @@ -843,9 +889,16 @@ export class Loader { init?: RequestInit, ): Promise => { try { - let shimmedModule = this.moduleShims.get( - this.asRequest(urlOrRequest, init).url, - ); + let shimmedModule = + this.moduleShims.get(this.asRequest(urlOrRequest, init).url) ?? + // Modules shimmed on the virtual network (e.g. base modules + // compiled into the host bundle) are served to every loader + // sharing that network, including loaders constructed outside the + // host's loader service. This is a module-fetch path, so a shim + // registered under a realm URL can't shadow a card-instance GET. + (await this.virtualNetwork?.getShimmedModule( + this.asRequest(urlOrRequest, init).url, + )); if (shimmedModule) { let response = new Response(); (response as any)[Symbol.for('shimmed-module')] = shimmedModule; @@ -940,6 +993,18 @@ export class Loader { } } + private vnShimIdentitiesCaptured = false; + + private captureVirtualNetworkShimIdentities() { + if (this.vnShimIdentitiesCaptured || !this.virtualNetwork) { + return; + } + this.vnShimIdentitiesCaptured = true; + for (let [id, module] of this.virtualNetwork.syncShimEntries()) { + this.captureIdentitiesOfModuleExports(module, id); + } + } + private captureIdentitiesOfModuleExports( module: any, moduleIdentifier: string, @@ -1027,6 +1092,15 @@ export class Loader { this.setCanonicalModuleURL(moduleIdentifier, canonicalURL); if (loaded.type === 'shimmed') { + // Loader-evaluated modules capture export identities dependency-first + // (a re-exporting module always evaluates after the module that + // declares the export, so first-wins capture lands on the declaring + // module). Shims carry no dependency chain, so a loader whose first + // shim load is a re-exporter would mis-attribute identities. Replay + // the network's whole sync-shim inventory once, in registration + // order (registrars put declaring modules first), before any + // individual shim's capture. + this.captureVirtualNetworkShimIdentities(); this.captureIdentitiesOfModuleExports(loaded.module, moduleIdentifier); this.setModule(moduleIdentifier, { diff --git a/packages/runtime-common/package-shim-handler.ts b/packages/runtime-common/package-shim-handler.ts index 2cccd4c7707..95b58a2fd10 100644 --- a/packages/runtime-common/package-shim-handler.ts +++ b/packages/runtime-common/package-shim-handler.ts @@ -440,10 +440,25 @@ export class PackageShimHandler { return null; }; + // Synchronously-shimmed modules in registration order. Loaders replay + // this inventory through their identity capture so that export + // identities don't depend on which shim a given loader happened to load + // first — loader-evaluated modules got that guarantee from + // dependency-first evaluation (a re-exporting module always evaluated + // after its source), but shims carry no dependency chain. Registration + // order therefore stands in for dependency order; registrars put + // declaring modules before their re-exporters. + private syncModules = new Map(); + + syncShimEntries(): ReadonlyMap { + return this.syncModules; + } + shimModule(moduleIdentifier: string, module: ModuleLike) { moduleIdentifier = this.resolveImport(moduleIdentifier); let key = trimModuleIdentifier(moduleIdentifier); this.moduleIds.set(key, async () => module); + this.syncModules.set(key, module); this.rememberExports(key, module); } @@ -506,6 +521,16 @@ export class PackageShimHandler { } } + // Module lookup for callers outside the fetch pipeline (the Loader's + // module-fetch path asks the virtual network for shims registered here + // before going to the network). Unlike `handle`, this is not restricted + // to the fake packages origin: the caller vouches that the URL is a + // module request, so a shim registered under a realm URL can be served + // without risking shadowing a card-instance GET of the same URL. + async lookupModule(url: string): Promise { + return (await this.getModule(url)) ?? (await this.getModuleByPrefix(url)); + } + private async getModule(url: string): Promise { let key = trimModuleIdentifier(url); let resolver = this.moduleIds.get(key); diff --git a/packages/runtime-common/virtual-network.ts b/packages/runtime-common/virtual-network.ts index 86f3ff54ad1..fb194a61f61 100644 --- a/packages/runtime-common/virtual-network.ts +++ b/packages/runtime-common/virtual-network.ts @@ -109,6 +109,22 @@ export class VirtualNetwork { this.packageShimHandler.shimAsyncModule(descriptor); } + // Lets a Loader serve a module shimmed on this network (under any URL, + // not just the fake packages origin) without a network fetch. Only the + // module-fetch path may call this: shims can be registered under realm + // URLs that also serve card instances, and only the caller knows the + // request is for a module rather than an instance document. + getShimmedModule(url: string): Promise { + return this.packageShimHandler.lookupModule(url); + } + + // Registration-ordered inventory of synchronously-shimmed modules, for + // loaders to replay through identity capture (see the note on the + // handler's syncShimEntries). + syncShimEntries(): ReadonlyMap { + return this.packageShimHandler.syncShimEntries(); + } + addURLMapping(from: URL, to: URL) { this.urlMappings.push([from.href, to.href]); // unresolveURL chases through urlMappings (via resolveURLMapping), so a new diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c238ef6ca67..78805c32af8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -877,8 +877,8 @@ importers: packages/base: dependencies: ember-provide-consume-context: - specifier: ^0.7.0 - version: 0.7.1(@ember/test-helpers@5.4.3(@babel/core@7.29.7))(ember-source@6.10.1(patch_hash=ea945024993105fb6cc4ae5cb5e9ea8e0eff6cd5fe0b0033c43dd0cf9453eb0d)(@glimmer/component@2.1.1)(rsvp@4.8.5)) + specifier: ^0.8.0 + version: 0.8.0(@ember/test-helpers@5.4.3(@babel/core@7.29.7))(@glimmer/component@2.1.1)(ember-source@6.10.1(patch_hash=ea945024993105fb6cc4ae5cb5e9ea8e0eff6cd5fe0b0033c43dd0cf9453eb0d)(@glimmer/component@2.1.1)(rsvp@4.8.5)) ember-source: specifier: 'catalog:' version: 6.10.1(patch_hash=ea945024993105fb6cc4ae5cb5e9ea8e0eff6cd5fe0b0033c43dd0cf9453eb0d)(@glimmer/component@2.1.1)(rsvp@4.8.5) @@ -922,6 +922,9 @@ importers: concurrently: specifier: 'catalog:' version: 8.2.2 + date-fns: + specifier: 'catalog:' + version: 2.30.0 ember-cli-htmlbars: specifier: ^6.3.0 version: 6.3.0