diff --git a/.github/allure-bundle-size.baseline.json b/.github/allure-bundle-size.baseline.json index 58ee34600b0..16b8b1a3f1c 100644 --- a/.github/allure-bundle-size.baseline.json +++ b/.github/allure-bundle-size.baseline.json @@ -1,8 +1,8 @@ { "rootPackage": "allure", - "measuredAt": "2026-09-02T07:44:48.293Z", + "measuredAt": "2026-09-03T07:55:54.137Z", "internalClosureCount": 34, "selfBytes": 31776768, - "depsBytes": 65581056, - "totalBytes": 97357824 + "depsBytes": 65585152, + "totalBytes": 97361920 } diff --git a/docs/quality_gates.md b/docs/quality_gates.md index 59f3ec667d5..0edd4684526 100644 --- a/docs/quality_gates.md +++ b/docs/quality_gates.md @@ -37,6 +37,45 @@ If `qualityGate` is configured and `--rerun` is enabled, Allure runs the test co Use `--rerun=0` or remove `--rerun` when the quality gate should validate the run. +## Generated artifacts + +When quality gates are enabled, Allure writes `quality-gate.json` to the generated report root. The file contains a flat +list of rule results and keeps related tests as report-scoped IDs in `testResults`. + +```json +[ + { + "success": false, + "expected": 0, + "actual": 1, + "rule": "maxFailures", + "message": "The number of failed tests 1 exceeds the allowed threshold value 0", + "environment": "chrome", + "testResults": ["4f1c2d"] + } +] +``` + +Use the generated report root `test-results.json` to resolve these IDs. The registry contains compact test details that +are enough to list and link related tests. + +```json +{ + "byId": { + "4f1c2d": { + "id": "4f1c2d", + "name": "checkout rejects expired card", + "status": "failed", + "duration": 1234, + "environment": "chrome" + } + } +} +``` + +Together, these files allow CI integrations to show which rule failed and which tests are related to it without reading +the full per-test result files. + ## Using external rules You can use external quality gate rules implemented by the community – just provide them to the `use` field in the quality gate configuration: diff --git a/packages/core/src/report.ts b/packages/core/src/report.ts index b66394447f1..40a4c2da08c 100644 --- a/packages/core/src/report.ts +++ b/packages/core/src/report.ts @@ -64,6 +64,8 @@ const INIT_REQUIRED_ERROR_MESSAGE = "report is not initialised. Call the start() const DEFAULT_READ_CONCURRENCY = 64; const MAX_READ_CONCURRENCY = 256; const TEST_RESULTS_REGISTRY_FILENAME = "test-results.json"; +const QUALITY_GATE_RESULTS_FILENAME = "quality-gate.json"; +const ROOT_INTEGRATION_FILENAMES = new Set([TEST_RESULTS_REGISTRY_FILENAME, QUALITY_GATE_RESULTS_FILENAME]); const readConcurrency = () => { const parsed = Number.parseInt(process.env.ALLURE_READ_CONCURRENCY ?? "", 10); @@ -263,6 +265,9 @@ export class AllureReport { } await this.#writeTestResultRegistry(); + if (this.#qualityGate) { + await this.#writeQualityGateFiles(); + } await this.#writeSummaryFiles(); await this.#generateRootSummary(); @@ -285,9 +290,10 @@ export class AllureReport { const summariesSnapshot = this.#cloneSummariesByPluginId(); const uploadProgressMessage = reportsToPublish.length === 1 ? `Publishing "${reportsToPublish[0].pluginId}" report` : "Publishing reports"; + const rootRemoteReportFiles = this.#getRootRemoteReportFiles(); const totalFilesToUpload = reportsToPublish.reduce((acc, report) => acc + Object.keys(report.files).length, 0) + - (this.#testResultsRegistryPath ? 1 : 0); + Object.keys(rootRemoteReportFiles).length; let summariesMutated = false; let reportCreated = false; let publishErrorMessage = "Report upload has failed, the report won't be published"; @@ -333,12 +339,12 @@ export class AllureReport { } } - if (this.#testResultsRegistryPath) { + if (Object.keys(rootRemoteReportFiles).length > 0) { publishErrorMessage = "Test results registry upload has failed, the report won't be published"; await client.uploadReport({ reportUuid: this.reportUuid, - files: { [TEST_RESULTS_REGISTRY_FILENAME]: this.#testResultsRegistryPath }, + files: rootRemoteReportFiles, onProgress: incrementUploadProgress, }); } @@ -1026,6 +1032,16 @@ export class AllureReport { ); }; + #getRootRemoteReportFiles = (): Record => ({ + ...(this.#testResultsRegistryPath ? { [TEST_RESULTS_REGISTRY_FILENAME]: this.#testResultsRegistryPath } : {}), + }); + + #writeQualityGateFiles = async (): Promise => { + const qualityGateResults = await this.#store.qualityGateResults(); + + await this.#reportFiles.addFile(QUALITY_GATE_RESULTS_FILENAME, Buffer.from(JSON.stringify(qualityGateResults))); + }; + #generateRootSummary = async (): Promise => { const summaries = [...this.#summariesByPluginId.values()].map(clonePluginSummary); @@ -1126,7 +1142,7 @@ export class AllureReport { const reportContent = await readdir(reportPath); for (const entry of reportContent) { - if (entry === TEST_RESULTS_REGISTRY_FILENAME) { + if (ROOT_INTEGRATION_FILENAMES.has(entry)) { continue; } @@ -1168,14 +1184,6 @@ export class AllureReport { console.info(`- ${href}`); }); } - - if (!this.#qualityGate) { - return; - } - - const qualityGateResults = await this.#store.qualityGateResultsByEnv(); - - await writeFile(join(this.#output, "quality-gate.json"), JSON.stringify(qualityGateResults)); } finally { await this.#finishPerfMetrics(); } diff --git a/packages/core/test/report.test.ts b/packages/core/test/report.test.ts index 513b4bc9e9c..38318abb637 100644 --- a/packages/core/test/report.test.ts +++ b/packages/core/test/report.test.ts @@ -145,17 +145,51 @@ describe("report", () => { const registry = JSON.parse(await readFile(join(output, "test-results.json"), "utf8")); const id = md5("result-1"); - expect(registry).toEqual({ - byId: { - [id]: { - id, - name: "failed test", - duration: 123, - status: "failed", - }, + const registryEntry = registry.byId[id]; + + expect(registryEntry).toEqual( + expect.objectContaining({ + id, + name: "failed test", + duration: 123, + status: "failed", + }), + ); + expect(registryEntry).not.toHaveProperty("labels"); + expect(registryEntry).not.toHaveProperty("steps"); + expect(registryEntry).not.toHaveProperty("attachments"); + expect(registryEntry).not.toHaveProperty("error"); + await expect(readFile(join(output, "index.html"), "utf8")).resolves.toBe("index"); + }); + + it("should write quality gate results with related test ids", async () => { + const output = await mkdtemp(join(tmpdir(), "allure3-quality-gate-resolved-")); + const config = await resolveConfig({ + name: "Allure Report", + output, + qualityGate: { + rules: [], }, }); - await expect(readFile(join(output, "index.html"), "utf8")).resolves.toBe("index"); + const allureReport = new AllureReport(config); + + await allureReport.start(); + const testResultId = "result-1"; + const qualityGateResult = { + success: false, + expected: 0, + actual: 1, + rule: "maxFailures", + message: "Failed tests exceed threshold", + testResults: [testResultId], + }; + + allureReport.realtimeDispatcher.sendQualityGateResults([qualityGateResult]); + await allureReport.done(); + + const qualityGateResults = JSON.parse(await readFile(join(output, "quality-gate.json"), "utf8")); + + expect(qualityGateResults).toEqual([qualityGateResult]); }); it("should not allow call done() before start()", async () => { @@ -784,6 +818,64 @@ describe("report", () => { ); }); + it("should keep quality gate results local and omit them from remote root uploads", async () => { + const output = await mkdtemp(join(tmpdir(), "allure3-quality-gate-local-only-")); + const p1 = createPlugin("p1", true, { publish: true }); + const config = await resolveConfig({ + name: "Allure Report", + output, + qualityGate: { + rules: [], + }, + }); + + config.plugins = [p1]; + (p1.plugin.done as Mock).mockImplementation(async (context) => { + await context.reportFiles.addFile("index.html", Buffer.from("index")); + }); + + const allureReport = new AllureReport({ + ...config, + allureService: allureServiceConfig(), + }); + + await allureReport.start(); + allureReport.realtimeDispatcher.sendQualityGateResults([ + { + success: false, + expected: 0, + actual: 1, + rule: "maxFailures", + message: "Failed tests exceed threshold", + testResults: [], + }, + ]); + await allureReport.done(); + + const qualityGateResults = JSON.parse(await readFile(join(output, "quality-gate.json"), "utf8")); + const uploadedRootFiles = (AllureServiceClientMock.prototype.uploadReport as Mock).mock.calls + .map(([options]) => options as { pluginId?: string; files: Record }) + .filter(({ pluginId }) => pluginId === undefined) + .map(({ files }) => files); + + expect(qualityGateResults).toEqual([ + { + success: false, + expected: 0, + actual: 1, + rule: "maxFailures", + message: "Failed tests exceed threshold", + testResults: [], + }, + ]); + expect(AllureServiceClientMock.prototype.uploadReport).toHaveBeenCalledWith( + expect.objectContaining({ + files: { "test-results.json": expect.any(String) }, + }), + ); + expect(uploadedRootFiles).toEqual([{ "test-results.json": expect.any(String) }]); + }); + const verifyUploadOptionsForwarding = async (uploadConcurrency?: number) => { const p1 = createPlugin("p1", true, { publish: true }); const config = await resolveConfig({ name: "Allure Report" }); diff --git a/packages/plugin-api/src/plugin.ts b/packages/plugin-api/src/plugin.ts index 2b794d919e1..007b9d5cd5c 100644 --- a/packages/plugin-api/src/plugin.ts +++ b/packages/plugin-api/src/plugin.ts @@ -57,7 +57,7 @@ export interface PluginContext { /** * Reduced test result information shared by report integrations. */ -export type TestResultSummary = Pick; +export type TestResultSummary = Pick; export interface TestResultRegistry { byId: Record; diff --git a/packages/plugin-api/src/utils/summary.ts b/packages/plugin-api/src/utils/summary.ts index 54421ad46d9..196dead08ca 100644 --- a/packages/plugin-api/src/utils/summary.ts +++ b/packages/plugin-api/src/utils/summary.ts @@ -14,6 +14,7 @@ export const convertToTestResultSummary = (tr: TestResult): TestResultSummary => name: tr.name, duration: tr.duration, status: tr.status, + ...(tr.environment ? { environment: tr.environment } : {}), }); export const createTestResultRegistry = (testResults: TestResult[]): TestResultRegistry => ({ diff --git a/packages/plugin-api/test/summary.test.ts b/packages/plugin-api/test/summary.test.ts index fae9d369ed2..072bf0ad3da 100644 --- a/packages/plugin-api/test/summary.test.ts +++ b/packages/plugin-api/test/summary.test.ts @@ -30,25 +30,46 @@ const testResult = (args: Partial = {}): TestResult => ({ describe("summary utils", () => { it("convertToTestResultSummary maps fields", () => { expect( - convertToTestResultSummary(testResult({ id: "id-1", name: "name-1", status: "failed", duration: 123 })), + convertToTestResultSummary( + testResult({ + id: "id-1", + name: "name-1", + status: "failed", + duration: 123, + environment: "chrome", + }), + ), ).toEqual({ id: "id-1", name: "name-1", status: "failed", duration: 123, + environment: "chrome", }); }); it("createTestResultRegistry indexes reduced test results by id", () => { expect( createTestResultRegistry([ - testResult({ id: "id-1", name: "name-1", status: "failed", duration: 123 }), - testResult({ id: "id-2", name: "name-2", status: "passed", duration: 456 }), + testResult({ + id: "id-1", + name: "name-1", + status: "failed", + duration: 123, + environment: "chrome", + }), + testResult({ id: "id-2", name: "name-2", status: "passed", duration: 456, environment: "firefox" }), ]), ).toEqual({ byId: { - "id-1": { id: "id-1", name: "name-1", duration: 123, status: "failed" }, - "id-2": { id: "id-2", name: "name-2", duration: 456, status: "passed" }, + "id-1": { + id: "id-1", + name: "name-1", + duration: 123, + environment: "chrome", + status: "failed", + }, + "id-2": { id: "id-2", name: "name-2", duration: 456, environment: "firefox", status: "passed" }, }, }); }); diff --git a/packages/web-awesome/test/index.test.tsx b/packages/web-awesome/test/index.test.tsx index 5f67eba3c05..949f569e182 100644 --- a/packages/web-awesome/test/index.test.tsx +++ b/packages/web-awesome/test/index.test.tsx @@ -138,7 +138,7 @@ describe("App", () => { await waitFor(() => { expect(screen.queryByTestId("loader")).not.toBeInTheDocument(); }); - }); + }, 30_000); it("should fetch env-scoped data when an environment is selected", async () => { await selectEnvironment("");