diff --git a/.github/workflows/generate-hacs-data.yml b/.github/workflows/generate-hacs-data.yml index 4117d1c5a5a..5e8dbf12954 100644 --- a/.github/workflows/generate-hacs-data.yml +++ b/.github/workflows/generate-hacs-data.yml @@ -45,16 +45,67 @@ jobs: categories: ${{ steps.set-matrix.outputs.categories }} steps: - id: set-matrix + env: + EVENT_NAME: ${{ github.event_name }} + INPUT_CATEGORY: ${{ inputs.category }} run: | - if [[ "${{ github.event_name }}" == "workflow_dispatch" ]] && [[ "${{ inputs.category }}" != "None" ]] && [[ "${{ inputs.category }}" != "" ]]; then - echo "categories=['${{ inputs.category }}']" >> $GITHUB_OUTPUT + if [[ "$EVENT_NAME" == "workflow_dispatch" ]] && [[ "$INPUT_CATEGORY" != "None" ]] && [[ -n "$INPUT_CATEGORY" ]]; then + categories=("$INPUT_CATEGORY") else - echo "categories=['appdaemon','integration','plugin','python_script','template','theme']" >> $GITHUB_OUTPUT + categories=(appdaemon integration plugin python_script template theme) fi + categories_json=$(printf '%s\n' "${categories[@]}" | jq -R . | jq -cs .) + echo "categories=$categories_json" >> $GITHUB_OUTPUT + echo "categories: $categories_json" + + # Fetches the currently published data once and shares it with every + # category leg, so the whole run works off one consistent baseline. + # Footgun / trade-off: any single fetch failing (after the retries) fails + # this job and therefore the whole run -- so one category's data being + # unavailable blocks publishing for all of them. This is intentional (one + # consistent snapshot), unlike the previous behaviour where a fetch failure + # was isolated to that category's own leg. + fetch-stored-data: + name: Fetch stored data + runs-on: ubuntu-latest + needs: generate-matrix + if: github.repository == 'hacs/integration' + timeout-minutes: 10 + steps: + - name: Fetch stored data + env: + CATEGORIES: ${{ needs.generate-matrix.outputs.categories }} + run: | + mkdir -p outputdata/stored + + fetch_stored_hacs_content() { + curl \ + --silent --show-error --fail --retry 5 --retry-all-errors \ + --connect-timeout 10 --max-time 60 \ + --header "User-Agent: HACS/Generator" \ + "https://data-v2.hacs.xyz/$1" \ + --output "$2" + } + + for category in $(echo "$CATEGORIES" | jq -r '.[]'); do + echo "Fetching stored data for $category" + fetch_stored_hacs_content "${category}/data.json" "outputdata/stored/${category}.json" + done + + echo "Fetching removed repositories" + fetch_stored_hacs_content "removed/repositories.json" "outputdata/stored/removed.json" + + - name: Upload stored data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: stored-data + path: outputdata/stored + if-no-files-found: error + retention-days: 1 category-data: runs-on: ubuntu-latest - needs: generate-matrix + needs: [generate-matrix, fetch-stored-data] if: github.repository == 'hacs/integration' name: Generate ${{ matrix.category }} data strategy: @@ -83,11 +134,18 @@ jobs: scripts/install/frontend scripts/install/pip_packages --requirement requirements_generate_data.txt + - name: Download stored data + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: stored-data + path: outputdata/stored + - name: Generate ${{ matrix.category }} data run: python3 -m scripts.data.generate_category_data ${{ matrix.category }} env: DATA_GENERATOR_TOKEN: ${{ secrets.DATA_GENERATOR_TOKEN }} FORCE_REPOSITORY_UPDATE: ${{ inputs.forceRepositoryUpdate }} + HACS_STORED_DATA_DIR: outputdata/stored - name: Validate output with JQ run: | @@ -129,7 +187,7 @@ jobs: summarize: name: Summarize runs-on: ubuntu-latest - needs: category-data + needs: [generate-matrix, category-data] if: ${{ always() && github.repository == 'hacs/integration' }} outputs: changedCategories: ${{ steps.combined.outputs.changedCategories }} @@ -146,6 +204,7 @@ jobs: env: HACS_CHANGED_PCT_TARGET: ${{ vars.HACS_CHANGED_PCT_TARGET }} HACS_DIFF_TARGET: ${{ vars.HACS_DIFF_TARGET }} + CATEGORIES: ${{ needs.generate-matrix.outputs.categories }} with: script: | const fs = require('fs'); @@ -156,9 +215,13 @@ jobs: core.info(`[global] diffTarget: ${diffTarget}`); - const subDirectories = fs.readdirSync("outputdata", { withFileTypes: true }) - .filter(entry => entry.isDirectory()) - .map(entry => entry.name) + // Only consider the expected category directories that produced a + // summary -- ignores the stored-data artifact and any category + // whose leg failed to write one. + const categories = JSON.parse(process.env.CATEGORIES); + const subDirectories = categories.filter( + category => fs.existsSync(`outputdata/${category}/summary.json`) + ) for (const directory of subDirectories) { let changedPctTarget = Number(process.env.HACS_CHANGED_PCT_TARGET) @@ -287,7 +350,7 @@ jobs: notify_on_failure: runs-on: ubuntu-latest name: Trigger Discord notification when jobs fail - needs: ["generate-matrix", "category-data", "summarize", "publish"] + needs: ["generate-matrix", "fetch-stored-data", "category-data", "summarize", "publish"] if: ${{ always() && github.repository == 'hacs/integration' && contains(join(needs.*.result, ','), 'failure') && github.event_name == 'schedule' }} steps: - name: Send notification diff --git a/scripts/data/generate_category_data.py b/scripts/data/generate_category_data.py index fe04c61df21..ca04f65f252 100644 --- a/scripts/data/generate_category_data.py +++ b/scripts/data/generate_category_data.py @@ -58,6 +58,53 @@ OUTPUT_DIR = os.path.join(os.getcwd(), "outputdata") COMPARE_IGNORE = {"etag_releases", "etag_repository", "last_fetched"} +# Directory holding the preflight snapshot of the existing published data. When +# set, snapshots are read from here instead of fetched; unset outside the workflow. +STORED_DATA_DIR = os.getenv("HACS_STORED_DATA_DIR") + + +def _read_snapshot[T]( + hacs: AdjustedHacs, + filename: str, + expected_type: type[T], +) -> T | None: + """Read a snapshot file from ``STORED_DATA_DIR``. + + Returns the parsed JSON when available, or ``None`` (so the caller fetches + instead) when the dir is unset, the file is missing or unreadable, or the + content is not of ``expected_type``. + """ + if not STORED_DATA_DIR: + return None + try: + with open(os.path.join(STORED_DATA_DIR, filename), encoding="utf-8") as file: + data = json.load(file) + except (OSError, json.JSONDecodeError) as err: + hacs.log.warning("Could not read snapshot %s (%s), fetching instead", filename, err) + return None + if not isinstance(data, expected_type): + hacs.log.warning( + "Snapshot %s has unexpected type %s, fetching instead", + filename, + type(data).__name__, + ) + return None + return data + + +async def get_stored_data(hacs: AdjustedHacs, category: str) -> dict[str, dict[str, Any]]: + """Return existing category data from the snapshot dir when available, else fetch.""" + if (data := _read_snapshot(hacs, f"{category}.json", dict)) is not None: + return data + return await hacs.data_client.get_data(category, validate=False) + + +async def get_removed_repositories(hacs: AdjustedHacs) -> list[str]: + """Return the removed-repositories list from the snapshot dir when available, else fetch.""" + if (removed := _read_snapshot(hacs, "removed.json", list)) is not None: + return removed + return await hacs.data_client.get_repositories("removed") + def jsonprint(data: any): print( @@ -309,7 +356,7 @@ async def generate_data_for_category( removed = ( [] if repository_name is not None - else await self.data_client.get_repositories("removed") + else await get_removed_repositories(self) ) await self.data.register_base_data( category, @@ -461,7 +508,7 @@ async def generate_category_data(category: str, repository_name: str = None): os.makedirs(os.path.join(OUTPUT_DIR, category), exist_ok=True) os.makedirs(os.path.join(OUTPUT_DIR, "diff"), exist_ok=True) force = os.environ.get("FORCE_REPOSITORY_UPDATE") == "True" - stored_data = await hacs.data_client.get_data(category, validate=False) + stored_data = await get_stored_data(hacs, category) current_data = ( next( ( diff --git a/tests/scripts/data/test_generate_category_data.py b/tests/scripts/data/test_generate_category_data.py index 6b602d9feb8..c72f9d791c5 100644 --- a/tests/scripts/data/test_generate_category_data.py +++ b/tests/scripts/data/test_generate_category_data.py @@ -2,13 +2,19 @@ import asyncio import json +import logging import os from typing import Any from homeassistant.core import HomeAssistant import pytest -from scripts.data.generate_category_data import OUTPUT_DIR, generate_category_data +from scripts.data.generate_category_data import ( + OUTPUT_DIR, + generate_category_data, + get_removed_repositories, + get_stored_data, +) from tests.common import ( FIXTURES_PATH, @@ -311,3 +317,160 @@ async def test_generate_category_data_with_30plus_prereleases( f"scripts/data/test_generate_category_data_with_30plus_prereleases/{ category_test_data['category']}.json", ) + + +# Shapes mirror the real data client: get_data -> {repo-id: {...}}, removed -> [full_name]. +_FETCHED_STORED = {"1296269": {"full_name": "octocat/Hello-World", "category": "plugin"}} +_FETCHED_REMOVED = ["octocat/removed-repo"] + + +class _StubDataClient: + """Minimal data client recording that a fetch happened.""" + + def __init__(self) -> None: + self.calls: list[tuple[str, str]] = [] + + async def get_data(self, section: str, *, validate: bool) -> dict[str, dict[str, Any]]: + self.calls.append(("get_data", section)) + return _FETCHED_STORED + + async def get_repositories(self, section: str) -> list[str]: + self.calls.append(("get_repositories", section)) + return _FETCHED_REMOVED + + +class _StubHacs: + """Minimal HACS stand-in exposing only a data client and a logger.""" + + def __init__(self) -> None: + self.data_client = _StubDataClient() + self.log = logging.getLogger("test.generate_category_data") + + +_MODULE = "scripts.data.generate_category_data" + + +async def test_get_stored_data_reads_from_existing_dir(tmp_path, monkeypatch): + """When the snapshot dir is set, stored data is read from it, not fetched.""" + monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path)) + payload = {"1": {"full_name": "octocat/Hello-World"}} + (tmp_path / "integration.json").write_text(json.dumps(payload)) + + hacs = _StubHacs() + assert await get_stored_data(hacs, "integration") == payload + assert hacs.data_client.calls == [] + + +async def test_get_removed_repositories_reads_from_existing_dir(tmp_path, monkeypatch): + """When the snapshot dir is set, the removed list is read from it, not fetched.""" + monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path)) + removed = ["octocat/Hello-World", "hacs/integration"] + (tmp_path / "removed.json").write_text(json.dumps(removed)) + + hacs = _StubHacs() + assert await get_removed_repositories(hacs) == removed + assert hacs.data_client.calls == [] + + +async def test_get_stored_data_falls_back_to_fetch(monkeypatch): + """Without the snapshot dir, stored data is fetched from the data client.""" + monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", None) + hacs = _StubHacs() + assert await get_stored_data(hacs, "plugin") == _FETCHED_STORED + assert hacs.data_client.calls == [("get_data", "plugin")] + + +async def test_get_removed_repositories_falls_back_to_fetch(monkeypatch): + """Without the snapshot dir, the removed list is fetched from the data client.""" + monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", None) + hacs = _StubHacs() + assert await get_removed_repositories(hacs) == _FETCHED_REMOVED + assert hacs.data_client.calls == [("get_repositories", "removed")] + + +async def test_get_stored_data_falls_back_when_snapshot_missing(tmp_path, monkeypatch): + """A missing snapshot file falls back to fetching instead of raising.""" + monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path)) + hacs = _StubHacs() + assert await get_stored_data(hacs, "plugin") == _FETCHED_STORED + assert hacs.data_client.calls == [("get_data", "plugin")] + + +async def test_get_stored_data_falls_back_when_snapshot_invalid(tmp_path, monkeypatch): + """An invalid-JSON snapshot file falls back to fetching instead of raising.""" + monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path)) + (tmp_path / "plugin.json").write_text("{ not valid json") + + hacs = _StubHacs() + assert await get_stored_data(hacs, "plugin") == _FETCHED_STORED + assert hacs.data_client.calls == [("get_data", "plugin")] + + +async def test_get_removed_repositories_falls_back_when_snapshot_missing(tmp_path, monkeypatch): + """A missing removed snapshot falls back to fetching instead of raising.""" + monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path)) + hacs = _StubHacs() + assert await get_removed_repositories(hacs) == _FETCHED_REMOVED + assert hacs.data_client.calls == [("get_repositories", "removed")] + + +async def test_get_stored_data_falls_back_on_wrong_shape(tmp_path, monkeypatch): + """Valid JSON of the wrong type (list, not dict) falls back to fetching.""" + monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path)) + (tmp_path / "plugin.json").write_text(json.dumps(["not", "a", "dict"])) + + hacs = _StubHacs() + assert await get_stored_data(hacs, "plugin") == _FETCHED_STORED + assert hacs.data_client.calls == [("get_data", "plugin")] + + +async def test_get_removed_repositories_falls_back_on_wrong_shape(tmp_path, monkeypatch): + """Valid JSON of the wrong type (dict, not list) falls back to fetching.""" + monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path)) + (tmp_path / "removed.json").write_text(json.dumps({"not": "a list"})) + + hacs = _StubHacs() + assert await get_removed_repositories(hacs) == _FETCHED_REMOVED + assert hacs.data_client.calls == [("get_repositories", "removed")] + + +@pytest.mark.parametrize( + "category_test_data", category_test_data_parametrized(categories=["integration"]) +) +async def test_generate_category_data_from_stored_snapshot( + hass: HomeAssistant, + response_mocker: ResponseMocker, + snapshots: SnapshotFixture, + category_test_data: CategoryTestData, + tmp_path, + monkeypatch, +): + """Generating from the stored snapshot reproduces the fetch-path output. + + Feeds the same inputs the fetch path uses (empty category data + the removed + fixture) through STORED_DATA_DIR, and asserts the generated data.json matches + the fetch-path snapshot -- i.e. sourcing the baseline from disk is equivalent + to fetching it. + """ + category = category_test_data["category"] + + stored_dir = tmp_path / "stored" + stored_dir.mkdir() + (stored_dir / f"{category}.json").write_text("{}") + with open( + os.path.join( + FIXTURES_PATH, "proxy", "data-v2.hacs.xyz", "removed", "repositories.json" + ), + encoding="utf-8", + ) as file: + (stored_dir / "removed.json").write_text(file.read()) + monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(stored_dir)) + + await generate_category_data(category) + + with open(f"{OUTPUT_DIR}/{category}/data.json", encoding="utf-8") as file: + snapshots.assert_match( + safe_json_dumps(recursive_remove_key( + json.loads(file.read()), ("last_fetched",))), + f"scripts/data/generate_category_data/{category}//data.json", + ) diff --git a/tests/snapshots/api-usage/tests/scripts/data/test_generate_category_datatest-generate-category-data-from-stored-snapshot-hacs-test-org-integration-basic.json b/tests/snapshots/api-usage/tests/scripts/data/test_generate_category_datatest-generate-category-data-from-stored-snapshot-hacs-test-org-integration-basic.json new file mode 100644 index 00000000000..8c0b57fadc8 --- /dev/null +++ b/tests/snapshots/api-usage/tests/scripts/data/test_generate_category_datatest-generate-category-data-from-stored-snapshot-hacs-test-org-integration-basic.json @@ -0,0 +1,19 @@ +{ + "tests/scripts/data/test_generate_category_data.py::test_generate_category_data_from_stored_snapshot[hacs-test-org/integration-basic]": { + "https://api.github.com/rate_limit": 1, + "https://api.github.com/repos/hacs-test-org/integration-basic": 1, + "https://api.github.com/repos/hacs-test-org/integration-basic-custom": 1, + "https://api.github.com/repos/hacs-test-org/integration-basic-custom/contents/custom_components/example/manifest.json": 1, + "https://api.github.com/repos/hacs-test-org/integration-basic-custom/contents/hacs.json": 1, + "https://api.github.com/repos/hacs-test-org/integration-basic-custom/git/trees/1.0.0": 1, + "https://api.github.com/repos/hacs-test-org/integration-basic-custom/releases": 1, + "https://api.github.com/repos/hacs-test-org/integration-basic/contents/custom_components/example/manifest.json": 1, + "https://api.github.com/repos/hacs-test-org/integration-basic/contents/hacs.json": 1, + "https://api.github.com/repos/hacs-test-org/integration-basic/git/trees/1.0.0": 1, + "https://api.github.com/repos/hacs-test-org/integration-basic/releases": 1, + "https://api.github.com/repos/hacs/default/contents/integration": 1, + "https://api.github.com/repos/hacs/integration": 1, + "https://api.github.com/repos/hacs/integration/branches/main": 1, + "https://api.github.com/repos/hacs/integration/releases": 1 + } +} \ No newline at end of file