Fetch existing HACS data once before generation

.github/workflows/generate-hacs-data.yml

@@ -45,16 +45,67 @@ jobs:
       categories: ${{ steps.set-matrix.outputs.categories }}
     steps:
       - id: set-matrix
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          INPUT_CATEGORY: ${{ inputs.category }}
         run: |
-          if [[ "${{ github.event_name }}" == "workflow_dispatch"  ]] && [[ "${{ inputs.category }}" != "None"  ]] && [[ "${{ inputs.category }}" != ""  ]]; then
-            echo "categories=['${{ inputs.category }}']" >> $GITHUB_OUTPUT
+          if [[ "$EVENT_NAME" == "workflow_dispatch" ]] && [[ "$INPUT_CATEGORY" != "None" ]] && [[ -n "$INPUT_CATEGORY" ]]; then
+            categories=("$INPUT_CATEGORY")
           else
-            echo "categories=['appdaemon','integration','plugin','python_script','template','theme']" >> $GITHUB_OUTPUT
+            categories=(appdaemon integration plugin python_script template theme)
           fi
+          categories_json=$(printf '%s\n' "${categories[@]}" | jq -R . | jq -cs .)
+          echo "categories=$categories_json" >> $GITHUB_OUTPUT
+          echo "categories: $categories_json"
+
+  # Fetches the currently published data once and shares it with every
+  # category leg, so the whole run works off one consistent baseline.
+  # Footgun / trade-off: any single fetch failing (after the retries) fails
+  # this job and therefore the whole run -- so one category's data being
+  # unavailable blocks publishing for all of them. This is intentional (one
+  # consistent snapshot), unlike the previous behaviour where a fetch failure
+  # was isolated to that category's own leg.
+  fetch-stored-data:
+    name: Fetch stored data
+    runs-on: ubuntu-latest
+    needs: generate-matrix
+    if: github.repository == 'hacs/integration'
+    timeout-minutes: 10
+    steps:
+      - name: Fetch stored data
+        env:
+          CATEGORIES: ${{ needs.generate-matrix.outputs.categories }}
+        run: |
+          mkdir -p outputdata/stored
+
+          fetch_stored_hacs_content() {
+            curl \
+              --silent --show-error --fail --retry 5 --retry-all-errors \
+              --connect-timeout 10 --max-time 60 \
+              --header "User-Agent: HACS/Generator" \
+              "https://data-v2.hacs.xyz/$1" \
+              --output "$2"
+          }
+
+          for category in $(echo "$CATEGORIES" | jq -r '.[]'); do
+            echo "Fetching stored data for $category"
+            fetch_stored_hacs_content "${category}/data.json" "outputdata/stored/${category}.json"
+          done
+
+          echo "Fetching removed repositories"
+          fetch_stored_hacs_content "removed/repositories.json" "outputdata/stored/removed.json"
+
+      - name: Upload stored data
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: stored-data
+          path: outputdata/stored
+          if-no-files-found: error
+          retention-days: 1
 
   category-data:
     runs-on: ubuntu-latest
-    needs: generate-matrix
+    needs: [generate-matrix, fetch-stored-data]
     if: github.repository == 'hacs/integration'
     name: Generate ${{ matrix.category }} data
     strategy:
@@ -83,11 +134,18 @@ jobs:
           scripts/install/frontend
           scripts/install/pip_packages --requirement requirements_generate_data.txt
 
+      - name: Download stored data
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: stored-data
+          path: outputdata/stored
+
       - name: Generate ${{ matrix.category }} data
         run: python3 -m scripts.data.generate_category_data ${{ matrix.category }}
         env:
           DATA_GENERATOR_TOKEN: ${{ secrets.DATA_GENERATOR_TOKEN }}
           FORCE_REPOSITORY_UPDATE: ${{ inputs.forceRepositoryUpdate }}
+          HACS_STORED_DATA_DIR: outputdata/stored
 
       - name: Validate output with JQ
         run: |
@@ -129,7 +187,7 @@ jobs:
   summarize:
     name: Summarize
     runs-on: ubuntu-latest
-    needs: category-data
+    needs: [generate-matrix, category-data]
     if: ${{ always() && github.repository == 'hacs/integration' }}
     outputs:
       changedCategories: ${{ steps.combined.outputs.changedCategories }}
@@ -146,6 +204,7 @@ jobs:
         env:
           HACS_CHANGED_PCT_TARGET: ${{ vars.HACS_CHANGED_PCT_TARGET }}
           HACS_DIFF_TARGET: ${{ vars.HACS_DIFF_TARGET }}
+          CATEGORIES: ${{ needs.generate-matrix.outputs.categories }}
         with:
           script: |
             const fs = require('fs');
@@ -156,9 +215,13 @@ jobs:
 
             core.info(`[global] diffTarget: ${diffTarget}`);
 
-            const subDirectories = fs.readdirSync("outputdata", { withFileTypes: true })
-                .filter(entry => entry.isDirectory())
-                .map(entry => entry.name)
+            // Only consider the expected category directories that produced a
+            // summary -- ignores the stored-data artifact and any category
+            // whose leg failed to write one.
+            const categories = JSON.parse(process.env.CATEGORIES);
+            const subDirectories = categories.filter(
+                category => fs.existsSync(`outputdata/${category}/summary.json`)
+            )
 
             for (const directory of subDirectories) {
               let changedPctTarget = Number(process.env.HACS_CHANGED_PCT_TARGET)
@@ -287,7 +350,7 @@ jobs:
   notify_on_failure:
     runs-on: ubuntu-latest
     name: Trigger Discord notification when jobs fail
-    needs: ["generate-matrix", "category-data", "summarize", "publish"]
+    needs: ["generate-matrix", "fetch-stored-data", "category-data", "summarize", "publish"]
     if: ${{ always() && github.repository == 'hacs/integration' && contains(join(needs.*.result, ','), 'failure') && github.event_name == 'schedule' }}
     steps:
       - name: Send notification

scripts/data/generate_category_data.py

@@ -58,6 +58,53 @@
 OUTPUT_DIR = os.path.join(os.getcwd(), "outputdata")
 COMPARE_IGNORE = {"etag_releases", "etag_repository", "last_fetched"}
 
+# Directory holding the preflight snapshot of the existing published data. When
+# set, snapshots are read from here instead of fetched; unset outside the workflow.
+STORED_DATA_DIR = os.getenv("HACS_STORED_DATA_DIR")
+
+
+def _read_snapshot[T](
+    hacs: AdjustedHacs,
+    filename: str,
+    expected_type: type[T],
+) -> T | None:
+    """Read a snapshot file from ``STORED_DATA_DIR``.
+
+    Returns the parsed JSON when available, or ``None`` (so the caller fetches
+    instead) when the dir is unset, the file is missing or unreadable, or the
+    content is not of ``expected_type``.
+    """
+    if not STORED_DATA_DIR:
+        return None
+    try:
+        with open(os.path.join(STORED_DATA_DIR, filename), encoding="utf-8") as file:
+            data = json.load(file)
+    except (OSError, json.JSONDecodeError) as err:
+        hacs.log.warning("Could not read snapshot %s (%s), fetching instead", filename, err)
+        return None
+    if not isinstance(data, expected_type):
+        hacs.log.warning(
+            "Snapshot %s has unexpected type %s, fetching instead",
+            filename,
+            type(data).__name__,
+        )
+        return None
+    return data
+
+
+async def get_stored_data(hacs: AdjustedHacs, category: str) -> dict[str, dict[str, Any]]:
+    """Return existing category data from the snapshot dir when available, else fetch."""
+    if (data := _read_snapshot(hacs, f"{category}.json", dict)) is not None:
+        return data
+    return await hacs.data_client.get_data(category, validate=False)
+
+
+async def get_removed_repositories(hacs: AdjustedHacs) -> list[str]:
+    """Return the removed-repositories list from the snapshot dir when available, else fetch."""
+    if (removed := _read_snapshot(hacs, "removed.json", list)) is not None:
+        return removed
+    return await hacs.data_client.get_repositories("removed")
+
 
 def jsonprint(data: any):
     print(
@@ -309,7 +356,7 @@ async def generate_data_for_category(
         removed = (
             []
             if repository_name is not None
-            else await self.data_client.get_repositories("removed")
+            else await get_removed_repositories(self)
         )
         await self.data.register_base_data(
             category,
@@ -461,7 +508,7 @@ async def generate_category_data(category: str, repository_name: str = None):
         os.makedirs(os.path.join(OUTPUT_DIR, category), exist_ok=True)
         os.makedirs(os.path.join(OUTPUT_DIR, "diff"), exist_ok=True)
         force = os.environ.get("FORCE_REPOSITORY_UPDATE") == "True"
-        stored_data = await hacs.data_client.get_data(category, validate=False)
+        stored_data = await get_stored_data(hacs, category)
         current_data = (
             next(
                 (

tests/scripts/data/test_generate_category_data.py

@@ -2,13 +2,19 @@
 
 import asyncio
 import json
+import logging
 import os
 from typing import Any
 
 from homeassistant.core import HomeAssistant
 import pytest
 
-from scripts.data.generate_category_data import OUTPUT_DIR, generate_category_data
+from scripts.data.generate_category_data import (
+    OUTPUT_DIR,
+    generate_category_data,
+    get_removed_repositories,
+    get_stored_data,
+)
 
 from tests.common import (
     FIXTURES_PATH,
@@ -311,3 +317,160 @@ async def test_generate_category_data_with_30plus_prereleases(
         f"scripts/data/test_generate_category_data_with_30plus_prereleases/{
             category_test_data['category']}.json",
     )
+
+
+# Shapes mirror the real data client: get_data -> {repo-id: {...}}, removed -> [full_name].
+_FETCHED_STORED = {"1296269": {"full_name": "octocat/Hello-World", "category": "plugin"}}
+_FETCHED_REMOVED = ["octocat/removed-repo"]
+
+
+class _StubDataClient:
+    """Minimal data client recording that a fetch happened."""
+
+    def __init__(self) -> None:
+        self.calls: list[tuple[str, str]] = []
+
+    async def get_data(self, section: str, *, validate: bool) -> dict[str, dict[str, Any]]:
+        self.calls.append(("get_data", section))
+        return _FETCHED_STORED
+
+    async def get_repositories(self, section: str) -> list[str]:
+        self.calls.append(("get_repositories", section))
+        return _FETCHED_REMOVED
+
+
+class _StubHacs:
+    """Minimal HACS stand-in exposing only a data client and a logger."""
+
+    def __init__(self) -> None:
+        self.data_client = _StubDataClient()
+        self.log = logging.getLogger("test.generate_category_data")
+
+
+_MODULE = "scripts.data.generate_category_data"
+
+
+async def test_get_stored_data_reads_from_existing_dir(tmp_path, monkeypatch):
+    """When the snapshot dir is set, stored data is read from it, not fetched."""
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path))
+    payload = {"1": {"full_name": "octocat/Hello-World"}}
+    (tmp_path / "integration.json").write_text(json.dumps(payload))
+
+    hacs = _StubHacs()
+    assert await get_stored_data(hacs, "integration") == payload
+    assert hacs.data_client.calls == []
+
+
+async def test_get_removed_repositories_reads_from_existing_dir(tmp_path, monkeypatch):
+    """When the snapshot dir is set, the removed list is read from it, not fetched."""
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path))
+    removed = ["octocat/Hello-World", "hacs/integration"]
+    (tmp_path / "removed.json").write_text(json.dumps(removed))
+
+    hacs = _StubHacs()
+    assert await get_removed_repositories(hacs) == removed
+    assert hacs.data_client.calls == []
+
+
+async def test_get_stored_data_falls_back_to_fetch(monkeypatch):
+    """Without the snapshot dir, stored data is fetched from the data client."""
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", None)
+    hacs = _StubHacs()
+    assert await get_stored_data(hacs, "plugin") == _FETCHED_STORED
+    assert hacs.data_client.calls == [("get_data", "plugin")]
+
+
+async def test_get_removed_repositories_falls_back_to_fetch(monkeypatch):
+    """Without the snapshot dir, the removed list is fetched from the data client."""
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", None)
+    hacs = _StubHacs()
+    assert await get_removed_repositories(hacs) == _FETCHED_REMOVED
+    assert hacs.data_client.calls == [("get_repositories", "removed")]
+
+
+async def test_get_stored_data_falls_back_when_snapshot_missing(tmp_path, monkeypatch):
+    """A missing snapshot file falls back to fetching instead of raising."""
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path))
+    hacs = _StubHacs()
+    assert await get_stored_data(hacs, "plugin") == _FETCHED_STORED
+    assert hacs.data_client.calls == [("get_data", "plugin")]
+
+
+async def test_get_stored_data_falls_back_when_snapshot_invalid(tmp_path, monkeypatch):
+    """An invalid-JSON snapshot file falls back to fetching instead of raising."""
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path))
+    (tmp_path / "plugin.json").write_text("{ not valid json")
+
+    hacs = _StubHacs()
+    assert await get_stored_data(hacs, "plugin") == _FETCHED_STORED
+    assert hacs.data_client.calls == [("get_data", "plugin")]
+
+
+async def test_get_removed_repositories_falls_back_when_snapshot_missing(tmp_path, monkeypatch):
+    """A missing removed snapshot falls back to fetching instead of raising."""
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path))
+    hacs = _StubHacs()
+    assert await get_removed_repositories(hacs) == _FETCHED_REMOVED
+    assert hacs.data_client.calls == [("get_repositories", "removed")]
+
+
+async def test_get_stored_data_falls_back_on_wrong_shape(tmp_path, monkeypatch):
+    """Valid JSON of the wrong type (list, not dict) falls back to fetching."""
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path))
+    (tmp_path / "plugin.json").write_text(json.dumps(["not", "a", "dict"]))
+
+    hacs = _StubHacs()
+    assert await get_stored_data(hacs, "plugin") == _FETCHED_STORED
+    assert hacs.data_client.calls == [("get_data", "plugin")]
+
+
+async def test_get_removed_repositories_falls_back_on_wrong_shape(tmp_path, monkeypatch):
+    """Valid JSON of the wrong type (dict, not list) falls back to fetching."""
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path))
+    (tmp_path / "removed.json").write_text(json.dumps({"not": "a list"}))
+
+    hacs = _StubHacs()
+    assert await get_removed_repositories(hacs) == _FETCHED_REMOVED
+    assert hacs.data_client.calls == [("get_repositories", "removed")]
+
+
+@pytest.mark.parametrize(
+    "category_test_data", category_test_data_parametrized(categories=["integration"])
+)
+async def test_generate_category_data_from_stored_snapshot(
+    hass: HomeAssistant,
+    response_mocker: ResponseMocker,
+    snapshots: SnapshotFixture,
+    category_test_data: CategoryTestData,
+    tmp_path,
+    monkeypatch,
+):
+    """Generating from the stored snapshot reproduces the fetch-path output.
+
+    Feeds the same inputs the fetch path uses (empty category data + the removed
+    fixture) through STORED_DATA_DIR, and asserts the generated data.json matches
+    the fetch-path snapshot -- i.e. sourcing the baseline from disk is equivalent
+    to fetching it.
+    """
+    category = category_test_data["category"]
+
+    stored_dir = tmp_path / "stored"
+    stored_dir.mkdir()
+    (stored_dir / f"{category}.json").write_text("{}")
+    with open(
+        os.path.join(
+            FIXTURES_PATH, "proxy", "data-v2.hacs.xyz", "removed", "repositories.json"
+        ),
+        encoding="utf-8",
+    ) as file:
+        (stored_dir / "removed.json").write_text(file.read())
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(stored_dir))
+
+    await generate_category_data(category)
+
+    with open(f"{OUTPUT_DIR}/{category}/data.json", encoding="utf-8") as file:
+        snapshots.assert_match(
+            safe_json_dumps(recursive_remove_key(
+                json.loads(file.read()), ("last_fetched",))),
+            f"scripts/data/generate_category_data/{category}//data.json",
+        )