Fetch existing HACS data once before generation

.github/workflows/generate-hacs-data.yml

@@ -45,17 +45,59 @@ jobs:
       categories: ${{ steps.set-matrix.outputs.categories }}
     steps:
       - id: set-matrix
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          INPUT_CATEGORY: ${{ inputs.category }}
         run: |
-          if [[ "${{ github.event_name }}" == "workflow_dispatch"  ]] && [[ "${{ inputs.category }}" != "None"  ]] && [[ "${{ inputs.category }}" != ""  ]]; then
-            echo "categories=['${{ inputs.category }}']" >> $GITHUB_OUTPUT
+          if [[ "$EVENT_NAME" == "workflow_dispatch" ]] && [[ "$INPUT_CATEGORY" != "None" ]] && [[ -n "$INPUT_CATEGORY" ]]; then
+            categories=("$INPUT_CATEGORY")
           else
-            echo "categories=['appdaemon','integration','plugin','python_script','template','theme']" >> $GITHUB_OUTPUT
+            categories=(appdaemon integration plugin python_script template theme)
           fi
+          categories_json=$(printf '%s\n' "${categories[@]}" | jq -R . | jq -cs .)
+          echo "categories=$categories_json" >> $GITHUB_OUTPUT
+          echo "categories: $categories_json"
 
-  category-data:
+  preflight:
+    name: Fetch existing data
     runs-on: ubuntu-latest
     needs: generate-matrix
     if: github.repository == 'hacs/integration'
+    steps:
+      - name: Fetch existing published data
+        env:
+          CATEGORIES: ${{ needs.generate-matrix.outputs.categories }}
+        run: |
+          mkdir -p outputdata/existing
+
+          fetch() {
+            curl \
+              --silent --show-error --fail --retry 5 --retry-all-errors \
+              --header "User-Agent: HACS/Generator" \
+              "https://data-v2.hacs.xyz/$1" \
+              --output "$2"
+          }
+
+          for category in $(echo "$CATEGORIES" | jq -r '.[]'); do
+            echo "Fetching existing data for $category"
+            fetch "${category}/data.json" "outputdata/existing/${category}.json"
+          done
+
+          echo "Fetching removed repositories"
+          fetch "removed/repositories.json" "outputdata/existing/removed.json"
+
+      - name: Upload existing data
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: existing-data
+          path: outputdata/existing
+          if-no-files-found: error
+          retention-days: 1
+
+  category-data:
+    runs-on: ubuntu-latest
+    needs: [generate-matrix, preflight]
+    if: github.repository == 'hacs/integration'
     name: Generate ${{ matrix.category }} data
     strategy:
       fail-fast: false
@@ -83,11 +125,18 @@ jobs:
           scripts/install/frontend
           scripts/install/pip_packages --requirement requirements_generate_data.txt
 
+      - name: Download existing data
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: existing-data
+          path: outputdata/existing
+
       - name: Generate ${{ matrix.category }} data
         run: python3 -m scripts.data.generate_category_data ${{ matrix.category }}
         env:
           DATA_GENERATOR_TOKEN: ${{ secrets.DATA_GENERATOR_TOKEN }}
           FORCE_REPOSITORY_UPDATE: ${{ inputs.forceRepositoryUpdate }}
+          HACS_EXISTING_DATA_DIR: outputdata/existing
 
       - name: Validate output with JQ
         run: |
@@ -287,7 +336,7 @@ jobs:
   notify_on_failure:
     runs-on: ubuntu-latest
     name: Trigger Discord notification when jobs fail
-    needs: ["generate-matrix", "category-data", "summarize", "publish"]
+    needs: ["generate-matrix", "preflight", "category-data", "summarize", "publish"]
     if: ${{ always() && github.repository == 'hacs/integration' && contains(join(needs.*.result, ','), 'failure') && github.event_name == 'schedule' }}
     steps:
       - name: Send notification

scripts/data/generate_category_data.py

@@ -58,6 +58,28 @@
 OUTPUT_DIR = os.path.join(os.getcwd(), "outputdata")
 COMPARE_IGNORE = {"etag_releases", "etag_repository", "last_fetched"}
 
+EXISTING_DATA_DIR = os.getenv("HACS_EXISTING_DATA_DIR")
+
+
+async def get_stored_data(hacs: AdjustedHacs, category: str) -> dict[str, dict[str, Any]]:
+    """Return existing category data from the snapshot dir when set, else fetch."""
+    if EXISTING_DATA_DIR:
+        with open(
+            os.path.join(EXISTING_DATA_DIR, f"{category}.json"), encoding="utf-8"
+        ) as file:
+            return json.load(file)
+    return await hacs.data_client.get_data(category, validate=False)
+
+
+async def get_removed_repositories(hacs: AdjustedHacs) -> list[str]:
+    """Return the removed-repositories list from the snapshot dir when set, else fetch."""
+    if EXISTING_DATA_DIR:
+        with open(
+            os.path.join(EXISTING_DATA_DIR, "removed.json"), encoding="utf-8"
+        ) as file:
+            return json.load(file)
+    return await hacs.data_client.get_repositories("removed")
+
 
 def jsonprint(data: any):
     print(
@@ -309,7 +331,7 @@ async def generate_data_for_category(
         removed = (
             []
             if repository_name is not None
-            else await self.data_client.get_repositories("removed")
+            else await get_removed_repositories(self)
         )
         await self.data.register_base_data(
             category,
@@ -461,7 +483,7 @@ async def generate_category_data(category: str, repository_name: str = None):
         os.makedirs(os.path.join(OUTPUT_DIR, category), exist_ok=True)
         os.makedirs(os.path.join(OUTPUT_DIR, "diff"), exist_ok=True)
         force = os.environ.get("FORCE_REPOSITORY_UPDATE") == "True"
-        stored_data = await hacs.data_client.get_data(category, validate=False)
+        stored_data = await get_stored_data(hacs, category)
         current_data = (
             next(
                 (

tests/scripts/data/test_generate_category_data.py

@@ -8,7 +8,12 @@
 from homeassistant.core import HomeAssistant
 import pytest
 
-from scripts.data.generate_category_data import OUTPUT_DIR, generate_category_data
+from scripts.data.generate_category_data import (
+    OUTPUT_DIR,
+    generate_category_data,
+    get_removed_repositories,
+    get_stored_data,
+)
 
 from tests.common import (
     FIXTURES_PATH,
@@ -311,3 +316,66 @@ async def test_generate_category_data_with_30plus_prereleases(
         f"scripts/data/test_generate_category_data_with_30plus_prereleases/{
             category_test_data['category']}.json",
     )
+
+
+class _StubDataClient:
+    """Minimal data client recording that a fetch happened."""
+
+    def __init__(self) -> None:
+        self.calls: list[tuple[str, str]] = []
+
+    async def get_data(self, section: str, *, validate: bool) -> dict[str, Any]:
+        self.calls.append(("get_data", section))
+        return {"fetched": section}
+
+    async def get_repositories(self, section: str) -> list[str]:
+        self.calls.append(("get_repositories", section))
+        return [f"fetched/{section}"]
+
+
+class _StubHacs:
+    """Minimal HACS stand-in exposing only a data client."""
+
+    def __init__(self) -> None:
+        self.data_client = _StubDataClient()
+
+
+_MODULE = "scripts.data.generate_category_data"
+
+
+async def test_get_stored_data_reads_from_existing_dir(tmp_path, monkeypatch):
+    """When the snapshot dir is set, stored data is read from it, not fetched."""
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    payload = {"1": {"full_name": "octocat/Hello-World"}}
+    (tmp_path / "integration.json").write_text(json.dumps(payload))
+
+    hacs = _StubHacs()
+    assert await get_stored_data(hacs, "integration") == payload
+    assert hacs.data_client.calls == []
+
+
+async def test_get_removed_repositories_reads_from_existing_dir(tmp_path, monkeypatch):
+    """When the snapshot dir is set, the removed list is read from it, not fetched."""
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    removed = ["octocat/Hello-World", "hacs/integration"]
+    (tmp_path / "removed.json").write_text(json.dumps(removed))
+
+    hacs = _StubHacs()
+    assert await get_removed_repositories(hacs) == removed
+    assert hacs.data_client.calls == []
+
+
+async def test_get_stored_data_falls_back_to_fetch(monkeypatch):
+    """Without the snapshot dir, stored data is fetched from the data client."""
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", None)
+    hacs = _StubHacs()
+    assert await get_stored_data(hacs, "plugin") == {"fetched": "plugin"}
+    assert hacs.data_client.calls == [("get_data", "plugin")]
+
+
+async def test_get_removed_repositories_falls_back_to_fetch(monkeypatch):
+    """Without the snapshot dir, the removed list is fetched from the data client."""
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", None)
+    hacs = _StubHacs()
+    assert await get_removed_repositories(hacs) == ["fetched/removed"]
+    assert hacs.data_client.calls == [("get_repositories", "removed")]