Fetch existing HACS data once before generation

.github/workflows/generate-hacs-data.yml

@@ -45,17 +45,60 @@ jobs:
       categories: ${{ steps.set-matrix.outputs.categories }}
     steps:
       - id: set-matrix
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          INPUT_CATEGORY: ${{ inputs.category }}
         run: |
-          if [[ "${{ github.event_name }}" == "workflow_dispatch"  ]] && [[ "${{ inputs.category }}" != "None"  ]] && [[ "${{ inputs.category }}" != ""  ]]; then
-            echo "categories=['${{ inputs.category }}']" >> $GITHUB_OUTPUT
+          if [[ "$EVENT_NAME" == "workflow_dispatch" ]] && [[ "$INPUT_CATEGORY" != "None" ]] && [[ -n "$INPUT_CATEGORY" ]]; then
+            categories=("$INPUT_CATEGORY")
           else
-            echo "categories=['appdaemon','integration','plugin','python_script','template','theme']" >> $GITHUB_OUTPUT
+            categories=(appdaemon integration plugin python_script template theme)
           fi
+          categories_json=$(printf '%s\n' "${categories[@]}" | jq -R . | jq -cs .)
+          echo "categories=$categories_json" >> $GITHUB_OUTPUT
+          echo "categories: $categories_json"
 
-  category-data:
+  fetch-stored-data:
+    name: Fetch stored data
     runs-on: ubuntu-latest
     needs: generate-matrix
     if: github.repository == 'hacs/integration'
+    steps:
+      - name: Fetch existing published data
+        env:
+          CATEGORIES: ${{ needs.generate-matrix.outputs.categories }}
+        run: |
+          mkdir -p outputdata/existing
+
+          fetch_stored_hacs_content() {
+            curl \
+              --silent --show-error --fail --retry 5 --retry-all-errors \
+              --connect-timeout 10 --max-time 60 \
+              --header "User-Agent: HACS/Generator" \
+              "https://data-v2.hacs.xyz/$1" \
+              --output "$2"
+          }
+
+          for category in $(echo "$CATEGORIES" | jq -r '.[]'); do
+            echo "Fetching existing data for $category"
+            fetch_stored_hacs_content "${category}/data.json" "outputdata/existing/${category}.json"
+          done
+
+          echo "Fetching removed repositories"
+          fetch_stored_hacs_content "removed/repositories.json" "outputdata/existing/removed.json"
+
+      - name: Upload existing data
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: existing-data
+          path: outputdata/existing
+          if-no-files-found: error
+          retention-days: 1
+
+  category-data:
+    runs-on: ubuntu-latest
+    needs: [generate-matrix, fetch-stored-data]
+    if: github.repository == 'hacs/integration'
     name: Generate ${{ matrix.category }} data
     strategy:
       fail-fast: false
@@ -83,11 +126,18 @@ jobs:
           scripts/install/frontend
           scripts/install/pip_packages --requirement requirements_generate_data.txt
 
+      - name: Download existing data
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: existing-data
+          path: outputdata/existing
+
       - name: Generate ${{ matrix.category }} data
         run: python3 -m scripts.data.generate_category_data ${{ matrix.category }}
         env:
           DATA_GENERATOR_TOKEN: ${{ secrets.DATA_GENERATOR_TOKEN }}
           FORCE_REPOSITORY_UPDATE: ${{ inputs.forceRepositoryUpdate }}
+          HACS_EXISTING_DATA_DIR: outputdata/existing
 
       - name: Validate output with JQ
         run: |
@@ -287,7 +337,7 @@ jobs:
   notify_on_failure:
     runs-on: ubuntu-latest
     name: Trigger Discord notification when jobs fail
-    needs: ["generate-matrix", "category-data", "summarize", "publish"]
+    needs: ["generate-matrix", "fetch-stored-data", "category-data", "summarize", "publish"]
     if: ${{ always() && github.repository == 'hacs/integration' && contains(join(needs.*.result, ','), 'failure') && github.event_name == 'schedule' }}
     steps:
       - name: Send notification

scripts/data/generate_category_data.py

@@ -58,6 +58,52 @@
 OUTPUT_DIR = os.path.join(os.getcwd(), "outputdata")
 COMPARE_IGNORE = {"etag_releases", "etag_repository", "last_fetched"}
 
+EXISTING_DATA_DIR = os.getenv("HACS_EXISTING_DATA_DIR")
+
+
+def _read_snapshot(
+    hacs: AdjustedHacs,
+    filename: str,
+    expected_type: type,
+) -> object | None:
+    """Read a snapshot file from ``EXISTING_DATA_DIR``.
+
+    Returns the parsed JSON when available, or ``None`` (so the caller fetches
+    instead) when the dir is unset, the file is missing or unreadable, or the
+    content is not of ``expected_type``.
+    """
+    if not EXISTING_DATA_DIR:
+        return None
+    try:
+        with open(os.path.join(EXISTING_DATA_DIR, filename), encoding="utf-8") as file:
+            data = json.load(file)
+    except (OSError, json.JSONDecodeError) as err:
+        hacs.log.warning(
+            "Could not read snapshot %s (%s), fetching instead", filename, err)
+        return None
+    if not isinstance(data, expected_type):
+        hacs.log.warning(
+            "Snapshot %s has unexpected type %s, fetching instead",
+            filename,
+            type(data).__name__,
+        )
+        return None
+    return data
+
+
+async def get_stored_data(hacs: AdjustedHacs, category: str) -> dict[str, dict[str, Any]]:
+    """Return existing category data from the snapshot dir when available, else fetch."""
+    if (data := _read_snapshot(hacs, f"{category}.json", dict)) is not None:
+        return data
+    return await hacs.data_client.get_data(category, validate=False)
+
+
+async def get_removed_repositories(hacs: AdjustedHacs) -> list[str]:
+    """Return the removed-repositories list from the snapshot dir when available, else fetch."""
+    if (removed := _read_snapshot(hacs, "removed.json", list)) is not None:
+        return removed
+    return await hacs.data_client.get_repositories("removed")
+
 
 def jsonprint(data: any):
     print(
@@ -309,7 +355,7 @@ async def generate_data_for_category(
         removed = (
             []
             if repository_name is not None
-            else await self.data_client.get_repositories("removed")
+            else await get_removed_repositories(self)
         )
         await self.data.register_base_data(
             category,
@@ -461,7 +507,7 @@ async def generate_category_data(category: str, repository_name: str = None):
         os.makedirs(os.path.join(OUTPUT_DIR, category), exist_ok=True)
         os.makedirs(os.path.join(OUTPUT_DIR, "diff"), exist_ok=True)
         force = os.environ.get("FORCE_REPOSITORY_UPDATE") == "True"
-        stored_data = await hacs.data_client.get_data(category, validate=False)
+        stored_data = await get_stored_data(hacs, category)
         current_data = (
             next(
                 (

tests/scripts/data/test_generate_category_data.py

@@ -2,13 +2,19 @@
 
 import asyncio
 import json
+import logging
 import os
 from typing import Any
 
 from homeassistant.core import HomeAssistant
 import pytest
 
-from scripts.data.generate_category_data import OUTPUT_DIR, generate_category_data
+from scripts.data.generate_category_data import (
+    OUTPUT_DIR,
+    generate_category_data,
+    get_removed_repositories,
+    get_stored_data,
+)
 
 from tests.common import (
     FIXTURES_PATH,
@@ -311,3 +317,118 @@ async def test_generate_category_data_with_30plus_prereleases(
         f"scripts/data/test_generate_category_data_with_30plus_prereleases/{
             category_test_data['category']}.json",
     )
+
+
+# Shapes mirror the real data client: get_data -> {repo-id: {...}}, removed -> [full_name].
+_FETCHED_STORED = {"1296269": {"full_name": "octocat/Hello-World", "category": "plugin"}}
+_FETCHED_REMOVED = ["octocat/removed-repo"]
+
+
+class _StubDataClient:
+    """Minimal data client recording that a fetch happened."""
+
+    def __init__(self) -> None:
+        self.calls: list[tuple[str, str]] = []
+
+    async def get_data(self, section: str, *, validate: bool) -> dict[str, dict[str, Any]]:
+        self.calls.append(("get_data", section))
+        return _FETCHED_STORED
+
+    async def get_repositories(self, section: str) -> list[str]:
+        self.calls.append(("get_repositories", section))
+        return _FETCHED_REMOVED
+
+
+class _StubHacs:
+    """Minimal HACS stand-in exposing only a data client and a logger."""
+
+    def __init__(self) -> None:
+        self.data_client = _StubDataClient()
+        self.log = logging.getLogger("test.generate_category_data")
+
+
+_MODULE = "scripts.data.generate_category_data"
+
+
+async def test_get_stored_data_reads_from_existing_dir(tmp_path, monkeypatch):
+    """When the snapshot dir is set, stored data is read from it, not fetched."""
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    payload = {"1": {"full_name": "octocat/Hello-World"}}
+    (tmp_path / "integration.json").write_text(json.dumps(payload))
+
+    hacs = _StubHacs()
+    assert await get_stored_data(hacs, "integration") == payload
+    assert hacs.data_client.calls == []
+
+
+async def test_get_removed_repositories_reads_from_existing_dir(tmp_path, monkeypatch):
+    """When the snapshot dir is set, the removed list is read from it, not fetched."""
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    removed = ["octocat/Hello-World", "hacs/integration"]
+    (tmp_path / "removed.json").write_text(json.dumps(removed))
+
+    hacs = _StubHacs()
+    assert await get_removed_repositories(hacs) == removed
+    assert hacs.data_client.calls == []
+
+
+async def test_get_stored_data_falls_back_to_fetch(monkeypatch):
+    """Without the snapshot dir, stored data is fetched from the data client."""
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", None)
+    hacs = _StubHacs()
+    assert await get_stored_data(hacs, "plugin") == _FETCHED_STORED
+    assert hacs.data_client.calls == [("get_data", "plugin")]
+
+
+async def test_get_removed_repositories_falls_back_to_fetch(monkeypatch):
+    """Without the snapshot dir, the removed list is fetched from the data client."""
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", None)
+    hacs = _StubHacs()
+    assert await get_removed_repositories(hacs) == _FETCHED_REMOVED
+    assert hacs.data_client.calls == [("get_repositories", "removed")]
+
+
+async def test_get_stored_data_falls_back_when_snapshot_missing(tmp_path, monkeypatch):
+    """A missing snapshot file falls back to fetching instead of raising."""
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    hacs = _StubHacs()
+    assert await get_stored_data(hacs, "plugin") == _FETCHED_STORED
+    assert hacs.data_client.calls == [("get_data", "plugin")]
+
+
+async def test_get_stored_data_falls_back_when_snapshot_invalid(tmp_path, monkeypatch):
+    """An invalid-JSON snapshot file falls back to fetching instead of raising."""
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    (tmp_path / "plugin.json").write_text("{ not valid json")
+
+    hacs = _StubHacs()
+    assert await get_stored_data(hacs, "plugin") == _FETCHED_STORED
+    assert hacs.data_client.calls == [("get_data", "plugin")]
+
+
+async def test_get_removed_repositories_falls_back_when_snapshot_missing(tmp_path, monkeypatch):
+    """A missing removed snapshot falls back to fetching instead of raising."""
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    hacs = _StubHacs()
+    assert await get_removed_repositories(hacs) == _FETCHED_REMOVED
+    assert hacs.data_client.calls == [("get_repositories", "removed")]
+
+
+async def test_get_stored_data_falls_back_on_wrong_shape(tmp_path, monkeypatch):
+    """Valid JSON of the wrong type (list, not dict) falls back to fetching."""
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    (tmp_path / "plugin.json").write_text(json.dumps(["not", "a", "dict"]))
+
+    hacs = _StubHacs()
+    assert await get_stored_data(hacs, "plugin") == _FETCHED_STORED
+    assert hacs.data_client.calls == [("get_data", "plugin")]
+
+
+async def test_get_removed_repositories_falls_back_on_wrong_shape(tmp_path, monkeypatch):
+    """Valid JSON of the wrong type (dict, not list) falls back to fetching."""
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    (tmp_path / "removed.json").write_text(json.dumps({"not": "a list"}))
+
+    hacs = _StubHacs()
+    assert await get_removed_repositories(hacs) == _FETCHED_REMOVED
+    assert hacs.data_client.calls == [("get_repositories", "removed")]