Add sharding support to HACS data generation pipeline

.github/workflows/generate-hacs-data.yml

@@ -42,26 +42,50 @@ jobs:
     runs-on: ubuntu-latest
     if: github.repository == 'hacs/integration'
     outputs:
+      shards: ${{ steps.set-matrix.outputs.shards }}
       categories: ${{ steps.set-matrix.outputs.categories }}
     steps:
       - id: set-matrix
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          INPUT_CATEGORY: ${{ inputs.category }}
         run: |
-          if [[ "${{ github.event_name }}" == "workflow_dispatch"  ]] && [[ "${{ inputs.category }}" != "None"  ]] && [[ "${{ inputs.category }}" != ""  ]]; then
-            echo "categories=['${{ inputs.category }}']" >> $GITHUB_OUTPUT
+          # Number of shards each category is split into (default 1).
+          declare -A shard_counts=( [integration]=3 [plugin]=2 )
+
+          if [[ "$EVENT_NAME" == "workflow_dispatch" ]] && [[ "$INPUT_CATEGORY" != "None" ]] && [[ -n "$INPUT_CATEGORY" ]]; then
+            categories=("$INPUT_CATEGORY")
           else
-            echo "categories=['appdaemon','integration','plugin','python_script','template','theme']" >> $GITHUB_OUTPUT
+            categories=(appdaemon integration plugin python_script template theme)
           fi
 
+          shards_json="[]"
+          for category in "${categories[@]}"; do
+            count=${shard_counts[$category]:-1}
+            for ((shard=1; shard<=count; shard++)); do
+              shards_json=$(jq -c \
+                --arg c "$category" --argjson s "$shard" --argjson t "$count" \
+                '. + [{category: $c, shard: $s, shards: $t}]' <<< "$shards_json")
+            done
+          done
+
+          categories_json=$(printf '%s\n' "${categories[@]}" | jq -R . | jq -cs .)
+
+          echo "shards=$shards_json" >> $GITHUB_OUTPUT
+          echo "categories=$categories_json" >> $GITHUB_OUTPUT
+          echo "shards: $shards_json"
+          echo "categories: $categories_json"
+
   category-data:
     runs-on: ubuntu-latest
     needs: generate-matrix
     if: github.repository == 'hacs/integration'
-    name: Generate ${{ matrix.category }} data
+    name: Generate ${{ matrix.category }} data (shard ${{ matrix.shard }}/${{ matrix.shards }})
     strategy:
       fail-fast: false
       max-parallel: 1
       matrix:
-        category: ${{ fromJSON( needs.generate-matrix.outputs.categories )}}
+        include: ${{ fromJSON( needs.generate-matrix.outputs.shards )}}
     steps:
       - name: Checkout the repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
@@ -83,12 +107,64 @@ jobs:
           scripts/install/frontend
           scripts/install/pip_packages --requirement requirements_generate_data.txt
 
-      - name: Generate ${{ matrix.category }} data
-        run: python3 -m scripts.data.generate_category_data ${{ matrix.category }}
+      - name: Generate ${{ matrix.category }} data (shard ${{ matrix.shard }}/${{ matrix.shards }})
+        run: python3 -m scripts.data.generate_category_data ${{ matrix.category }} --shard ${{ matrix.shard }}/${{ matrix.shards }}
         env:
           DATA_GENERATOR_TOKEN: ${{ secrets.DATA_GENERATOR_TOKEN }}
           FORCE_REPOSITORY_UPDATE: ${{ inputs.forceRepositoryUpdate }}
 
+      - name: Upload shard artifacts
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: shard-${{ matrix.category }}-${{ matrix.shard }}
+          path: outputdata/_shards/${{ matrix.category }}/${{ matrix.shard }}
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    runs-on: ubuntu-latest
+    needs: [generate-matrix, category-data]
+    if: github.repository == 'hacs/integration'
+    name: Merge ${{ matrix.category }} data
+    strategy:
+      fail-fast: false
+      matrix:
+        category: ${{ fromJSON( needs.generate-matrix.outputs.categories )}}
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        id: python
+        with:
+          python-version: "3.14"
+          cache: 'pip'
+          cache-dependency-path: |
+            requirements_base.txt
+            requirements_generate_data.txt
+
+      - name: Install dependencies
+        run:  |
+          scripts/install/frontend
+          scripts/install/pip_packages --requirement requirements_generate_data.txt
+
+      # Each shard artifact contains data.json + stored.json at its root. Without
+      # merge-multiple, download-artifact nests each artifact in its own
+      # name-scoped subdirectory, so the per-shard files never collide.
+      - name: Download shard artifacts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: shard-${{ matrix.category }}-*
+          path: outputdata/_shards/${{ matrix.category }}
+
+      - name: Merge ${{ matrix.category }} shards
+        run: python3 -m scripts.data.merge_category_data ${{ matrix.category }}
+        env:
+          DATA_GENERATOR_TOKEN: ${{ secrets.DATA_GENERATOR_TOKEN }}
+
       - name: Validate output with JQ
         run: |
           jq -c . outputdata/${{ matrix.category }}/data.json
@@ -129,7 +205,7 @@ jobs:
   summarize:
     name: Summarize
     runs-on: ubuntu-latest
-    needs: category-data
+    needs: merge
     if: ${{ always() && github.repository == 'hacs/integration' }}
     outputs:
       changedCategories: ${{ steps.combined.outputs.changedCategories }}
@@ -159,6 +235,7 @@ jobs:
             const subDirectories = fs.readdirSync("outputdata", { withFileTypes: true })
                 .filter(entry => entry.isDirectory())
                 .map(entry => entry.name)
+                .filter(directory => fs.existsSync(`outputdata/${directory}/summary.json`))
 
             for (const directory of subDirectories) {
               let changedPctTarget = Number(process.env.HACS_CHANGED_PCT_TARGET)
@@ -287,7 +364,7 @@ jobs:
   notify_on_failure:
     runs-on: ubuntu-latest
     name: Trigger Discord notification when jobs fail
-    needs: ["generate-matrix", "category-data", "summarize", "publish"]
+    needs: ["generate-matrix", "category-data", "merge", "summarize", "publish"]
     if: ${{ always() && github.repository == 'hacs/integration' && contains(join(needs.*.result, ','), 'failure') && github.event_name == 'schedule' }}
     steps:
       - name: Send notification