ci: extract inline CI python scripts into .github/scripts

.github/scripts/check_no_ci_required.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""Decide whether a set of changed files requires running CI.
+
+Extracted from the inline ./subclass.py heredoc in the "no-ci-required"
+job of .github/workflows/ci-skip.yml. Reads the list of changed file
+paths from stdin (one per line) and prints either:
+
+    result=verified   all changed files match the filtered paths
+    result=skipped    at least one changed file is outside them
+
+Exits with status 1 if no changed files are provided.
+"""
+
+import os
+import re
+import shlex
+import sys
+
+paths = [
+    r'^\.github/dependabot\.yml$',
+    r'^\.github/workflows/audits\.yml$',
+    r'^\.github/workflows/book\.yml$',
+    r'^\.github/workflows/ci-skip\.yml$',
+    r'^\.github/workflows/lints\.yml$',
+    r'^\.github/workflows/release-docker-hub\.yml$',
+    r'^contrib/debian/copyright$',
+    r'^doc/.*',
+    r'.*\.md$',
+    r'^COPYING$',
+    r'^INSTALL$',
+]
+paths_regex = '(?:%s)' % '|'.join(paths)
+
+lex = shlex.shlex(posix = True)
+lex.whitespace = '\n\r'
+lex.whitespace_split = True
+lex.commenters = ''
+changed_files = list(lex)
+if len(changed_files) == 0:
+    sys.exit(1)
+
+verified = True
+for f in changed_files:
+    if not re.match(paths_regex, f):
+        verified = False
+
+print('result=verified' if verified else 'result=skipped')

.github/scripts/run_rpc_tests.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+"""Run a list of RPC tests using a custom test handler.
+
+Extracted from the inline ./subclass.py heredoc in the "RPC test" job of
+.github/workflows/ci.yml. It subclasses rpc-tests.RPCTestHandler so test
+subprocesses inherit the caller's environment unchanged.
+
+Run from the repository root. Configuration is read from the environment:
+
+    SRC_DIR    Source directory passed through to run_tests (required).
+    EXEEXT     Executable file extension, e.g. ".exe" on Windows.
+    RPC_TESTS  JSON-encoded list of test names to run (required).
+
+Exits with status 1 if any test fails.
+"""
+
+import importlib
+import json
+import os
+import subprocess
+import sys
+
+sys.path.append('qa/pull-tester')
+rpc_tests = importlib.import_module('rpc-tests')
+
+src_dir = os.environ["SRC_DIR"]
+build_dir = '.'
+exeext = os.environ["EXEEXT"]
+
+
+class MyTestHandler(rpc_tests.RPCTestHandler):
+    def start_test(self, args, stdout, stderr):
+        return subprocess.Popen(
+            args,
+            universal_newlines=True,
+            stdout=stdout,
+            stderr=stderr)
+
+
+test_list = json.loads(os.environ["RPC_TESTS"])
+all_passed = rpc_tests.run_tests(MyTestHandler, test_list, src_dir, build_dir, exeext, jobs=len(test_list))
+if all_passed == False:
+    sys.exit(1)

.github/scripts/set_rpc_test_shards.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Assign RPC tests to shards and emit the shard matrix as JSON.
+
+Extracted from the inline ./subclass.py heredoc in the "set-rpc-tests" job
+of .github/workflows/ci.yml. Prints a JSON list of {shard, rpc_tests}
+objects on stdout.
+
+Run from the repository root. Configuration is read from the environment:
+
+    SRC_DIR             Source directory (unused here, kept for parity).
+    IS_INTEROP_REQUEST  "true" to skip the not-yet-passing test set.
+"""
+
+import importlib
+import json
+import os
+import sys
+
+sys.path.append('qa/pull-tester')
+rpc_tests = importlib.import_module('rpc-tests')
+
+src_dir = os.environ["SRC_DIR"]
+is_interop_request = os.environ["IS_INTEROP_REQUEST"] == "true"
+SHARDS = 10
+
+# While we are getting the existing tests to pass, skip tests that are
+# not expected to pass; this makes it easier to distinguish expected
+# failures from unexpected ones.
+if is_interop_request:
+    old_shards = 0
+else:
+    num_old_tests = len(rpc_tests.BASE_SCRIPTS + rpc_tests.ZMQ_SCRIPTS)
+    num_new_tests = len(rpc_tests.NEW_SCRIPTS)
+    old_shards = (num_old_tests * SHARDS) // (num_old_tests + num_new_tests)
+
+new_shards = SHARDS - old_shards
+
+# These tests are ordered longest-test-first, to favor running tests in
+# parallel with the regular test runner. For chunking purposes, assign
+# tests to shards in round-robin order.
+test_shards = {}
+if old_shards > 0:
+    for i, test in enumerate(rpc_tests.BASE_SCRIPTS + rpc_tests.ZMQ_SCRIPTS):
+        test_shards.setdefault(i % old_shards, []).append(test)
+for i, test in enumerate(rpc_tests.NEW_SCRIPTS):
+    test_shards.setdefault(old_shards + (i % new_shards), []).append(test)
+
+test_list = []
+for i, tests in test_shards.items():
+    test_list.append({
+        'shard': 'shard-%d' % i,
+        'rpc_tests': tests,
+    })
+
+# These tests involve enough shielded spends (consuming all CPU cores)
+# that we can't run them in parallel, or fail intermittently so we run
+# them separately to enable not requiring that they pass.
+if not is_interop_request:
+    for test in rpc_tests.SERIAL_SCRIPTS + rpc_tests.FLAKY_SCRIPTS:
+        test_list.append({
+            'shard': test,
+            'rpc_tests': [test],
+        })
+
+print(json.dumps(test_list))

.github/workflows/ci-skip.yml

@@ -49,44 +49,7 @@ jobs:
       - name: Check whether the changes are only to the set of filtered paths
         id: no-ci-required
         run: |
-          cat <<EOF > ./subclass.py
-          import os
-          import re
-          import shlex
-          import sys
-
-          paths = [
-              r'^\.github/dependabot\.yml$',
-              r'^\.github/workflows/audits\.yml$',
-              r'^\.github/workflows/book\.yml$',
-              r'^\.github/workflows/ci-skip\.yml$',
-              r'^\.github/workflows/lints\.yml$',
-              r'^\.github/workflows/release-docker-hub\.yml$',
-              r'^contrib/debian/copyright$',
-              r'^doc/.*',
-              r'.*\.md$',
-              r'^COPYING$',
-              r'^INSTALL$',
-          ]
-          paths_regex = '(?:%s)' % '|'.join(paths)
-
-          lex = shlex.shlex(posix = True)
-          lex.whitespace = '\n\r'
-          lex.whitespace_split = True
-          lex.commenters = ''
-          changed_files = list(lex)
-          if len(changed_files) == 0:
-              sys.exit(1)
-
-          verified = True
-          for f in changed_files:
-              if not re.match(paths_regex, f):
-                  verified = False
-
-          print('result=verified' if verified else 'result=skipped')
-          EOF
-
-          git diff --name-only ${{ github.sha }}...$HEAD_SHA -- | python3 ./subclass.py >> $GITHUB_OUTPUT
+          git diff --name-only ${{ github.sha }}...$HEAD_SHA -- | python3 .github/scripts/check_no_ci_required.py >> $GITHUB_OUTPUT
         env:
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
 

.github/workflows/ci.yml

@@ -211,62 +211,8 @@ jobs:
         env:
           IS_INTEROP_REQUEST: ${{ endsWith(github.event.action, '-interop-request') }}
         run: |
-          cat <<EOF > ./subclass.py
-          import importlib
-          import json
-          import os
-          import sys
-
-          sys.path.append('qa/pull-tester')
-          rpc_tests = importlib.import_module('rpc-tests')
-
-          src_dir = os.environ["SRC_DIR"]
-          is_interop_request = os.environ["IS_INTEROP_REQUEST"] == "true"
-          SHARDS = 10
-
-          # While we are getting the existing tests to pass, skip tests that are
-          # not expected to pass; this makes it easier to distinguish expected
-          # failures from unexpected ones.
-          if is_interop_request:
-              old_shards = 0
-          else:
-              num_old_tests = len(rpc_tests.BASE_SCRIPTS + rpc_tests.ZMQ_SCRIPTS)
-              num_new_tests = len(rpc_tests.NEW_SCRIPTS)
-              old_shards = (num_old_tests * SHARDS) // (num_old_tests + num_new_tests)
-
-          new_shards = SHARDS - old_shards
-
-          # These tests are ordered longest-test-first, to favor running tests in
-          # parallel with the regular test runner. For chunking purposes, assign
-          # tests to shards in round-robin order.
-          test_shards = {}
-          if old_shards > 0:
-              for i, test in enumerate(rpc_tests.BASE_SCRIPTS + rpc_tests.ZMQ_SCRIPTS):
-                  test_shards.setdefault(i % old_shards, []).append(test)
-          for i, test in enumerate(rpc_tests.NEW_SCRIPTS):
-              test_shards.setdefault(old_shards + (i % new_shards), []).append(test)
-
-          test_list = []
-          for i, tests in test_shards.items():
-              test_list.append({
-                  'shard': 'shard-%d' % i,
-                  'rpc_tests': tests,
-              })
-
-          # These tests involve enough shielded spends (consuming all CPU cores)
-          # that we can't run them in parallel, or fail intermittently so we run
-          # them separately to enable not requiring that they pass.
-          if not is_interop_request:
-              for test in rpc_tests.SERIAL_SCRIPTS + rpc_tests.FLAKY_SCRIPTS:
-                  test_list.append({
-                      'shard': test,
-                      'rpc_tests': [test],
-                  })
-
-          print(json.dumps(test_list))
-          EOF
           RPC_MATRIX_JSON=$(echo '${{ steps.set-matrices.outputs.rpc_test_matrix }}')
-          RPC_SHARDS_JSON=$(SRC_DIR=$(pwd) python3 ./subclass.py)
+          RPC_SHARDS_JSON=$(SRC_DIR=$(pwd) python3 .github/scripts/set_rpc_test_shards.py)
           echo "$RPC_SHARDS_JSON" | jq -r '[.[] | .shard] | @json "rpc_test_shards=\(.)"' >> $GITHUB_OUTPUT
           echo -e "$RPC_MATRIX_JSON\n$RPC_SHARDS_JSON" | jq -r -s 'add | @json "rpc_test_shards_matrix=\(.)"' >> $GITHUB_OUTPUT
 
@@ -850,35 +796,12 @@ jobs:
           fi
 
       - name: RPC test ${{ matrix.shard }}
+        env:
+          EXEEXT: ${{ matrix.file_ext }}
+          RPC_TESTS: ${{ toJSON(matrix.rpc_tests) }}
         run: |
-          cat <<EOF > ./subclass.py
-          import importlib
-          import os
-          import subprocess
-          import sys
-
-          sys.path.append('qa/pull-tester')
-          rpc_tests = importlib.import_module('rpc-tests')
-
-          src_dir = os.environ["SRC_DIR"]
-          build_dir = '.'
-          exeext = '${{ matrix.file_ext }}'
-
-          class MyTestHandler(rpc_tests.RPCTestHandler):
-              def start_test(self, args, stdout, stderr):
-                  return subprocess.Popen(
-                      args,
-                      universal_newlines=True,
-                      stdout=stdout,
-                      stderr=stderr)
-
-          test_list = ${{ toJSON(matrix.rpc_tests) }}
-          all_passed = rpc_tests.run_tests(MyTestHandler, test_list, src_dir, build_dir, exeext, jobs=len(test_list))
-          if all_passed == False:
-              sys.exit(1)
-          EOF
           . ./venv/bin/activate
-          ZEBRAD=$(pwd)/${{ format('src/zebrad{0}', matrix.file_ext) }} ZAINOD=$(pwd)/${{ format('src/zainod{0}', matrix.file_ext) }} ZALLET=$(pwd)/${{ format('src/zallet{0}', matrix.file_ext) }} SRC_DIR=$(pwd) python3 ./subclass.py
+          ZEBRAD=$(pwd)/${{ format('src/zebrad{0}', matrix.file_ext) }} ZAINOD=$(pwd)/${{ format('src/zainod{0}', matrix.file_ext) }} ZALLET=$(pwd)/${{ format('src/zallet{0}', matrix.file_ext) }} SRC_DIR=$(pwd) python3 .github/scripts/run_rpc_tests.py
 
       - uses: ./.github/actions/finish-interop
         if: always()