fix bench restore output handling

agent/site.py

@@ -16,7 +16,7 @@
 from agent.base import AgentException, Base
 from agent.database import Database
 from agent.job import job, step
-from agent.utils import b2mb, compute_file_hash, db_client_cli, db_dump_cli, get_size
+from agent.utils import b2mb, compute_file_hash, db_client_cli, db_dump_cli, get_size, parse_json_output
 
 if TYPE_CHECKING:
     from agent.bench import Bench
@@ -125,7 +125,7 @@ def restore_site(
         )
         try:
             return self.bench_execute(
-                "--force restore "
+                "--force restore --verbose "
                 f"--mariadb-root-username {temp_user} "
                 f"--mariadb-root-password {temp_password} "
                 f"--admin-password {admin_password} "
@@ -645,7 +645,9 @@ def clear_website_cache(self):
 
     @step("Uninstall Unavailable Apps")
     def uninstall_unavailable_apps(self, apps_to_keep):
-        installed_apps = json.loads(self.bench_execute("execute frappe.get_installed_apps")["output"])
+        installed_apps = parse_json_output(
+            self.bench_execute("execute frappe.get_installed_apps")["output"]
+        )
         for app in installed_apps:
             if app not in apps_to_keep:
                 self.bench_execute(f"remove-from-installed-apps '{app}'")
@@ -928,7 +930,7 @@ def get_usage(self):
 
     def get_analytics(self):
         analytics = self.bench_execute("execute frappe.utils.get_site_info")["output"]
-        return json.loads(analytics)
+        return parse_json_output(analytics)
 
     def get_database_size(self):
         config = {}
@@ -974,7 +976,7 @@ def describe_database_table(self, doctype, columns=None):
             command += f"--column {column} "
         try:
             output = self.bench_execute(command)["output"]
-            return json.loads(output)
+            return parse_json_output(output)
         except Exception:
             return {}
 
@@ -984,7 +986,7 @@ def apps(self):
 
     @property
     def apps_as_json(self):
-        return json.loads(self.bench_execute("list-apps -f json")["output"])[self.name]
+        return parse_json_output(self.bench_execute("list-apps -f json")["output"])[self.name]
 
     @job("Add Database Index")
     def add_database_index(self, doctype, columns=None):

agent/tests/test_site.py

@@ -11,6 +11,7 @@
 from agent.bench import Bench, _get_cors_origins, _normalize_cors_origins
 from agent.server import Server
 from agent.site import Site
+from agent.utils import parse_json_output
 
 
 class TestSite(unittest.TestCase):
@@ -74,6 +75,11 @@ def _get_test_bench(self) -> Bench:
         server.benches_directory = self.benches_directory
         return Bench(self.bench_name, server)
 
+    def _get_test_site(self, site_name: str = "test.local") -> Site:
+        self._create_test_site(site_name)
+        self._make_site_config(site_name)
+        return Site(site_name, self._get_test_bench())
+
     def _render_bench_nginx(self, cors_origins, standalone=False):
         output_file = os.path.join(self.test_dir, "nginx.conf")
         with patch.object(Server, "__init__", new=lambda x: None):
@@ -283,6 +289,69 @@ def test_normalize_cors_origins_strips_path_from_url(self):
             ["https://example.com"],
         )
 
+    def test_parse_json_output_extracts_json_from_noisy_logs(self):
+        output = 'Duckwalk override: custom app loaded\n  ["frappe", "erpnext", "insights"]'
+
+        self.assertEqual(parse_json_output(output), ["frappe", "erpnext", "insights"])
+
+    def test_restore_site_uses_verbose_flag(self):
+        site = self._get_test_site("restore.local")
+
+        with (
+            patch.object(site.bench, "create_mariadb_user", return_value=("db", "temp-user", "temp-pass")),
+            patch.object(site.bench, "drop_mariadb_user"),
+            patch.object(Site, "bench_execute", return_value={"output": ""}) as bench_execute,
+            patch.object(Site, "restore_site", new=Site.restore_site.__wrapped__),
+        ):
+            site.restore_site(
+                "root-pass",
+                "admin-pass",
+                os.path.join(site.bench.sites_directory, "restore.local", "private", "backups", "db.sql.gz"),
+                "",
+                "",
+            )
+
+        self.assertIn("--force restore --verbose", bench_execute.call_args.args[0])
+
+    def test_uninstall_unavailable_apps_parses_noisy_installed_apps_output(self):
+        site = self._get_test_site("apps.local")
+
+        with (
+            patch.object(
+                Site,
+                "bench_execute",
+                side_effect=[
+                    {
+                        "output": (
+                            "Duckwalk override: custom app loaded\n"
+                            '["frappe", "erpnext", "insights", "greendigit"]'
+                        )
+                    },
+                    {"output": ""},
+                    {"output": ""},
+                    {"output": ""},
+                    {"output": ""},
+                ],
+            ) as bench_execute,
+            patch.object(
+                Site,
+                "uninstall_unavailable_apps",
+                new=Site.uninstall_unavailable_apps.__wrapped__,
+            ),
+        ):
+            site.uninstall_unavailable_apps(["frappe", "erpnext"])
+
+        self.assertEqual(
+            [call.args[0] for call in bench_execute.call_args_list],
+            [
+                "execute frappe.get_installed_apps",
+                "remove-from-installed-apps 'insights'",
+                "clear-cache",
+                "remove-from-installed-apps 'greendigit'",
+                "clear-cache",
+            ],
+        )
+
     def test_get_cors_origins_with_quoted_allow_cors(self):
         sites = [
             SimpleNamespace(

agent/utils.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import hashlib
+import json
 import os
 import re
 import shutil
@@ -164,6 +165,32 @@ def end_execution(
     return res
 
 
+def parse_json_output(output: str):
+    """
+    Parse JSON from command output that may contain extra logs around the payload.
+    """
+    try:
+        return json.loads(output)
+    except json.JSONDecodeError:
+        decoder = json.JSONDecoder()
+        candidate = None
+
+        for match in re.finditer(r"[\[{]", output):
+            try:
+                value, end = decoder.raw_decode(output[match.start() :])
+            except json.JSONDecodeError:
+                continue
+
+            candidate = value
+            if not output[match.start() + end :].strip():
+                return value
+
+        if candidate is not None:
+            return candidate
+
+        raise
+
+
 def compute_file_hash(file_path, algorithm="sha256", raise_exception=True):
     try:
         """Compute the hash of a file using the specified algorithm."""