fix(job): Make job timeout configurable so that job can pass as param (backport #384)

agent/cli.py

@@ -72,10 +72,11 @@ def ping_server(password: str):
 @click.option("--user", default="frappe")
 @click.option("--db-port", default=3306)
 @click.option("--workers", required=True, type=int)
+@click.option("--job-timeout", required=False, type=int, default=4 * 3600)
 @click.option("--proxy-ip", required=False, type=str, default=None)
 @click.option("--sentry-dsn", required=False, type=str)
 @click.option("--press-url", required=False, type=str)
-def config(name, user, workers, proxy_ip=None, sentry_dsn=None, press_url=None, db_port=3306):
+def config(name, user, workers, job_timeout, proxy_ip=None, sentry_dsn=None, press_url=None, db_port=3306):
     config = {
         "benches_directory": f"/home/{user}/benches",
         "name": name,
@@ -88,6 +89,7 @@ def config(name, user, workers, proxy_ip=None, sentry_dsn=None, press_url=None,
         "web_port": 25052,
         "press_url": "https://frappecloud.com",
         "db_port": db_port,
+        "job_timeout": job_timeout,
     }
     if press_url:
         config["press_url"] = press_url
@@ -123,6 +125,12 @@ def sentry(sentry_dsn):
     Server().setup_sentry(sentry_dsn)
 
 
+@setup.command()
+@click.option("--job-timeout", required=True)
+def job_timeout(job_timeout):
+    Server().setup_job_timeout(job_timeout)
+
+
 @setup.command()
 def supervisor():
     Server().setup_supervisor()

agent/job.py

@@ -36,7 +36,7 @@
     except ImportError:
         pass
 
-
+DEFAULT_TIMEOUT = 4 * 3600
 agent_database = SqliteDatabase(
     "jobs.sqlite3",
     timeout=15,
@@ -176,10 +176,13 @@ def wrapper(wrapped, instance: Base, args, kwargs):
     return wrapper
 
 
-def job(name: str, priority="default", on_success=None, on_failure=None):
+def job(name: str, priority="default", timeout=None, on_success=None, on_failure=None):
     @wrapt.decorator
     def wrapper(wrapped, instance: Base, args, kwargs):
+        from flask import has_request_context, request
+
         from agent.base import AgentException
+        from agent.server import Server
 
         if get_current_job(connection=connection()):
             instance.job_record.start()
@@ -195,12 +198,20 @@ def wrapper(wrapped, instance: Base, args, kwargs):
                 instance.job_record.success(result)
             return result
         agent_job_id = get_agent_job_id()
+        agent_job_timeout = None
+        if has_request_context() and request and request.is_json:
+            agent_job_timeout = request.json.get("agent_job_timeout", None)
         instance.job_record.enqueue(name, wrapped, args, kwargs, agent_job_id)
+        final_timeout = (
+            agent_job_timeout or timeout or Server().config.get("job_timeout", None) or DEFAULT_TIMEOUT
+        )
+        if not 0 <= final_timeout <= 24 * 3600:
+            final_timeout = DEFAULT_TIMEOUT
         queue(priority).enqueue_call(
             wrapped,
             args=args,
             kwargs=kwargs,
-            timeout=4 * 3600,
+            timeout=final_timeout,
             result_ttl=24 * 3600,
             job_id=str(instance.job_record.model.id),
             on_success=on_success or callback,

agent/pages/deactivated.html

@@ -117,7 +117,7 @@ <h1>
 
     <script>
         const dashboardUrl = document.querySelector('.dashboard-url');
-        const siteName = window.location.hostname;        
+        const siteName = window.location.hostname;
         dashboardUrl.href = `https://frappecloud.com/dashboard/sites/${siteName}`;
     </script>
 </html>

agent/pages/exceeded.html

@@ -118,7 +118,7 @@ <h1>
 
 <script>
     const dashboardUrl = document.querySelector('.dashboard-url');
-    const siteName = window.location.hostname;        
+    const siteName = window.location.hostname;
     dashboardUrl.href = `https://frappecloud.com/dashboard/sites/${siteName}`;
 </script>
 

agent/pages/suspended.html

@@ -116,7 +116,7 @@ <h1>
 
     <script>
         const dashboardUrl = document.querySelector('.dashboard-url');
-        const siteName = window.location.hostname;        
+        const siteName = window.location.hostname;
         dashboardUrl.href = `https://frappecloud.com/dashboard/sites/${siteName}`;
     </script>
 </html>

agent/server.py

@@ -755,6 +755,10 @@ def setup_sentry(self, sentry_dsn):
         self.update_config({"sentry_dsn": sentry_dsn})
         self.setup_supervisor()
 
+    def setup_job_timeout(self, job_timeout):
+        self.update_config({"job_timeout": job_timeout})
+        self.setup_supervisor()
+
     def setup_nginx(self):
         self._generate_nginx_config()
         self._generate_agent_nginx_config()

agent/site.py

@@ -786,7 +786,10 @@ def tables_to_restore(self):
 
     @job("Backup Site", priority="low")
     def backup_job(
-        self, with_files=False, offsite=None, keep_files_locally_after_offsite_backup: bool = False
+        self,
+        with_files=False,
+        offsite=None,
+        keep_files_locally_after_offsite_backup: bool = False,
     ):
         backup_files = self.backup(with_files)
         uploaded_files = (

agent/templates/agent/nginx.conf.jinja2

@@ -102,7 +102,7 @@ server {
 		location /metrics/mariadb {
 			proxy_pass http://127.0.0.1:9104/metrics;
 		}
-	
+
 		location /metrics/mariadb_proxy {
 			proxy_pass http://127.0.0.1:9104/metrics;
 		}
@@ -126,7 +126,7 @@ server {
 		location /metrics/blackbox {
 			proxy_pass http://127.0.0.1:9115/blackbox/metrics;
 		}
-	
+
 		location /metrics/grafana {
 			proxy_pass http://127.0.0.1:3000/grafana/metrics;
 		}
@@ -209,7 +209,7 @@ server {
 		auth_basic_user_file /home/frappe/agent/nginx/grafana.htpasswd;
 		proxy_pass http://127.0.0.1:9115/blackbox;
 	}
-	
+
 	location /grafana {
 		auth_basic "Grafana UI";
 		auth_basic_user_file /home/frappe/agent/nginx/grafana-ui.htpasswd;
@@ -240,7 +240,7 @@ server {
 	location /kibana/ {
 		auth_basic "Kibana";
 		auth_basic_user_file /home/frappe/agent/nginx/kibana.htpasswd;
-		
+
 		proxy_set_header Upgrade $http_upgrade;
 		proxy_set_header Connection "upgrade";
 		proxy_set_header host $host;

agent/web.py

@@ -56,6 +56,16 @@ class ExecuteReturn(TypedDict):
 
 application = Flask(__name__)
 
+SENSITIVE_CONFIG_KEYS = {
+    "access_token",
+    "redis_port",
+    "redis_password",
+    "db_password",
+    "db_user",
+    "db_host",
+    "db_port",
+}
+
 
 def validate_bench(fn):
     @wraps(fn)
@@ -317,6 +327,25 @@ def pull_docker_images():
     return {"job": job}
 
 
+@application.route("/server/get-config", methods=["GET"])
+def get_server_config():
+    config = dict(Server().config or {})
+    return {key: value for key, value in config.items() if key not in SENSITIVE_CONFIG_KEYS}
+
+
+@application.route("/server/update-config", methods=["POST"])
+def update_server_config():
+    config = request.json
+    if not isinstance(config, dict):
+        return jsonify({"error": "Invalid config payload; expected a JSON object."}), 400
+    sanitized_config = {key: value for key, value in config.items() if key not in SENSITIVE_CONFIG_KEYS}
+    stripped_keys = set(config.keys()) - set(sanitized_config.keys())
+    if stripped_keys:
+        log.warning("Stripping sensitive config in updating: %s", (",").join(sorted(stripped_keys)))
+    Server().update_config(sanitized_config)
+    return {"update_config": True}
+
+
 @application.route("/nfs/add-to-acl", methods=["POST"])
 def add_to_acl():
     data = request.json