diff --git a/agent/cli.py b/agent/cli.py index 2383de7d..4eee3d3d 100644 --- a/agent/cli.py +++ b/agent/cli.py @@ -72,10 +72,11 @@ def ping_server(password: str): @click.option("--user", default="frappe") @click.option("--db-port", default=3306) @click.option("--workers", required=True, type=int) +@click.option("--job-timeout", required=False, type=int, default=4 * 3600) @click.option("--proxy-ip", required=False, type=str, default=None) @click.option("--sentry-dsn", required=False, type=str) @click.option("--press-url", required=False, type=str) -def config(name, user, workers, proxy_ip=None, sentry_dsn=None, press_url=None, db_port=3306): +def config(name, user, workers, job_timeout, proxy_ip=None, sentry_dsn=None, press_url=None, db_port=3306): config = { "benches_directory": f"/home/{user}/benches", "name": name, @@ -88,6 +89,7 @@ def config(name, user, workers, proxy_ip=None, sentry_dsn=None, press_url=None, "web_port": 25052, "press_url": "https://frappecloud.com", "db_port": db_port, + "job_timeout": job_timeout, } if press_url: config["press_url"] = press_url @@ -123,6 +125,12 @@ def sentry(sentry_dsn): Server().setup_sentry(sentry_dsn) +@setup.command() +@click.option("--job-timeout", required=True) +def job_timeout(job_timeout): + Server().setup_job_timeout(job_timeout) + + @setup.command() def supervisor(): Server().setup_supervisor() diff --git a/agent/job.py b/agent/job.py index fe1d39f5..61e8c5f3 100644 --- a/agent/job.py +++ b/agent/job.py @@ -36,7 +36,7 @@ except ImportError: pass - +DEFAULT_TIMEOUT = 4 * 3600 agent_database = SqliteDatabase( "jobs.sqlite3", timeout=15, @@ -176,10 +176,13 @@ def wrapper(wrapped, instance: Base, args, kwargs): return wrapper -def job(name: str, priority="default", on_success=None, on_failure=None): +def job(name: str, priority="default", timeout=None, on_success=None, on_failure=None): @wrapt.decorator def wrapper(wrapped, instance: Base, args, kwargs): + from flask import has_request_context, request + from agent.base import AgentException + from agent.server import Server if get_current_job(connection=connection()): instance.job_record.start() @@ -195,12 +198,20 @@ def wrapper(wrapped, instance: Base, args, kwargs): instance.job_record.success(result) return result agent_job_id = get_agent_job_id() + agent_job_timeout = None + if has_request_context() and request and request.is_json: + agent_job_timeout = request.json.get("agent_job_timeout", None) instance.job_record.enqueue(name, wrapped, args, kwargs, agent_job_id) + final_timeout = ( + agent_job_timeout or timeout or Server().config.get("job_timeout", None) or DEFAULT_TIMEOUT + ) + if not 0 <= final_timeout <= 24 * 3600: + final_timeout = DEFAULT_TIMEOUT queue(priority).enqueue_call( wrapped, args=args, kwargs=kwargs, - timeout=4 * 3600, + timeout=final_timeout, result_ttl=24 * 3600, job_id=str(instance.job_record.model.id), on_success=on_success or callback, diff --git a/agent/pages/deactivated.html b/agent/pages/deactivated.html index d65f4a3a..aacb9f3f 100644 --- a/agent/pages/deactivated.html +++ b/agent/pages/deactivated.html @@ -117,7 +117,7 @@

diff --git a/agent/pages/exceeded.html b/agent/pages/exceeded.html index 629dfe49..839364c5 100644 --- a/agent/pages/exceeded.html +++ b/agent/pages/exceeded.html @@ -118,7 +118,7 @@

diff --git a/agent/pages/suspended.html b/agent/pages/suspended.html index b9cd1e2a..b3eb8d92 100644 --- a/agent/pages/suspended.html +++ b/agent/pages/suspended.html @@ -116,7 +116,7 @@

diff --git a/agent/server.py b/agent/server.py index a60c856d..67ddf27e 100644 --- a/agent/server.py +++ b/agent/server.py @@ -755,6 +755,10 @@ def setup_sentry(self, sentry_dsn): self.update_config({"sentry_dsn": sentry_dsn}) self.setup_supervisor() + def setup_job_timeout(self, job_timeout): + self.update_config({"job_timeout": job_timeout}) + self.setup_supervisor() + def setup_nginx(self): self._generate_nginx_config() self._generate_agent_nginx_config() diff --git a/agent/site.py b/agent/site.py index a6654aa6..2fa5a6f2 100644 --- a/agent/site.py +++ b/agent/site.py @@ -786,7 +786,10 @@ def tables_to_restore(self): @job("Backup Site", priority="low") def backup_job( - self, with_files=False, offsite=None, keep_files_locally_after_offsite_backup: bool = False + self, + with_files=False, + offsite=None, + keep_files_locally_after_offsite_backup: bool = False, ): backup_files = self.backup(with_files) uploaded_files = ( diff --git a/agent/templates/agent/nginx.conf.jinja2 b/agent/templates/agent/nginx.conf.jinja2 index b8c9a91e..e2336a69 100644 --- a/agent/templates/agent/nginx.conf.jinja2 +++ b/agent/templates/agent/nginx.conf.jinja2 @@ -102,7 +102,7 @@ server { location /metrics/mariadb { proxy_pass http://127.0.0.1:9104/metrics; } - + location /metrics/mariadb_proxy { proxy_pass http://127.0.0.1:9104/metrics; } @@ -126,7 +126,7 @@ server { location /metrics/blackbox { proxy_pass http://127.0.0.1:9115/blackbox/metrics; } - + location /metrics/grafana { proxy_pass http://127.0.0.1:3000/grafana/metrics; } @@ -209,7 +209,7 @@ server { auth_basic_user_file /home/frappe/agent/nginx/grafana.htpasswd; proxy_pass http://127.0.0.1:9115/blackbox; } - + location /grafana { auth_basic "Grafana UI"; auth_basic_user_file /home/frappe/agent/nginx/grafana-ui.htpasswd; @@ -240,7 +240,7 @@ server { location /kibana/ { auth_basic "Kibana"; auth_basic_user_file /home/frappe/agent/nginx/kibana.htpasswd; - + proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; proxy_set_header host $host; diff --git a/agent/web.py b/agent/web.py index e6e46b1c..e9a358c1 100644 --- a/agent/web.py +++ b/agent/web.py @@ -56,6 +56,16 @@ class ExecuteReturn(TypedDict): application = Flask(__name__) +SENSITIVE_CONFIG_KEYS = { + "access_token", + "redis_port", + "redis_password", + "db_password", + "db_user", + "db_host", + "db_port", +} + def validate_bench(fn): @wraps(fn) @@ -317,6 +327,25 @@ def pull_docker_images(): return {"job": job} +@application.route("/server/get-config", methods=["GET"]) +def get_server_config(): + config = dict(Server().config or {}) + return {key: value for key, value in config.items() if key not in SENSITIVE_CONFIG_KEYS} + + +@application.route("/server/update-config", methods=["POST"]) +def update_server_config(): + config = request.json + if not isinstance(config, dict): + return jsonify({"error": "Invalid config payload; expected a JSON object."}), 400 + sanitized_config = {key: value for key, value in config.items() if key not in SENSITIVE_CONFIG_KEYS} + stripped_keys = set(config.keys()) - set(sanitized_config.keys()) + if stripped_keys: + log.warning("Stripping sensitive config in updating: %s", (",").join(sorted(stripped_keys))) + Server().update_config(sanitized_config) + return {"update_config": True} + + @application.route("/nfs/add-to-acl", methods=["POST"]) def add_to_acl(): data = request.json