From f02bda90a27c5fb9de0d46cef187ffcb5f76a38b Mon Sep 17 00:00:00 2001 From: 20vikash Date: Fri, 29 May 2026 06:53:59 +0000 Subject: [PATCH 1/3] feat(job): Delete 1 month old jobs --- agent/job.py | 12 ++++++++++++ agent/server.py | 21 +++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/agent/job.py b/agent/job.py index 61e8c5f3..05293830 100644 --- a/agent/job.py +++ b/agent/job.py @@ -222,6 +222,18 @@ def wrapper(wrapped, instance: Base, args, kwargs): return wrapper +def cleanup(): + cutoff = datetime.datetime.now() - datetime.timedelta(days=30) # 1 month old data + + old_jobs = JobModel.select(JobModel.id).where(JobModel.enqueue < cutoff) + + deleted_steps = StepModel.delete().where(StepModel.job.in_(old_jobs)).execute() + + deleted_jobs = JobModel.delete().where(JobModel.id.in_(old_jobs)).execute() + + print(f"Deleted {deleted_jobs} jobs and {deleted_steps} steps") + + def get_agent_job_id(): from flask import request diff --git a/agent/server.py b/agent/server.py index c13c9996..cee524f3 100644 --- a/agent/server.py +++ b/agent/server.py @@ -829,6 +829,23 @@ def get_running_bench_containers(self) -> list[str]: """Get the actual containers skipping the `Server.benches` property""" return self.execute("docker ps --format '{{.Names}}'")["output"].split("\n") + def setup_cleanup_jobs(self): + import sys + + from crontab import CronTab + + cron = CronTab(user=True) + + command = f"cd {self.directory} && {sys.executable} -c 'from agent.job import cleanup; cleanup()'" + + cron.remove_all(command=command) + + job = cron.new(command=command) + job.hour.on(2) + job.minute.on(0) + + cron.write() + @property def job_record(self): if self.job is None: @@ -869,6 +886,8 @@ def update_agent_web(self, url=None, branch="master"): self.execute("sudo supervisorctl restart agent:web") run_patches() + self.setup_cleanup_jobs() + def update_agent_cli( # noqa: C901 self, restart_redis=True, @@ -938,6 +957,8 @@ def update_agent_cli( # noqa: C901 if not skip_patches: run_patches() + self.setup_cleanup_jobs() + @staticmethod def run_ncdu_command(path: str, excludes: list | None = None) -> str | None: cmd = ["ncdu", path, "-x", "-o", "/dev/stdout"] From 3ed5a70ce997b7891d9d3032d72e1dae25de076e Mon Sep 17 00:00:00 2001 From: 20vikash Date: Fri, 29 May 2026 07:53:21 +0000 Subject: [PATCH 2/3] feat(cli): Add cleanup jobs as a cli option --- agent/cleanup_job.py | 19 +++++++++++++++++++ agent/cli.py | 24 ++++++++++++++++++++++++ agent/job.py | 12 ------------ agent/server.py | 21 ++------------------- 4 files changed, 45 insertions(+), 31 deletions(-) create mode 100644 agent/cleanup_job.py diff --git a/agent/cleanup_job.py b/agent/cleanup_job.py new file mode 100644 index 00000000..fac47153 --- /dev/null +++ b/agent/cleanup_job.py @@ -0,0 +1,19 @@ +import datetime + +from agent.job import JobModel, StepModel + + +def cleanup(): + cutoff = datetime.datetime.now() - datetime.timedelta(days=30) # 1 month old data + + old_jobs = JobModel.select(JobModel.id).where(JobModel.enqueue < cutoff) + + deleted_steps = StepModel.delete().where(StepModel.job.in_(old_jobs)).execute() + + deleted_jobs = JobModel.delete().where(JobModel.id.in_(old_jobs)).execute() + + print(f"Deleted {deleted_jobs} jobs and {deleted_steps} steps") + + +if __name__ == "__main__": + cleanup() diff --git a/agent/cli.py b/agent/cli.py index 4eee3d3d..92cc35c2 100644 --- a/agent/cli.py +++ b/agent/cli.py @@ -256,6 +256,30 @@ def proxysql(password): Server().setup_proxysql(password) +@setup.command() +def agent_db_cleanup(): + from crontab import CronTab + + script_directory = os.path.dirname(__file__) + agent_directory = os.path.dirname(os.path.dirname(script_directory)) + + cron = CronTab(user=True) + + command = ( + f"cd {agent_directory} && " + f"{agent_directory}/env/bin/python " + f"{agent_directory}/repo/agent/cleanup_job.py" + ) + + cron.remove_all(command=command) + + job = cron.new(command=command) + job.hour.on(2) + job.minute.on(0) + + cron.write() + + @cli.group() def run(): pass diff --git a/agent/job.py b/agent/job.py index 05293830..61e8c5f3 100644 --- a/agent/job.py +++ b/agent/job.py @@ -222,18 +222,6 @@ def wrapper(wrapped, instance: Base, args, kwargs): return wrapper -def cleanup(): - cutoff = datetime.datetime.now() - datetime.timedelta(days=30) # 1 month old data - - old_jobs = JobModel.select(JobModel.id).where(JobModel.enqueue < cutoff) - - deleted_steps = StepModel.delete().where(StepModel.job.in_(old_jobs)).execute() - - deleted_jobs = JobModel.delete().where(JobModel.id.in_(old_jobs)).execute() - - print(f"Deleted {deleted_jobs} jobs and {deleted_steps} steps") - - def get_agent_job_id(): from flask import request diff --git a/agent/server.py b/agent/server.py index cee524f3..dca0ebd6 100644 --- a/agent/server.py +++ b/agent/server.py @@ -829,23 +829,6 @@ def get_running_bench_containers(self) -> list[str]: """Get the actual containers skipping the `Server.benches` property""" return self.execute("docker ps --format '{{.Names}}'")["output"].split("\n") - def setup_cleanup_jobs(self): - import sys - - from crontab import CronTab - - cron = CronTab(user=True) - - command = f"cd {self.directory} && {sys.executable} -c 'from agent.job import cleanup; cleanup()'" - - cron.remove_all(command=command) - - job = cron.new(command=command) - job.hour.on(2) - job.minute.on(0) - - cron.write() - @property def job_record(self): if self.job is None: @@ -886,7 +869,7 @@ def update_agent_web(self, url=None, branch="master"): self.execute("sudo supervisorctl restart agent:web") run_patches() - self.setup_cleanup_jobs() + self.execute(f"{self.directory}/env/bin/agent setup agent-db-cleanup") def update_agent_cli( # noqa: C901 self, @@ -957,7 +940,7 @@ def update_agent_cli( # noqa: C901 if not skip_patches: run_patches() - self.setup_cleanup_jobs() + self.execute(f"{self.directory}/env/bin/agent setup agent-db-cleanup") @staticmethod def run_ncdu_command(path: str, excludes: list | None = None) -> str | None: From 29466cf96c565469ca8f32bcc82a59e9fb524c98 Mon Sep 17 00:00:00 2001 From: 20vikash Date: Fri, 29 May 2026 07:57:51 +0000 Subject: [PATCH 3/3] chore(server): Don't call cleanup job cli --- agent/server.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/agent/server.py b/agent/server.py index dca0ebd6..c13c9996 100644 --- a/agent/server.py +++ b/agent/server.py @@ -869,8 +869,6 @@ def update_agent_web(self, url=None, branch="master"): self.execute("sudo supervisorctl restart agent:web") run_patches() - self.execute(f"{self.directory}/env/bin/agent setup agent-db-cleanup") - def update_agent_cli( # noqa: C901 self, restart_redis=True, @@ -940,8 +938,6 @@ def update_agent_cli( # noqa: C901 if not skip_patches: run_patches() - self.execute(f"{self.directory}/env/bin/agent setup agent-db-cleanup") - @staticmethod def run_ncdu_command(path: str, excludes: list | None = None) -> str | None: cmd = ["ncdu", path, "-x", "-o", "/dev/stdout"]