From ae6d7a89c398ebfa45a28cddf98ec90d2570d2c7 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Fri, 19 Feb 2021 23:05:25 +0100 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=91=8C=20IMPROVE:=20Garbage=20collect?= =?UTF-8?q?=20on=20process=20termination?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- aiida/manage/external/rmq.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/aiida/manage/external/rmq.py b/aiida/manage/external/rmq.py index 8bfaf100f0..c1f98d721f 100644 --- a/aiida/manage/external/rmq.py +++ b/aiida/manage/external/rmq.py @@ -9,7 +9,9 @@ ########################################################################### # pylint: disable=cyclic-import """Components to communicate tasks to RabbitMQ.""" +import asyncio from collections.abc import Mapping +import gc import logging from kiwipy import communications, Future @@ -220,4 +222,10 @@ async def _continue(self, communicator, pid, nowait, tag=None): LOGGER.exception('failed to serialize the result for process<%d>', pid) raise + # before returning we ensure the garbage collection runs, to clear any memory used by the task + await asyncio.sleep(0) # allow other async tasks to complete first + del node + del result + gc.collect() + return serialized From 1f702b0a09a1ea4f0a00396f62a67114091de0e3 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Fri, 19 Feb 2021 23:18:34 +0100 Subject: [PATCH 2/3] fix pre-commit --- aiida/manage/external/rmq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiida/manage/external/rmq.py b/aiida/manage/external/rmq.py index c1f98d721f..8365594678 100644 --- a/aiida/manage/external/rmq.py +++ b/aiida/manage/external/rmq.py @@ -226,6 +226,6 @@ async def _continue(self, communicator, pid, nowait, tag=None): await asyncio.sleep(0) # allow other async tasks to complete first del node del result - gc.collect() + gc.collect() return serialized From 0e4a7f612a5c06e987505bd9e8c86e32e73c06b3 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Sat, 20 Feb 2021 00:05:28 +0100 Subject: [PATCH 3/3] increase async sleep to 1 --- aiida/manage/external/rmq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiida/manage/external/rmq.py b/aiida/manage/external/rmq.py index 8365594678..d7eabb5d6d 100644 --- a/aiida/manage/external/rmq.py +++ b/aiida/manage/external/rmq.py @@ -223,7 +223,7 @@ async def _continue(self, communicator, pid, nowait, tag=None): raise # before returning we ensure the garbage collection runs, to clear any memory used by the task - await asyncio.sleep(0) # allow other async tasks to complete first + await asyncio.sleep(1) # allow other async tasks to complete first del node del result gc.collect()