Acknowledge process task if corresponding node cannot be loaded (#2936)

Before this commit the `ProcessLauncher._continue` method was raising a
`plumpy.TaskRejected` exception if the node of a process task cannot be
loaded, either because it does not exist or it cannot be uniquely
resolved. However, this will cause RabbitMQ to requeue the task, causing
the task to be resent. This will result in the task ping-ponging between
RabbitMQ and the daemon workers without end. This situation typically
occurs when a user deletes a process node while it has not yet properly
terminated and therefore the tasks has not yet been acknowledged. This
situation is unrecoverable so the task should simply be acknowledged.

aiida/manage/external/rmq.py

@@ -136,7 +136,6 @@ def _continue(self, communicator, pid, nowait, tag=None):
         :param nowait: if True don't wait for the process to finish, just return the pid, otherwise wait and
             return the results
         :param tag: the tag of the checkpoint to continue from
-        :raises plumpy.TaskRejected: if the node corresponding to the task cannot be loaded
         """
         from aiida.common import exceptions
         from aiida.engine.exceptions import PastException
@@ -145,9 +144,14 @@ def _continue(self, communicator, pid, nowait, tag=None):
 
         try:
             node = load_node(pk=pid)
-        except (exceptions.MultipleObjectsError, exceptions.NotExistent) as exception:
+        except (exceptions.MultipleObjectsError, exceptions.NotExistent):
+            # In this case, the process node corresponding to the process id, cannot be resolved uniquely or does not
+            # exist. The latter being the most common case, where someone deleted the node, before the process was
+            # properly terminated. Since the node is never coming back and so the process will never be able to continue
+            # we raise `Return` instead of `TaskRejected` because the latter would cause the task to be resent and start
+            # to ping-pong between RabbitMQ and the daemon workers.
             LOGGER.exception('Cannot continue process<%d>', pid)
-            raise plumpy.TaskRejected('Cannot continue process: {}'.format(exception))
+            raise gen.Return(False)
 
         if node.is_terminated: