From 332b0d1b4a9095f4e43893ec741f901b74b36ceb Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 21 Apr 2022 14:16:49 -0400 Subject: [PATCH] 17301: Shorted OOM message. Limit added detail to 40 lines. Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- sdk/cwl/arvados_cwl/arvcontainer.py | 2 +- sdk/cwl/arvados_cwl/executor.py | 22 ++++++++++++++-------- sdk/cwl/tests/test_container.py | 2 ++ 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py index b04fb190e9..f75bde81e6 100644 --- a/sdk/cwl/arvados_cwl/arvcontainer.py +++ b/sdk/cwl/arvados_cwl/arvcontainer.py @@ -394,7 +394,7 @@ class ArvadosContainer(JobBase): processStatus = "permanentFail" if rcode == 137: - logger.warning("%s This container was killed on the compute instance. The most common reason is that it attempted to allocate too much RAM and was targeted by the Out Of Memory (OOM) killer. Try resubmitting with a higher 'ramMin'.", + logger.warning("%s Container may have been killed for using too much RAM. Try resubmitting with a higher 'ramMin'.", self.arvrunner.label(self)) else: processStatus = "permanentFail" diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py index 6cc971afa7..680ca0b7b2 100644 --- a/sdk/cwl/arvados_cwl/executor.py +++ b/sdk/cwl/arvados_cwl/executor.py @@ -261,19 +261,25 @@ The 'jobs' API is no longer supported. if current is None: return runtime_status = current.get('runtime_status', {}) - # In case of status being an error, only report the first one. - if kind in ('error', 'warning', 'activity'): + if kind in ('error', 'warning'): updatemessage = runtime_status.get(kind, "") if not updatemessage: - updatemessage = message; + updatemessage = message # Subsequent messages tacked on in detail updatedetail = runtime_status.get(kind+'Detail', "") - if updatedetail: - updatedetail += "\n" - updatedetail += message + "\n" - if detail: - updatedetail += detail + "\n" + maxlines = 40 + if updatedetail.count("\n") < maxlines: + if updatedetail: + updatedetail += "\n" + updatedetail += message + "\n" + + if detail: + updatedetail += detail + "\n" + + if updatedetail.count("\n") >= maxlines: + updatedetail += "\nSome messages may have been omitted. Check the full log." + runtime_status.update({ kind: updatemessage, kind+'Detail': updatedetail, diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py index 798c5af289..4d87d27e6e 100644 --- a/sdk/cwl/tests/test_container.py +++ b/sdk/cwl/tests/test_container.py @@ -1385,6 +1385,8 @@ class TestWorkflow(unittest.TestCase): runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999993+99"}]} + runner.api.containers().current().execute.return_value = {} + runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.num_retries = 0 -- 2.30.2