From dc5e352fbba183d4eab1196b77d5f190e692eda3 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 22 Feb 2024 16:00:25 -0500 Subject: [PATCH] 19744: Extend test_done() to check for a usage report This isn't comprehensive but it at least checks the happy path detect obvious problems in the interface between a-c-r and crunchstat-summary. Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- sdk/cwl/arvados_cwl/arvcontainer.py | 4 +- ..._9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt | 10 +++++ ...9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt | 17 +++++++ sdk/cwl/tests/test_container.py | 45 ++++++++++++++++++- .../crunchstat_summary/reader.py | 8 ++-- 5 files changed, 77 insertions(+), 7 deletions(-) create mode 100644 sdk/cwl/tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt create mode 100644 sdk/cwl/tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py index bf1199a054..63e04a157e 100644 --- a/sdk/cwl/arvados_cwl/arvcontainer.py +++ b/sdk/cwl/arvados_cwl/arvcontainer.py @@ -28,6 +28,7 @@ from cwltool.job import JobBase import arvados.collection import crunchstat_summary.summarizer +import crunchstat_summary.reader from .arvdocker import arv_docker_get_image from . import done @@ -535,7 +536,8 @@ class ArvadosContainer(JobBase): if logc is not None: try: - summerizer = crunchstat_summary.summarizer.NewSummarizer(self.uuid, arv=self.arvrunner.api) + summerizer = crunchstat_summary.summarizer.Summarizer(crunchstat_summary.reader.CollectionReader(logc.manifest_locator(), collection_object=logc), + label=self.name, arv=self.arvrunner.api) summerizer.run() with logc.open("usage_report.html", "wt") as mr: mr.write(summerizer.html_report()) diff --git a/sdk/cwl/tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt b/sdk/cwl/tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt new file mode 100644 index 0000000000..e8e79cc59a --- /dev/null +++ b/sdk/cwl/tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt @@ -0,0 +1,10 @@ +2018-10-03T18:21:16.944508412Z crunchstat: keepcalls 0 put 0 get -- interval 10.0000 seconds 0 put 0 get +2018-10-03T18:21:16.944508412Z crunchstat: net:keep0 0 tx 0 rx -- interval 10.0000 seconds 0 tx 0 rx +2018-10-03T18:21:16.944508412Z crunchstat: keepcache 0 hit 0 miss -- interval 10.0000 seconds 0 hit 0 miss +2018-10-03T18:21:16.944508412Z crunchstat: fuseops 0 write 0 read -- interval 10.0000 seconds 0 write 0 read +2018-10-03T18:21:16.944508412Z crunchstat: blkio:0:0 0 write 0 read -- interval 10.0000 seconds 0 write 0 read +2018-10-03T18:21:26.954764471Z crunchstat: keepcalls 0 put 0 get -- interval 10.0000 seconds 0 put 0 get +2018-10-03T18:21:26.954764471Z crunchstat: net:keep0 0 tx 0 rx -- interval 10.0000 seconds 0 tx 0 rx +2018-10-03T18:21:26.954764471Z crunchstat: keepcache 0 hit 0 miss -- interval 10.0000 seconds 0 hit 0 miss +2018-10-03T18:21:26.954764471Z crunchstat: fuseops 0 write 0 read -- interval 10.0000 seconds 0 write 0 read +2018-10-03T18:21:26.954764471Z crunchstat: blkio:0:0 0 write 0 read -- interval 10.0000 seconds 0 write 0 read diff --git a/sdk/cwl/tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt b/sdk/cwl/tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt new file mode 100644 index 0000000000..6580843b16 --- /dev/null +++ b/sdk/cwl/tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt @@ -0,0 +1,17 @@ +2018-10-03T18:21:07.823780191Z notice: reading stats from /sys/fs/cgroup/cpuacct//slurm_compute0/uid_0/job_6478/step_batch/c1df52c9940aae3f0fd586cacd7c0d7cb81b33aec973a67c9a7519bfe38ea914/cgroup.procs +2018-10-03T18:21:07.823841282Z notice: monitoring temp dir /tmp/crunch-run.9tee4-dz642-lymtndkpy39eibk.438029160 +2018-10-03T18:21:07.823917514Z notice: reading stats from /sys/fs/cgroup/memory//slurm_compute0/uid_0/job_6478/step_batch/c1df52c9940aae3f0fd586cacd7c0d7cb81b33aec973a67c9a7519bfe38ea914/memory.stat +2018-10-03T18:21:07.824136521Z mem 0 cache 0 swap 0 pgmajfault 1187840 rss +2018-10-03T18:21:07.824187182Z notice: reading stats from /sys/fs/cgroup/cpuacct//slurm_compute0/uid_0/job_6478/step_batch/c1df52c9940aae3f0fd586cacd7c0d7cb81b33aec973a67c9a7519bfe38ea914/cpuacct.stat +2018-10-03T18:21:07.824253726Z notice: reading stats from /sys/fs/cgroup/cpuset//slurm_compute0/uid_0/job_6478/step_batch/c1df52c9940aae3f0fd586cacd7c0d7cb81b33aec973a67c9a7519bfe38ea914/cpuset.cpus +2018-10-03T18:21:07.824296720Z cpu 0.0000 user 0.0100 sys 20.00 cpus +2018-10-03T18:21:07.824361476Z notice: reading stats from /sys/fs/cgroup/blkio//slurm_compute0/uid_0/job_6478/step_batch/c1df52c9940aae3f0fd586cacd7c0d7cb81b33aec973a67c9a7519bfe38ea914/blkio.io_service_bytes +2018-10-03T18:21:07.824551021Z statfs 397741461504 available 4869779456 used 402611240960 total +2018-10-03T18:21:17.824503045Z mem 172032 cache 0 swap 0 pgmajfault 68247552 rss +2018-10-03T18:21:17.824702097Z cpu 2.0000 user 0.3800 sys 20.00 cpus -- interval 10.0004 seconds 2.0000 user 0.3700 sys +2018-10-03T18:21:17.824984621Z net:eth0 51930 tx 844687 rx +2018-10-03T18:21:17.825021992Z statfs 397740937216 available 4870303744 used 402611240960 total -- interval 10.0005 seconds 524288 used +2018-10-03T18:21:27.824480114Z mem 172032 cache 0 swap 0 pgmajfault 69525504 rss +2018-10-03T18:21:27.826909728Z cpu 2.0600 user 0.3900 sys 20.00 cpus -- interval 10.0022 seconds 0.0600 user 0.0100 sys +2018-10-03T18:21:27.827141860Z net:eth0 55888 tx 859480 rx -- interval 10.0022 seconds 3958 tx 14793 rx +2018-10-03T18:21:27.827177703Z statfs 397744787456 available 4866453504 used 402611240960 total -- interval 10.0022 seconds -3850240 used diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py index 440fead870..b95b8eb67b 100644 --- a/sdk/cwl/tests/test_container.py +++ b/sdk/cwl/tests/test_container.py @@ -23,6 +23,7 @@ import cwltool.load_tool from cwltool.update import INTERNAL_VERSION from schema_salad.ref_resolver import Loader from schema_salad.sourceline import cmap +import io from .matcher import JsonDiffMatcher, StripYAMLComments from .mock_discovery import get_rootDesc @@ -518,11 +519,47 @@ class TestContainer(unittest.TestCase): runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() + runner.api.container_requests().get().execute.return_value = {"container_uuid":"zzzzz-xvhdp-zzzzzzzzzzzzzzz"} + runner.api.containers().get().execute.return_value = {"state":"Complete", "output": "abc+123", "exit_code": 0} - col().open.return_value = [] + # Need to noop-out the close method otherwise it gets + # discarded when closed and we can't call getvalue() to check + # it. + class NoopCloseStringIO(io.StringIO): + def close(self): + pass + + usage_report = NoopCloseStringIO() + def colreader_action(name, mode): + nonlocal usage_report + if name == "node.json": + return io.StringIO("""{ + "ProviderType": "c5.large", + "VCPUs": 2, + "RAM": 4294967296, + "IncludedScratch": 8000000000000, + "AddedScratch": 0, + "Price": 0.085, + "Preemptible": false, + "CUDA": { + "DriverVersion": "", + "HardwareCapability": "", + "DeviceCount": 0 + } +}""") + if name == 'crunchstat.txt': + return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt", "rt") + if name == 'arv-mount.txt': + return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt", "rt") + if name == 'usage_report.html': + return usage_report + return None + + col().open.side_effect = colreader_action + col().__iter__.return_value = ['node.json', 'crunchstat.txt', 'arv-mount.txt'] loadingContext, runtimeContext = self.helper(runner) @@ -550,12 +587,16 @@ class TestContainer(unittest.TestCase): "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz", "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", "modified_at": "2017-05-26T12:01:22Z", - "properties": {} + "properties": {}, + "name": "testjob" }) self.assertFalse(api.collections().create.called) self.assertFalse(runner.runtime_status_error.called) + # Assert that something was written to the usage report + self.assertTrue(len(usage_report.getvalue()) > 0) + arvjob.collect_outputs.assert_called_with("keep:abc+123", 0) arvjob.output_callback.assert_called_with({"out": "stuff"}, "success") runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2") diff --git a/tools/crunchstat-summary/crunchstat_summary/reader.py b/tools/crunchstat-summary/crunchstat_summary/reader.py index 2af770f51b..e79c4ef5b3 100644 --- a/tools/crunchstat-summary/crunchstat_summary/reader.py +++ b/tools/crunchstat-summary/crunchstat_summary/reader.py @@ -12,12 +12,12 @@ from crunchstat_summary import logger class CollectionReader(object): - def __init__(self, collection_id, api_client=None): + def __init__(self, collection_id, api_client=None, collection_object=None): self._collection_id = collection_id self._label = collection_id self._readers = [] self._api_client = api_client - self._collection = arvados.collection.CollectionReader(self._collection_id, api_client=self._api_client) + self._collection = collection_object or arvados.collection.CollectionReader(self._collection_id, api_client=self._api_client) def __str__(self): return self._label @@ -31,7 +31,7 @@ class CollectionReader(object): filenames = ['crunchstat.txt', 'arv-mount.txt'] for filename in filenames: try: - self._readers.append(self._collection.open(filename)) + self._readers.append(self._collection.open(filename, "rt")) except IOError: logger.warn('Unable to open %s', filename) self._label = "{}/{}".format(self._collection_id, filenames[0]) @@ -48,7 +48,7 @@ class CollectionReader(object): def node_info(self): try: - with self._collection.open("node.json") as f: + with self._collection.open("node.json", "rt") as f: return json.load(f) except IOError: logger.warn('Unable to open node.json') -- 2.30.2