19744: Extend test_done() to check for a usage report

This isn't comprehensive but it at least checks the happy path detect
obvious problems in the interface between a-c-r and
crunchstat-summary.

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index bf1199a05414c6bc94e64f074d38412496eceae3..63e04a157e29d4edb39844495e1c24e326251eb8 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -28,6 +28,7 @@ from cwltool.job import JobBase
 import arvados.collection
 
 import crunchstat_summary.summarizer
+import crunchstat_summary.reader
 
 from .arvdocker import arv_docker_get_image
 from . import done
@@ -535,7 +536,8 @@ class ArvadosContainer(JobBase):
 
             if logc is not None:
                 try:
-                    summerizer = crunchstat_summary.summarizer.NewSummarizer(self.uuid, arv=self.arvrunner.api)
+                    summerizer = crunchstat_summary.summarizer.Summarizer(crunchstat_summary.reader.CollectionReader(logc.manifest_locator(), collection_object=logc),
+                                                                          label=self.name, arv=self.arvrunner.api)
                     summerizer.run()
                     with logc.open("usage_report.html", "wt") as mr:
                         mr.write(summerizer.html_report())

diff --git a/sdk/cwl/tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt b/sdk/cwl/tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt
new file mode 100644 (file)
index 0000000..e8e79cc
--- /dev/null
+++ b/sdk/cwl/tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt
@@ -0,0 +1,10 @@
+2018-10-03T18:21:16.944508412Z crunchstat: keepcalls 0 put 0 get -- interval 10.0000 seconds 0 put 0 get
+2018-10-03T18:21:16.944508412Z crunchstat: net:keep0 0 tx 0 rx -- interval 10.0000 seconds 0 tx 0 rx
+2018-10-03T18:21:16.944508412Z crunchstat: keepcache 0 hit 0 miss -- interval 10.0000 seconds 0 hit 0 miss
+2018-10-03T18:21:16.944508412Z crunchstat: fuseops 0 write 0 read -- interval 10.0000 seconds 0 write 0 read
+2018-10-03T18:21:16.944508412Z crunchstat: blkio:0:0 0 write 0 read -- interval 10.0000 seconds 0 write 0 read
+2018-10-03T18:21:26.954764471Z crunchstat: keepcalls 0 put 0 get -- interval 10.0000 seconds 0 put 0 get
+2018-10-03T18:21:26.954764471Z crunchstat: net:keep0 0 tx 0 rx -- interval 10.0000 seconds 0 tx 0 rx
+2018-10-03T18:21:26.954764471Z crunchstat: keepcache 0 hit 0 miss -- interval 10.0000 seconds 0 hit 0 miss
+2018-10-03T18:21:26.954764471Z crunchstat: fuseops 0 write 0 read -- interval 10.0000 seconds 0 write 0 read
+2018-10-03T18:21:26.954764471Z crunchstat: blkio:0:0 0 write 0 read -- interval 10.0000 seconds 0 write 0 read

diff --git a/sdk/cwl/tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt b/sdk/cwl/tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt
new file mode 100644 (file)
index 0000000..6580843
--- /dev/null
+++ b/sdk/cwl/tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt
@@ -0,0 +1,17 @@
+2018-10-03T18:21:07.823780191Z notice: reading stats from /sys/fs/cgroup/cpuacct//slurm_compute0/uid_0/job_6478/step_batch/c1df52c9940aae3f0fd586cacd7c0d7cb81b33aec973a67c9a7519bfe38ea914/cgroup.procs
+2018-10-03T18:21:07.823841282Z notice: monitoring temp dir /tmp/crunch-run.9tee4-dz642-lymtndkpy39eibk.438029160
+2018-10-03T18:21:07.823917514Z notice: reading stats from /sys/fs/cgroup/memory//slurm_compute0/uid_0/job_6478/step_batch/c1df52c9940aae3f0fd586cacd7c0d7cb81b33aec973a67c9a7519bfe38ea914/memory.stat
+2018-10-03T18:21:07.824136521Z mem 0 cache 0 swap 0 pgmajfault 1187840 rss
+2018-10-03T18:21:07.824187182Z notice: reading stats from /sys/fs/cgroup/cpuacct//slurm_compute0/uid_0/job_6478/step_batch/c1df52c9940aae3f0fd586cacd7c0d7cb81b33aec973a67c9a7519bfe38ea914/cpuacct.stat
+2018-10-03T18:21:07.824253726Z notice: reading stats from /sys/fs/cgroup/cpuset//slurm_compute0/uid_0/job_6478/step_batch/c1df52c9940aae3f0fd586cacd7c0d7cb81b33aec973a67c9a7519bfe38ea914/cpuset.cpus
+2018-10-03T18:21:07.824296720Z cpu 0.0000 user 0.0100 sys 20.00 cpus
+2018-10-03T18:21:07.824361476Z notice: reading stats from /sys/fs/cgroup/blkio//slurm_compute0/uid_0/job_6478/step_batch/c1df52c9940aae3f0fd586cacd7c0d7cb81b33aec973a67c9a7519bfe38ea914/blkio.io_service_bytes
+2018-10-03T18:21:07.824551021Z statfs 397741461504 available 4869779456 used 402611240960 total
+2018-10-03T18:21:17.824503045Z mem 172032 cache 0 swap 0 pgmajfault 68247552 rss
+2018-10-03T18:21:17.824702097Z cpu 2.0000 user 0.3800 sys 20.00 cpus -- interval 10.0004 seconds 2.0000 user 0.3700 sys
+2018-10-03T18:21:17.824984621Z net:eth0 51930 tx 844687 rx
+2018-10-03T18:21:17.825021992Z statfs 397740937216 available 4870303744 used 402611240960 total -- interval 10.0005 seconds 524288 used
+2018-10-03T18:21:27.824480114Z mem 172032 cache 0 swap 0 pgmajfault 69525504 rss
+2018-10-03T18:21:27.826909728Z cpu 2.0600 user 0.3900 sys 20.00 cpus -- interval 10.0022 seconds 0.0600 user 0.0100 sys
+2018-10-03T18:21:27.827141860Z net:eth0 55888 tx 859480 rx -- interval 10.0022 seconds 3958 tx 14793 rx
+2018-10-03T18:21:27.827177703Z statfs 397744787456 available 4866453504 used 402611240960 total -- interval 10.0022 seconds -3850240 used

diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 440fead8702cf9e434165074ec9108d6a6a9d4a5..b95b8eb67bbc4d83b357fcb209bccaf6ebf7dca4 100644 (file)
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -23,6 +23,7 @@ import cwltool.load_tool
 from cwltool.update import INTERNAL_VERSION
 from schema_salad.ref_resolver import Loader
 from schema_salad.sourceline import cmap
+import io
 
 from .matcher import JsonDiffMatcher, StripYAMLComments
 from .mock_discovery import get_rootDesc
@@ -518,11 +519,47 @@ class TestContainer(unittest.TestCase):
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
 
+        runner.api.container_requests().get().execute.return_value = {"container_uuid":"zzzzz-xvhdp-zzzzzzzzzzzzzzz"}
+
         runner.api.containers().get().execute.return_value = {"state":"Complete",
                                                               "output": "abc+123",
                                                               "exit_code": 0}
 
-        col().open.return_value = []
+        # Need to noop-out the close method otherwise it gets
+        # discarded when closed and we can't call getvalue() to check
+        # it.
+        class NoopCloseStringIO(io.StringIO):
+            def close(self):
+                pass
+
+        usage_report = NoopCloseStringIO()
+        def colreader_action(name, mode):
+            nonlocal usage_report
+            if name == "node.json":
+                return io.StringIO("""{
+    "ProviderType": "c5.large",
+    "VCPUs": 2,
+    "RAM": 4294967296,
+    "IncludedScratch": 8000000000000,
+    "AddedScratch": 0,
+    "Price": 0.085,
+    "Preemptible": false,
+    "CUDA": {
+        "DriverVersion": "",
+        "HardwareCapability": "",
+        "DeviceCount": 0
+    }
+}""")
+            if name == 'crunchstat.txt':
+                return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt", "rt")
+            if name == 'arv-mount.txt':
+                return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt", "rt")
+            if name == 'usage_report.html':
+                return usage_report
+            return None
+
+        col().open.side_effect = colreader_action
+        col().__iter__.return_value = ['node.json', 'crunchstat.txt', 'arv-mount.txt']
 
         loadingContext, runtimeContext = self.helper(runner)
 
@@ -550,12 +587,16 @@ class TestContainer(unittest.TestCase):
             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
             "modified_at": "2017-05-26T12:01:22Z",
-            "properties": {}
+            "properties": {},
+            "name": "testjob"
         })
 
         self.assertFalse(api.collections().create.called)
         self.assertFalse(runner.runtime_status_error.called)
 
+        # Assert that something was written to the usage report
+        self.assertTrue(len(usage_report.getvalue()) > 0)
+
         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")

diff --git a/tools/crunchstat-summary/crunchstat_summary/reader.py b/tools/crunchstat-summary/crunchstat_summary/reader.py
index 2af770f51bc7909ed5a2982fc860d990b8bd6158..e79c4ef5b36e7dfebc62ce22cbfe6a690b02aba0 100644 (file)
--- a/tools/crunchstat-summary/crunchstat_summary/reader.py
+++ b/tools/crunchstat-summary/crunchstat_summary/reader.py
@@ -12,12 +12,12 @@ from crunchstat_summary import logger
 
 
 class CollectionReader(object):
-    def __init__(self, collection_id, api_client=None):
+    def __init__(self, collection_id, api_client=None, collection_object=None):
         self._collection_id = collection_id
         self._label = collection_id
         self._readers = []
         self._api_client = api_client
-        self._collection = arvados.collection.CollectionReader(self._collection_id, api_client=self._api_client)
+        self._collection = collection_object or arvados.collection.CollectionReader(self._collection_id, api_client=self._api_client)
 
     def __str__(self):
         return self._label
@@ -31,7 +31,7 @@ class CollectionReader(object):
             filenames = ['crunchstat.txt', 'arv-mount.txt']
         for filename in filenames:
             try:
-                self._readers.append(self._collection.open(filename))
+                self._readers.append(self._collection.open(filename, "rt"))
             except IOError:
                 logger.warn('Unable to open %s', filename)
         self._label = "{}/{}".format(self._collection_id, filenames[0])
@@ -48,7 +48,7 @@ class CollectionReader(object):
 
     def node_info(self):
         try:
-            with self._collection.open("node.json") as f:
+            with self._collection.open("node.json", "rt") as f:
                 return json.load(f)
         except IOError:
             logger.warn('Unable to open node.json')