Add test for output-properties

Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic@capeannenterprises.com>

diff --git a/sdk/cwl/tests/test_make_output.py b/sdk/cwl/tests/test_make_output.py
index 590c82d207d590784c677a5831721ce577c99554..91b1194facc5dda325b8e0df928c9f8bc43028ae 100644 (file)
--- a/sdk/cwl/tests/test_make_output.py
+++ b/sdk/cwl/tests/test_make_output.py
@@ -39,7 +39,7 @@ class TestMakeOutput(unittest.TestCase):
         final.open.return_value = openmock
         openmock.__enter__.return_value = cwlout
 
-        _, runner.final_output_collection = runner.make_output_collection("Test output", ["foo"], "tag0,tag1,tag2", {
+        _, runner.final_output_collection = runner.make_output_collection("Test output", ["foo"], {'foo':'bar', 'baz':'quz'}, "tag0,tag1,tag2", {
             "foo": {
                 "class": "File",
                 "location": "keep:99999999999999999999999999999991+99/foo.txt",
@@ -56,7 +56,7 @@ class TestMakeOutput(unittest.TestCase):
 
         final.copy.assert_has_calls([mock.call('bar.txt', 'baz.txt', overwrite=False, source_collection=readermock)])
         final.copy.assert_has_calls([mock.call('foo.txt', 'foo.txt', overwrite=False, source_collection=readermock)])
-        final.save_new.assert_has_calls([mock.call(ensure_unique_name=True, name='Test output', owner_uuid='zzzzz-j7d0g-zzzzzzzzzzzzzzz', storage_classes=['foo'])])
+        final.save_new.assert_has_calls([mock.call(ensure_unique_name=True, name='Test output', owner_uuid='zzzzz-j7d0g-zzzzzzzzzzzzzzz', storage_classes=['foo'], properties={'foo':'bar', 'baz':'quz'})])
         self.assertEqual("""{
     "bar": {
         "basename": "baz.txt",

diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index cd46251300dfb95862cb7957f510e108dd78b281..c3303eea397b1a7765acac6701789221fff59332 100644 (file)
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -11,6 +11,7 @@ import logging
 import mock
 import sys
 import unittest
+import tempfile
 
 import arvados
 import arvados.collection
@@ -341,6 +342,32 @@ class TestSubmit(unittest.TestCase):
                 sys.stdin, sys.stderr, api_client=stubs.api)
         self.assertEqual(exited, 1)
 
+    def test_split_output_properties(self):
+        output_props = "foo:bar,baz:qux"
+        result = arvados_cwl.split_output_properties(output_props)
+
+        self.assertEqual(result, {'foo':'bar', 'baz':'qux'})
+
+    def test_parse_output_properties_yaml(self):
+        tmp = tempfile.NamedTemporaryFile()
+        with open(tmp.name, 'a') as f:
+            f.write("foo: 111\n") 
+            f.write("bar: 222\n") 
+            f.write("baz: 333") 
+
+        result = arvados_cwl.parse_output_properties_yaml(tmp.name)
+
+        self.assertEqual(result, {'foo':111, 'bar':222, 'baz':333})
+
+    @stubs
+    def test_error_when_invalid_yaml_file_path_specified(self, stubs):
+        file_path = "foo.txt"
+        with self.assertRaises(SystemExit):
+            arvados_cwl.main(
+                    ["--debug", "--output-properties-yaml", file_path,
+                     "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+                    sys.stdin, sys.stderr, api_client=stubs.api)
+
     @mock.patch("time.sleep")
     @stubs
     def test_submit_on_error(self, stubs, tm):
@@ -612,6 +639,58 @@ class TestSubmit(unittest.TestCase):
         self.assertEqual(capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
 
+    @stubs
+    def test_submit_output_properties(self, stubs):
+        capture_stdout = cStringIO.StringIO()
+        try:
+            exited = arvados_cwl.main(
+                 ["--debug", "--submit", "--no-wait", "--api=containers", "--output-properties=foo:bar,baz:qux",
+                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+            self.assertEqual(exited, 0)
+        except:
+            logging.exception("")
+
+        expect_container = copy.deepcopy(stubs.expect_container_spec)
+        expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
+                                       '--no-log-timestamps', '--disable-validate',
+                                       '--eval-timeout=20', '--thread-count=4',
+                                       '--enable-reuse', "--debug", '--on-error=continue',
+                                       "--output-properties=foo:bar,baz:qux", 
+                                       '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+
+        stubs.api.container_requests().create.assert_called_with(
+            body=JsonDiffMatcher(expect_container))
+        self.assertEqual(capture_stdout.getvalue(),
+                         stubs.expect_container_request_uuid + '\n')
+
+
+    @mock.patch("arvados_cwl.parse_output_properties_yaml", return_value = {'foo':'bar', 'baz':'quz'})
+    @stubs
+    def test_submit_output_properties_yaml(self, stubs, split_props):
+        capture_stdout = cStringIO.StringIO()
+        try:
+            exited = arvados_cwl.main(
+                 ["--debug", "--submit", "--no-wait", "--api=containers", "--output-properties-yaml=config.yaml",
+                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+            self.assertEqual(exited, 0)
+        except:
+            logging.exception("")
+
+        expect_container = copy.deepcopy(stubs.expect_container_spec)
+        expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
+                                       '--no-log-timestamps', '--disable-validate',
+                                       '--eval-timeout=20', '--thread-count=4',
+                                       '--enable-reuse', "--debug", '--on-error=continue',
+                                       "--output-properties-yaml=config.yaml", 
+                                       '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+
+        stubs.api.container_requests().create.assert_called_with(
+            body=JsonDiffMatcher(expect_container))
+        self.assertEqual(capture_stdout.getvalue(),
+                         stubs.expect_container_request_uuid + '\n')
+
     @stubs
     def test_submit_storage_classes(self, stubs):
         capture_stdout = cStringIO.StringIO()
@@ -641,7 +720,7 @@ class TestSubmit(unittest.TestCase):
     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
     @mock.patch("arvados_cwl.ArvCwlRunner.make_output_collection", return_value = (None, None))
     @stubs
-    def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
+    def test_collection_content_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
         def set_final_output(job_order, output_callback, runtimeContext):
             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
             return []
@@ -649,14 +728,14 @@ class TestSubmit(unittest.TestCase):
 
         try:
             exited = arvados_cwl.main(
-                ["--debug", "--local", "--storage-classes=foo",
+                    ["--debug", "--local", "--storage-classes=foo", "--output-properties=foo:bar,baz:qux",
                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
                 sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
             self.assertEqual(exited, 0)
         except:
             logging.exception("")
 
-        make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
+        make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], {"foo" : "bar", "baz" : "qux"}, '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 
     @mock.patch("arvados_cwl.task_queue.TaskQueue")
     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
@@ -677,7 +756,7 @@ class TestSubmit(unittest.TestCase):
         except:
             logging.exception("")
 
-        make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
+        make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], None, '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 
     @stubs
     def test_submit_container_output_ttl(self, stubs):

diff --git a/sdk/cwl/tests/test_util.py b/sdk/cwl/tests/test_util.py
index 2532bd596c0f3dfac3554d3b5fe31ffb9d7e50d4..5b3b7eacae66d77290a7661b028ff2255d512120 100644 (file)
--- a/sdk/cwl/tests/test_util.py
+++ b/sdk/cwl/tests/test_util.py
@@ -43,3 +43,17 @@ class TestUtil(unittest.TestCase):
         logger = mock.MagicMock()
 
         self.assertRaises(ApiError, get_current_container(api, num_retries=0, logger=logger))
+
+    def test_merge_dict(self):
+        a = {"a":1, "b":2}
+        b = {"b":3000, "c":4000}
+        c = merge_dict(a, b)
+
+        self.assertEqual(c, {"a":1, "b":3000, "c":4000})
+
+    def test_merge_dict_none_input(self):
+        a = None
+        b = None
+        c = merge_dict(a, b)
+
+        self.assertEqual(c, None)