15028: Update cwltool/schema-salad deps, fix tests

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 8e6bff4f04cb525106ee18a62cfb6146542f3c18..325fccb2c97e1dd9259ececbdbaab13bfa7cf39d 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -288,6 +288,10 @@ class ArvadosWorkflow(Workflow):
                 adjustDirObjs(packed, keepmount)
                 self.wf_pdh = upload_workflow_collection(self.arvrunner, shortname(self.tool["id"]), packed)
 
+        self.loadingContext = self.loadingContext.copy()
+        self.loadingContext.metadata = self.loadingContext.metadata.copy()
+        self.loadingContext.metadata["http://commonwl.org/cwltool#original_cwlVersion"] = "v1.0"
+
         wf_runner = cmap({
             "class": "CommandLineTool",
             "baseCommand": "cwltool",

diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index eeb44dbd7f232193b6e7102f85e3ae905b71dfc9..2875dbc22263314bd04c2699e93370d841c0c89c 100644 (file)
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -187,6 +187,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
         self.loadingContext.fetcher_constructor = self.fetcher_constructor
         self.loadingContext.resolver = partial(collectionResolver, self.api, num_retries=self.num_retries)
         self.loadingContext.construct_tool_object = self.arv_make_tool
+        self.loadingContext.do_update = False
 
         # Add a custom logging handler to the root logger for runtime status reporting
         # if running inside a container

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 9385bde63c4aa60b55de22d7c8b87908039444e3..3235f4763c36506184d90a74a60136474d30b1e4 100644 (file)
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -31,6 +31,7 @@ from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
 from cwltool.utils import aslist
 from cwltool.builder import substitute
 from cwltool.pack import pack
+from cwltool.update import INTERNAL_VERSION
 import schema_salad.validate as validate
 
 import arvados.collection
@@ -451,6 +452,10 @@ class Runner(Process):
                  collection_cache_size=256,
                  collection_cache_is_default=True):
 
+        loadingContext = loadingContext.copy()
+        loadingContext.metadata = loadingContext.metadata.copy()
+        loadingContext.metadata["cwlVersion"] = INTERNAL_VERSION
+
         super(Runner, self).__init__(tool.tool, loadingContext)
 
         self.arvrunner = runner

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 1052fb0d76606ddf160c685830a88649b7c40acf..3ccf49f9e8f083768334fdbf22f302614efd9355 100644 (file)
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,8 +33,8 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20181217162649',
-          'schema-salad==3.0.20181129082112',
+          'cwltool==1.0.20190422193858',
+          'schema-salad==4.1.20190305210046',
           'typing >= 3.6.4',
           'ruamel.yaml >=0.15.54, <=0.15.77',
           'arvados-python-client>=1.3.0.20190205182514',

diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 07d962bf9b55e2d33efab5b470b3146cab8c49f9..2ddd06535e6628ae83514a49c1e0a71df840a4d1 100644 (file)
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -57,7 +57,7 @@ class CollectionMock(object):
 class TestContainer(unittest.TestCase):
 
     def helper(self, runner, enable_reuse=True):
-        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
 
         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
@@ -66,7 +66,7 @@ class TestContainer(unittest.TestCase):
              "basedir": "",
              "make_fs_access": make_fs_access,
              "loader": Loader({}),
-             "metadata": {"cwlVersion": "v1.0"}})
+             "metadata": {"cwlVersion": "v1.1.0-dev1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
             {"work_api": "containers",
              "basedir": "",
@@ -400,7 +400,7 @@ class TestContainer(unittest.TestCase):
         runner.api.collections().get().execute.return_value = {
             "portable_data_hash": "99999999999999999999999999999993+99"}
 
-        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
 
         tool = cmap({
             "inputs": [],
@@ -607,7 +607,7 @@ class TestContainer(unittest.TestCase):
             "portable_data_hash": "99999999999999999999999999999994+99",
             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 
-        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
 
         tool = cmap({
             "inputs": [
@@ -697,7 +697,7 @@ class TestContainer(unittest.TestCase):
         runner.api.collections().get().execute.return_value = {
             "portable_data_hash": "99999999999999999999999999999993+99"}
 
-        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
 
         tool = cmap({"arguments": ["md5sum", "example.conf"],
                      "class": "CommandLineTool",
@@ -803,7 +803,7 @@ class TestContainer(unittest.TestCase):
             "class": "CommandLineTool",
             "hints": [
                 {
-                    "class": "http://commonwl.org/cwltool#TimeLimit",
+                    "class": "ToolTimeLimit",
                     "timelimit": 42
                 }
             ]

diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 022d75be11a09031746adf7a39975793ec293e83..9cac68aa6c2ccb61952c656868e620504e19b1ed 100644 (file)
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -34,7 +34,7 @@ if not os.getenv('ARVADOS_DEBUG'):
 class TestJob(unittest.TestCase):
 
     def helper(self, runner, enable_reuse=True):
-        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
 
         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
@@ -43,7 +43,7 @@ class TestJob(unittest.TestCase):
              "basedir": "",
              "make_fs_access": make_fs_access,
              "loader": Loader({}),
-             "metadata": {"cwlVersion": "v1.0"},
+             "metadata": {"cwlVersion": "v1.1.0-dev1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
              "makeTool": runner.arv_make_tool})
         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
             {"work_api": "jobs",
@@ -343,7 +343,7 @@ class TestJob(unittest.TestCase):
 
 class TestWorkflow(unittest.TestCase):
     def helper(self, runner, enable_reuse=True):
-        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
 
         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
@@ -358,7 +358,7 @@ class TestWorkflow(unittest.TestCase):
              "basedir": "",
              "make_fs_access": make_fs_access,
              "loader": document_loader,
-             "metadata": {"cwlVersion": "v1.0"},
+             "metadata": {"cwlVersion": "v1.1.0-dev1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
              "construct_tool_object": runner.arv_make_tool})
         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
             {"work_api": "jobs",
@@ -458,6 +458,7 @@ bytes(b'''{
     @mock.patch("arvados.collection.Collection")
     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
+        # TODO copy this over to test_container
         arvados_cwl.add_arv_hints()
 
         api = mock.MagicMock()
@@ -475,16 +476,15 @@ bytes(b'''{
         runner.num_retries = 0
 
         loadingContext, runtimeContext = self.helper(runner)
-
+        loadingContext.do_update = True
         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
-        metadata["cwlVersion"] = tool["cwlVersion"]
 
         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
 
         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
         arvtool.formatgraph = None
         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
-        
+
         next(it).run(runtimeContext)
         next(it).run(runtimeContext)
 

diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 9535f6ba206bfadf267dda54bbad19fcecf0f961..5f92cee942aca33060cd638a3a67376e28ae85ae 100644 (file)
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -84,6 +84,7 @@ def stubs(func):
 
         stubs.api = mock.MagicMock()
         stubs.api._rootDesc = get_rootDesc()
+        stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
 
         stubs.api.users().current().execute.return_value = {
             "uuid": stubs.fake_user_uuid,

diff --git a/sdk/cwl/tests/wf/scatter2_subwf.cwl b/sdk/cwl/tests/wf/scatter2_subwf.cwl
index 2af1155437126b6a8e9ddb5c027abefdd230a277..ecd36026f2e6c487667be13620a244777ba3abd8 100644 (file)
--- a/sdk/cwl/tests/wf/scatter2_subwf.cwl
+++ b/sdk/cwl/tests/wf/scatter2_subwf.cwl
@@ -6,7 +6,7 @@
   "$graph": [
     {
       "class": "Workflow",
-      "cwlVersion": "v1.0",
+      "cwlVersion": "v1.1.0-dev1",
       "hints": [],
       "id": "#main",
       "inputs": [
@@ -59,10 +59,10 @@
           "run": {
             "baseCommand": "sleep",
             "class": "CommandLineTool",
-            "id": "#main/sleep1/subtool",
+            "id": "#main/sleep1/run/subtool",
             "inputs": [
               {
-                "id": "#main/sleep1/subtool/sleeptime",
+                "id": "#main/sleep1/run/subtool/sleeptime",
                 "inputBinding": {
                   "position": 1
                 },
@@ -71,7 +71,7 @@
             ],
             "outputs": [
               {
-                "id": "#main/sleep1/subtool/out",
+                "id": "#main/sleep1/run/subtool/out",
                 "outputBinding": {
                   "outputEval": "out"
                 },
@@ -83,5 +83,5 @@
       ]
     }
   ],
-  "cwlVersion": "v1.0"
+  "cwlVersion": "v1.1.0-dev1"
 }
\ No newline at end of file