18002: Fixes for cwltool update

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 1029ca0e05bff2e0bf501f61fa65f19ea482b2e6..1e79566f4055578ce61c0b37cd9c753429e1da51 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -57,6 +57,12 @@ class ArvadosContainer(JobBase):
     def update_pipeline_component(self, r):
         pass
 
+    def _required_env(self):
+        env = {}
+        env["HOME"] = self.outdir
+        env["TMPDIR"] = self.tmpdir
+        return env
+
     def run(self, runtimeContext):
         # ArvadosCommandTool subclasses from cwltool.CommandLineTool,
         # which calls makeJobRunner() to get a new ArvadosContainer
@@ -234,8 +240,6 @@ class ArvadosContainer(JobBase):
                                 "path": "%s/%s" % (self.outdir, self.stdout)}
 
         (docker_req, docker_is_req) = self.get_requirement("DockerRequirement")
-        if not docker_req:
-            docker_req = {"dockerImageId": "arvados/jobs:"+__version__}
 
         container_request["container_image"] = arv_docker_get_image(self.arvrunner.api,
                                                                     docker_req,

diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index 13664a8dfb0d57df0477d4c627928b9be17ad8d7..83648f46aa89424652323729b0241e85d2d125e8 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -6,6 +6,7 @@ from cwltool.command_line_tool import CommandLineTool, ExpressionTool
 from .arvcontainer import ArvadosContainer
 from .pathmapper import ArvPathMapper
 from .runner import make_builder
+from ._version import __version__
 from functools import partial
 from schema_salad.sourceline import SourceLine
 from cwltool.errors import WorkflowException
@@ -57,6 +58,12 @@ class ArvadosCommandTool(CommandLineTool):
 
     def __init__(self, arvrunner, toolpath_object, loadingContext):
         super(ArvadosCommandTool, self).__init__(toolpath_object, loadingContext)
+
+        (docker_req, docker_is_req) = self.get_requirement("DockerRequirement")
+        if not docker_req:
+            self.hints.append({"class": "DockerRequirement",
+                               "dockerImageId": "arvados/jobs:"+__version__})
+
         self.arvrunner = arvrunner
 
     def make_job_runner(self, runtimeContext):

diff --git a/sdk/cwl/arvados_cwl/context.py b/sdk/cwl/arvados_cwl/context.py
index 77d4027ccbabccf72e3fe5f60ad049726c1b99d1..1e04dd5774ebb8bbc45ebdd417c35138f2d13a4d 100644 (file)
--- a/sdk/cwl/arvados_cwl/context.py
+++ b/sdk/cwl/arvados_cwl/context.py
@@ -41,3 +41,15 @@ class ArvRuntimeContext(RuntimeContext):
 
         if self.submit_request_uuid:
             self.submit_runner_cluster = self.submit_request_uuid[0:5]
+
+    def get_outdir(self) -> str:
+        """Return self.outdir or create one with self.tmp_outdir_prefix."""
+        return self.outdir
+
+    def get_tmpdir(self) -> str:
+        """Return self.tmpdir or create one with self.tmpdir_prefix."""
+        return self.tmpdir
+
+    def create_tmpdir(self) -> str:
+        """Return self.tmpdir or create one with self.tmpdir_prefix."""
+        return self.tmpdir

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 28eb50603deb081b2efd67d721b996d1bdf4333a..d6cc4173fc5e1e4a0b613cfc66f1cb6307d3516a 100644 (file)
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -40,7 +40,7 @@ setup(name='arvados-cwl-runner',
       # build.
       install_requires=[
           'cwltool==3.1.20210816212154',
-          'schema-salad==8.1.20210627200047',
+          'schema-salad==8.2.20210902094147',
           'arvados-python-client{}'.format(pysdk_dep),
           'setuptools',
           'ciso8601 >= 2.0.0',

diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 91283b0b622c5719630c82c19bb3c2c46ac7133f..8a380ff80b3c811ab2c8e050392f679674b2b20b 100644 (file)
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -123,7 +123,7 @@ class TestContainer(unittest.TestCase):
                 "baseCommand": "ls",
                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
                 "id": "#",
-                "class": "CommandLineTool"
+                "class": "org.w3id.cwl.cwl.CommandLineTool"
             })
 
             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
@@ -206,7 +206,7 @@ class TestContainer(unittest.TestCase):
             }],
             "baseCommand": "ls",
             "id": "#",
-            "class": "CommandLineTool"
+            "class": "org.w3id.cwl.cwl.CommandLineTool"
         })
 
         loadingContext, runtimeContext = self.helper(runner)
@@ -314,7 +314,7 @@ class TestContainer(unittest.TestCase):
             }],
             "baseCommand": "ls",
             "id": "#",
-            "class": "CommandLineTool"
+            "class": "org.w3id.cwl.cwl.CommandLineTool"
         })
 
         loadingContext, runtimeContext = self.helper(runner)
@@ -414,7 +414,7 @@ class TestContainer(unittest.TestCase):
             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
             "id": "#",
-            "class": "CommandLineTool"
+            "class": "org.w3id.cwl.cwl.CommandLineTool"
         })
 
         loadingContext, runtimeContext = self.helper(runner)
@@ -639,7 +639,7 @@ class TestContainer(unittest.TestCase):
             "baseCommand": "ls",
             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
             "id": "#",
-            "class": "CommandLineTool"
+            "class": "org.w3id.cwl.cwl.CommandLineTool"
         })
 
         loadingContext, runtimeContext = self.helper(runner)
@@ -720,7 +720,7 @@ class TestContainer(unittest.TestCase):
         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 
         tool = cmap({"arguments": ["md5sum", "example.conf"],
-                     "class": "CommandLineTool",
+                     "class": "org.w3id.cwl.cwl.CommandLineTool",
                      "hints": [
                          {
                              "class": "http://commonwl.org/cwltool#Secrets",
@@ -819,7 +819,7 @@ class TestContainer(unittest.TestCase):
             "baseCommand": "ls",
             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
             "id": "#",
-            "class": "CommandLineTool",
+            "class": "org.w3id.cwl.cwl.CommandLineTool",
             "hints": [
                 {
                     "class": "ToolTimeLimit",
@@ -862,7 +862,7 @@ class TestContainer(unittest.TestCase):
             "baseCommand": "ls",
             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
             "id": "#",
-            "class": "CommandLineTool",
+            "class": "org.w3id.cwl.cwl.CommandLineTool",
             "hints": [
                 {
                     "class": "http://arvados.org/cwl#OutputStorageClass",
@@ -936,7 +936,7 @@ class TestContainer(unittest.TestCase):
             "baseCommand": "ls",
             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
             "id": "#",
-            "class": "CommandLineTool",
+            "class": "org.w3id.cwl.cwl.CommandLineTool",
             "hints": [
             {
                 "class": "http://arvados.org/cwl#ProcessProperties",