13627: Fix crunch_script

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index e3bce68e6abdeb56fb03144c1496fe8103c55714..70c2173db9fa2f7ff5054ff4be7252bd64156b67 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -106,11 +106,14 @@ class ArvadosJob(JobBase):
 
         with Perf(metrics, "arv_docker_get_image %s" % self.name):
             (docker_req, docker_is_req) = self.get_requirement("DockerRequirement")
-            if docker_req and runtimeContextuse_container is not False:
+            if docker_req and runtimeContext.use_container is not False:
                 if docker_req.get("dockerOutputDirectory"):
                     raise SourceLine(docker_req, "dockerOutputDirectory", UnsupportedRequirement).makeError(
                         "Option 'dockerOutputDirectory' of DockerRequirement not supported.")
-                runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api, docker_req, pull_image, self.arvrunner.project_uuid)
+                runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api,
+                                                                           docker_req,
+                                                                           runtimeContext.pull_image,
+                                                                           self.arvrunner.project_uuid)
             else:
                 runtime_constraints["docker_image"] = "arvados/jobs"
 

diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index 742c671c74b1acc2c523e8beadfc243777146130..5b1806b354b1c6b891df57c7a54a35825b1f548c 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -21,7 +21,7 @@ class ArvadosCommandTool(CommandLineTool):
         elif runtimeContext.work_api == "jobs":
             return partial(ArvadosJob, self.arvrunner)
 
-    def make_path_mapper(self, reffiles, stagedir, runtimeContext):
+    def make_path_mapper(self, reffiles, stagedir, runtimeContext, separateDirs):
         if runtimeContext.work_api == "containers":
             return ArvPathMapper(self.arvrunner, reffiles+runtimeContext.extra_reffiles, runtimeContext.basedir,
                                  "/keep/%s",

diff --git a/sdk/cwl/arvados_cwl/crunch_script.py b/sdk/cwl/arvados_cwl/crunch_script.py
index 94e9c8fc84abcd2511935d63b7c413e77da38948..9f0c91f111b0f547c2bb60f3f9c48faf0bbe0404 100644 (file)
--- a/sdk/cwl/arvados_cwl/crunch_script.py
+++ b/sdk/cwl/arvados_cwl/crunch_script.py
@@ -27,7 +27,7 @@ from cwltool.process import shortname
 from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, normalizeFilesDirs
 from cwltool.load_tool import load_tool
 from cwltool.errors import WorkflowException
-from cwltool.context import RuntimeContext
+from arvados_cwl.context import ArvRuntimeContext
 
 from .fsaccess import CollectionFetcher, CollectionFsAccess
 
@@ -98,25 +98,27 @@ def run():
             debug = job_order_object["arv:debug"]
             del job_order_object["arv:debug"]
 
+        arvargs = argparse.Namespace()
+        arvargs.work_api = "jobs"
+        arvargs.output_name = output_name
+        arvargs.output_tags = output_tags
+        arvargs.thread_count = 1
+
         runner = arvados_cwl.ArvCwlRunner(api_client=arvados.safeapi.ThreadSafeApiCache(
             api_params={"model": OrderedJsonModel()}, keep_params={"num_retries": 4}),
-                                          output_name=output_name, output_tags=output_tags)
+                                          arvargs=arvargs)
 
         make_fs_access = functools.partial(CollectionFsAccess,
                                  collection_cache=runner.collection_cache)
 
-        t = load_tool(toolpath, runner.arv_make_tool,
-                      fetcher_constructor=functools.partial(CollectionFetcher,
-                                                  api_client=runner.api,
-                                                  fs_access=make_fs_access(""),
-                                                  num_retries=runner.num_retries))
+        t = load_tool(toolpath, runner.loadingContext)
 
         if debug:
             logger.setLevel(logging.DEBUG)
             logging.getLogger('arvados').setLevel(logging.DEBUG)
             logging.getLogger("cwltool").setLevel(logging.DEBUG)
 
-        args = RuntimeContext()
+        args = ArvRuntimeContext(vars(arvargs))
         args.project_uuid = arvados.current_job()["owner_uuid"]
         args.enable_reuse = enable_reuse
         args.on_error = on_error
@@ -135,7 +137,7 @@ def run():
         args.disable_js_validation = False
         args.tmp_outdir_prefix = "tmp"
 
-        runner.arv_executor(t, job_order_object, args)
+        runner.arv_executor(t, job_order_object, args, logger=logger)
     except Exception as e:
         if isinstance(e, WorkflowException):
             logging.info("Workflow error %s", e)

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 13d1d581bf6f481e797c526a3b6f90873780e023..55548130d3277a10540e867a08f6a0dcfd3e48a9 100644 (file)
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20180615142437',
+          'cwltool==1.0.20180615183820',
           'schema-salad==2.7.20180501211602',
           'typing >= 3.5.3',
           'ruamel.yaml >=0.13.11, <0.15',