sdk/cwl/arvados_cwl/arvtool.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from cwltool.command_line_tool import CommandLineTool
   6 from .arvjob import ArvadosJob
   7 from .arvcontainer import ArvadosContainer
   8 from .pathmapper import ArvPathMapper
   9 from functools import partial
  10 from schema_salad.sourceline import SourceLine
  11 from cwltool.errors import WorkflowException
  12
  13 def set_cluster_target(tool, arvrunner, builder, runtimeContext):
  14     cluster_target_req = None
  15     for field in ("hints", "requirements"):
  16         if field not in tool:
  17             continue
  18         for item in tool[field]:
  19             if item["class"] == "http://arvados.org/cwl#ClusterTarget":
  20                 cluster_target_req = item
  21
  22     if cluster_target_req is None:
  23         return runtimeContext
  24
  25     with SourceLine(cluster_target_req, None, WorkflowException, runtimeContext.debug):
  26         runtimeContext = runtimeContext.copy()
  27         runtimeContext.submit_runner_cluster = builder.do_eval(cluster_target_req.get("cluster_id")) or runtimeContext.submit_runner_cluster
  28         runtimeContext.project_uuid = builder.do_eval(cluster_target_req.get("project_uuid")) or runtimeContext.project_uuid
  29         if runtimeContext.submit_runner_cluster and runtimeContext.submit_runner_cluster not in arvrunner.api._rootDesc["remoteHosts"]:
  30             raise WorkflowException("Unknown or invalid cluster id '%s' known clusters are %s" % (runtimeContext.submit_runner_cluster,
  31                                                                                                       ", ".join(arvrunner.api._rootDesc["remoteHosts"].keys())))
  32     return runtimeContext
  33
  34 class ArvadosCommandTool(CommandLineTool):
  35     """Wrap cwltool CommandLineTool to override selected methods."""
  36
  37     def __init__(self, arvrunner, toolpath_object, loadingContext):
  38         super(ArvadosCommandTool, self).__init__(toolpath_object, loadingContext)
  39         self.arvrunner = arvrunner
  40
  41     def make_job_runner(self, runtimeContext):
  42         if runtimeContext.work_api == "containers":
  43             return partial(ArvadosContainer, self.arvrunner, runtimeContext)
  44         elif runtimeContext.work_api == "jobs":
  45             return partial(ArvadosJob, self.arvrunner)
  46         else:
  47             raise Exception("Unsupported work_api %s", runtimeContext.work_api)
  48
  49     def make_path_mapper(self, reffiles, stagedir, runtimeContext, separateDirs):
  50         if runtimeContext.work_api == "containers":
  51             return ArvPathMapper(self.arvrunner, reffiles+runtimeContext.extra_reffiles, runtimeContext.basedir,
  52                                  "/keep/%s",
  53                                  "/keep/%s/%s")
  54         elif runtimeContext.work_api == "jobs":
  55             return ArvPathMapper(self.arvrunner, reffiles, runtimeContext.basedir,
  56                                  "$(task.keep)/%s",
  57                                  "$(task.keep)/%s/%s")
  58
  59     def job(self, joborder, output_callback, runtimeContext):
  60
  61         # Workaround for #13365
  62         builderargs = runtimeContext.copy()
  63         builderargs.toplevel = True
  64         builderargs.tmp_outdir_prefix = ""
  65         builder = self._init_job(joborder, builderargs)
  66         joborder = builder.job
  67
  68         runtimeContext = runtimeContext.copy()
  69         runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext)
  70
  71         if runtimeContext.work_api == "containers":
  72             dockerReq, is_req = self.get_requirement("DockerRequirement")
  73             if dockerReq and dockerReq.get("dockerOutputDirectory"):
  74                 runtimeContext.outdir = dockerReq.get("dockerOutputDirectory")
  75                 runtimeContext.docker_outdir = dockerReq.get("dockerOutputDirectory")
  76             else:
  77                 runtimeContext.outdir = "/var/spool/cwl"
  78                 runtimeContext.docker_outdir = "/var/spool/cwl"
  79         elif runtimeContext.work_api == "jobs":
  80             runtimeContext.outdir = "$(task.outdir)"
  81             runtimeContext.docker_outdir = "$(task.outdir)"
  82             runtimeContext.tmpdir = "$(task.tmpdir)"
  83             runtimeContext.docker_tmpdir = "$(task.tmpdir)"
  84         return super(ArvadosCommandTool, self).job(joborder, output_callback, runtimeContext)