+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
# Crunch script integration for running arvados-cwl-runner (importing
# arvados_cwl module) inside a crunch job.
#
import functools
from arvados.api import OrderedJsonModel
-from cwltool.process import shortname, adjustFileObjs, adjustDirObjs, normalizeFilesDirs
+from cwltool.process import shortname
+from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, normalizeFilesDirs
from cwltool.load_tool import load_tool
from cwltool.errors import WorkflowException
+from arvados_cwl.context import ArvRuntimeContext
from .fsaccess import CollectionFetcher, CollectionFsAccess
output_tags = None
enable_reuse = True
on_error = "continue"
+ debug = False
+
if "arv:output_name" in job_order_object:
output_name = job_order_object["arv:output_name"]
del job_order_object["arv:output_name"]
on_error = job_order_object["arv:on_error"]
del job_order_object["arv:on_error"]
- runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()),
- output_name=output_name, output_tags=output_tags)
+ if "arv:debug" in job_order_object:
+ debug = job_order_object["arv:debug"]
+ del job_order_object["arv:debug"]
+
+ arvargs = argparse.Namespace()
+ arvargs.work_api = "jobs"
+ arvargs.output_name = output_name
+ arvargs.output_tags = output_tags
+ arvargs.thread_count = 1
+
+ runner = arvados_cwl.ArvCwlRunner(api_client=arvados.safeapi.ThreadSafeApiCache(
+ api_params={"model": OrderedJsonModel()}, keep_params={"num_retries": 4}),
+ arvargs=arvargs)
make_fs_access = functools.partial(CollectionFsAccess,
collection_cache=runner.collection_cache)
- t = load_tool(toolpath, runner.arv_make_tool,
- fetcher_constructor=functools.partial(CollectionFetcher,
- api_client=runner.api,
- fs_access=make_fs_access(""),
- num_retries=runner.num_retries))
+ t = load_tool(toolpath, runner.loadingContext)
+
+ if debug:
+ logger.setLevel(logging.DEBUG)
+ logging.getLogger('arvados').setLevel(logging.DEBUG)
+ logging.getLogger("cwltool").setLevel(logging.DEBUG)
- args = argparse.Namespace()
+ args = ArvRuntimeContext(vars(arvargs))
args.project_uuid = arvados.current_job()["owner_uuid"]
args.enable_reuse = enable_reuse
args.on_error = on_error
args.submit = False
- args.debug = False
+ args.debug = debug
args.quiet = False
args.ignore_docker_for_reuse = False
args.basedir = os.getcwd()
args.name = None
args.cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]}
args.make_fs_access = make_fs_access
-
- runner.arv_executor(t, job_order_object, **vars(args))
+ args.trash_intermediate = False
+ args.intermediate_output_ttl = 0
+ args.priority = arvados_cwl.DEFAULT_PRIORITY
+ args.do_validate = True
+ args.disable_js_validation = False
+ args.tmp_outdir_prefix = "tmp"
+
+ runner.arv_executor(t, job_order_object, args, logger=logger)
except Exception as e:
if isinstance(e, WorkflowException):
logging.info("Workflow error %s", e)