projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Merge branch 'master' into origin-8019-crunchrun-log-throttle
[arvados.git]
/
sdk
/
cwl
/
arvados_cwl
/
crunch_script.py
diff --git
a/sdk/cwl/arvados_cwl/crunch_script.py
b/sdk/cwl/arvados_cwl/crunch_script.py
index cc9d87899cc9973bcc02e645848ec175227b752d..65ef50826eef31459b8539f6fbc1dcce32cce5a9 100644
(file)
--- a/
sdk/cwl/arvados_cwl/crunch_script.py
+++ b/
sdk/cwl/arvados_cwl/crunch_script.py
@@
-19,13
+19,18
@@
import re
import functools
from arvados.api import OrderedJsonModel
import functools
from arvados.api import OrderedJsonModel
-from cwltool.process import shortname, adjustFileObjs, adjustDirObjs,
getListing,
normalizeFilesDirs
+from cwltool.process import shortname, adjustFileObjs, adjustDirObjs, normalizeFilesDirs
from cwltool.load_tool import load_tool
from cwltool.errors import WorkflowException
from cwltool.load_tool import load_tool
from cwltool.errors import WorkflowException
+from .fsaccess import CollectionFetcher, CollectionFsAccess
+
logger = logging.getLogger('arvados.cwl-runner')
def run():
logger = logging.getLogger('arvados.cwl-runner')
def run():
+ # Timestamps are added by crunch-job, so don't print redundant timestamps.
+ arvados.log_handler.setFormatter(logging.Formatter('%(name)s %(levelname)s: %(message)s'))
+
# Print package versions
logger.info(arvados_cwl.versionstring())
# Print package versions
logger.info(arvados_cwl.versionstring())
@@
-36,6
+41,7
@@
def run():
runner = None
try:
job_order_object = arvados.current_job()['script_parameters']
runner = None
try:
job_order_object = arvados.current_job()['script_parameters']
+ toolpath = "file://%s/%s" % (os.environ['TASK_KEEPMOUNT'], job_order_object.pop("cwl:tool"))
pdh_path = re.compile(r'^[0-9a-f]{32}\+\d+(/.+)?$')
pdh_path = re.compile(r'^[0-9a-f]{32}\+\d+(/.+)?$')
@@
-46,9
+52,8
@@
def run():
return v
def keeppathObj(v):
return v
def keeppathObj(v):
- v["location"] = keeppath(v["location"])
-
- job_order_object["cwl:tool"] = "file://%s/%s" % (os.environ['TASK_KEEPMOUNT'], job_order_object["cwl:tool"])
+ if "location" in v:
+ v["location"] = keeppath(v["location"])
for k,v in job_order_object.items():
if isinstance(v, basestring) and arvados.util.keep_locator_pattern.match(v):
for k,v in job_order_object.items():
if isinstance(v, basestring) and arvados.util.keep_locator_pattern.match(v):
@@
-60,37
+65,53
@@
def run():
adjustFileObjs(job_order_object, keeppathObj)
adjustDirObjs(job_order_object, keeppathObj)
normalizeFilesDirs(job_order_object)
adjustFileObjs(job_order_object, keeppathObj)
adjustDirObjs(job_order_object, keeppathObj)
normalizeFilesDirs(job_order_object)
- adjustDirObjs(job_order_object, functools.partial(getListing, arvados_cwl.fsaccess.CollectionFsAccess("", api_client=api)))
output_name = None
output_name = None
+ output_tags = None
enable_reuse = True
enable_reuse = True
+ on_error = "continue"
if "arv:output_name" in job_order_object:
output_name = job_order_object["arv:output_name"]
del job_order_object["arv:output_name"]
if "arv:output_tags" in job_order_object:
if "arv:output_name" in job_order_object:
output_name = job_order_object["arv:output_name"]
del job_order_object["arv:output_name"]
if "arv:output_tags" in job_order_object:
-
args.
output_tags = job_order_object["arv:output_tags"]
+ output_tags = job_order_object["arv:output_tags"]
del job_order_object["arv:output_tags"]
if "arv:enable_reuse" in job_order_object:
enable_reuse = job_order_object["arv:enable_reuse"]
del job_order_object["arv:enable_reuse"]
del job_order_object["arv:output_tags"]
if "arv:enable_reuse" in job_order_object:
enable_reuse = job_order_object["arv:enable_reuse"]
del job_order_object["arv:enable_reuse"]
+ if "arv:on_error" in job_order_object:
+ on_error = job_order_object["arv:on_error"]
+ del job_order_object["arv:on_error"]
+
runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()),
runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()),
- output_name=output_name)
+ output_name=output_name, output_tags=output_tags)
+
+ make_fs_access = functools.partial(CollectionFsAccess,
+ collection_cache=runner.collection_cache)
- t = load_tool(job_order_object, runner.arv_make_tool)
+ t = load_tool(toolpath, runner.arv_make_tool,
+ fetcher_constructor=functools.partial(CollectionFetcher,
+ api_client=runner.api,
+ fs_access=make_fs_access(""),
+ num_retries=runner.num_retries))
args = argparse.Namespace()
args.project_uuid = arvados.current_job()["owner_uuid"]
args.enable_reuse = enable_reuse
args = argparse.Namespace()
args.project_uuid = arvados.current_job()["owner_uuid"]
args.enable_reuse = enable_reuse
+ args.on_error = on_error
args.submit = False
args.submit = False
- args.debug =
Tru
e
+ args.debug =
Fals
e
args.quiet = False
args.ignore_docker_for_reuse = False
args.basedir = os.getcwd()
args.quiet = False
args.ignore_docker_for_reuse = False
args.basedir = os.getcwd()
+ args.name = None
args.cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]}
args.cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]}
- outputObj = runner.arv_executor(t, job_order_object, **vars(args))
+ args.make_fs_access = make_fs_access
+
+ runner.arv_executor(t, job_order_object, **vars(args))
except Exception as e:
if isinstance(e, WorkflowException):
logging.info("Workflow error %s", e)
except Exception as e:
if isinstance(e, WorkflowException):
logging.info("Workflow error %s", e)