projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
10081: Upload dependencies. Fix keep paths. Ensure correct quoting.
[arvados.git]
/
crunch_scripts
/
cwl-runner
diff --git
a/crunch_scripts/cwl-runner
b/crunch_scripts/cwl-runner
index 8a4de24d356c6bfe8ed9547bc057da9a3edea5ec..fe4e8009aca1af0b04fb483983a09fd59de25390 100755
(executable)
--- a/
crunch_scripts/cwl-runner
+++ b/
crunch_scripts/cwl-runner
@@
-18,8
+18,10
@@
import logging
import os
import json
import argparse
import os
import json
import argparse
+import re
from arvados.api import OrderedJsonModel
from arvados.api import OrderedJsonModel
-from cwltool.process import adjustFiles
+from cwltool.process import adjustFileObjs, adjustDirObjs
+from cwltool.load_tool import load_tool
# Print package versions
logging.info(cwltool.main.versionstring())
# Print package versions
logging.info(cwltool.main.versionstring())
@@
-29,17
+31,32
@@
api = arvados.api("v1")
try:
job_order_object = arvados.current_job()['script_parameters']
try:
job_order_object = arvados.current_job()['script_parameters']
+ pdh_path = re.compile(r'^[0-9a-f]{32}\+\d+(/.+)?$')
+
def keeppath(v):
def keeppath(v):
- if arvados.util.keep_locator_pattern.match(v):
- return "file://%s/%s" % (os.environ['TASK_KEEPMOUNT'], v)
+ if pdh_path.match(v):
+ return "keep:%s" % v
+ else:
+ return v
+
+ def keeppathObj(v):
+ v["location"] = keeppath(v["location"])
+
+ job_order_object["cwl:tool"] = "file://%s/%s" % (os.environ['TASK_KEEPMOUNT'], job_order_object["cwl:tool"])
- job_order_object["cwl:tool"] = keeppath(job_order_object["cwl:tool"])
+ for k,v in job_order_object.items():
+ if isinstance(v, basestring) and arvados.util.keep_locator_pattern.match(v):
+ job_order_object[k] = {
+ "class": "File",
+ "location": "keep:%s" % v
+ }
- adjustFiles(job_order_object, keeppath)
+ adjustFileObjs(job_order_object, keeppathObj)
+ adjustDirObjs(job_order_object, keeppathObj)
runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()))
runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()))
- t =
cwltool.main.load_tool(job_order_object, False, True, runner.arvMakeTool, True
)
+ t =
load_tool(job_order_object, runner.arvMakeTool
)
args = argparse.Namespace()
args.project_uuid = arvados.current_job()["owner_uuid"]
args = argparse.Namespace()
args.project_uuid = arvados.current_job()["owner_uuid"]
@@
-48,18
+65,21
@@
try:
args.debug = True
args.quiet = False
args.ignore_docker_for_reuse = False
args.debug = True
args.quiet = False
args.ignore_docker_for_reuse = False
- outputObj = runner.arvExecutor(t, job_order_object, "", args, cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]})
+ args.basedir = os.getcwd()
+ args.cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]}
+ outputObj = runner.arvExecutor(t, job_order_object, **vars(args))
files = {}
files = {}
- def capture(path):
+ def capture(fileobj):
+ path = fileobj["location"]
sp = path.split("/")
col = sp[0][5:]
if col not in files:
files[col] = set()
files[col].add("/".join(sp[1:]))
sp = path.split("/")
col = sp[0][5:]
if col not in files:
files[col] = set()
files[col].add("/".join(sp[1:]))
-
return
path
+
fileobj["location"] =
path
- adjustFiles(outputObj, capture)
+ adjustFile
Obj
s(outputObj, capture)
final = arvados.collection.Collection()
final = arvados.collection.Collection()
@@
-68,10
+88,10
@@
try:
for f in c:
final.copy(f, f, c, True)
for f in c:
final.copy(f, f, c, True)
- def makeRelative(
path
):
-
return "/".join(path
.split("/")[1:])
+ def makeRelative(
fileobj
):
+
fileobj["location"] = "/".join(fileobj["location"]
.split("/")[1:])
- adjustFiles(outputObj, makeRelative)
+ adjustFile
Obj
s(outputObj, makeRelative)
with final.open("cwl.output.json", "w") as f:
json.dump(outputObj, f, indent=4)
with final.open("cwl.output.json", "w") as f:
json.dump(outputObj, f, indent=4)