WORKSPACE=path Path to the Arvados source tree to build packages from
CWLTOOL=path (optional) Path to cwltool git repository.
SALAD=path (optional) Path to schema_salad git repository.
-PYCMD=pythonexec (optional) Specify the python executable to use in the docker image. Defaults to "python".
+PYCMD=pythonexec (optional) Specify the python executable to use in the docker image. Defaults to "python3".
EOF
cd "$WORKSPACE"
-py=python
+py=python3
pipcmd=pip
if [[ -n "$PYCMD" ]] ; then
py="$PYCMD"
- if [[ $py = python3 ]] ; then
- pipcmd=pip3
- fi
+fi
+if [[ $py = python3 ]] ; then
+ pipcmd=pip3
fi
(cd sdk/python && python setup.py sdist)
cwl_runner_version=$(cd sdk/python && nohash_version_from_git 1.0)
fi
-docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py --build-arg pipcmd=$pipcmd -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
+set -x
+docker build --no-cache --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py --build-arg pipcmd=$pipcmd -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
echo arv-keepdocker arvados/jobs $cwl_runner_version
arv-keepdocker arvados/jobs $cwl_runner_version
sdk/go/asyncbuf
sdk/go/stats
sdk/go/crunchrunner
-sdk/cwl
+sdk/cwl:py3
sdk/R
sdk/java-v2
tools/sync-groups
sdk/pam
sdk/python
sdk/python:py3
- sdk/cwl
sdk/cwl:py3
services/dockercleaner:py3
services/fuse
${verb}_${target}
;;
*)
- testargs["$target"]="${opts}"
+ argstarget=${target%:py3}
+ testargs["$argstarget"]="${opts}"
tt="${testfuncargs[${target}]}"
tt="${tt:-$target}"
do_$verb $tt
import re
import pkg_resources # part of setuptools
-### begin monkey patch ###
-# Monkey patch solution for bug #16169
-#
-# There is a bug in upstream cwltool where the version updater needs
-# to replace the document fragments in the loader index with the
-# updated ones, but actually it only does it for the root document.
-# Normally we just fix the bug in upstream but that's challenging
-# because current cwltool dropped support for Python 2.7 and we're
-# still supporting py2 in Arvados 2.0 (although py2 support will most
-# likely be dropped in Arvados 2.1). Making a bugfix fork comes with
-# its own complications (it would need to be added to PyPi) so monkey
-# patching is the least disruptive fix (and is relatively safe because
-# our cwltool dependency is pinned to a specific version). This
-# should be removed as soon as a bugfix goes into upstream cwltool and
-# we upgrade to it.
-#
-import cwltool.load_tool
-from cwltool.utils import visit_class
-from six.moves import urllib
-original_resolve_and_validate_document = cwltool.load_tool.resolve_and_validate_document
-def wrapped_resolve_and_validate_document(
- loadingContext, # type: LoadingContext
- workflowobj, # type: Union[CommentedMap, CommentedSeq]
- uri, # type: Text
- preprocess_only=False, # type: bool
- skip_schemas=None, # type: Optional[bool]
- ):
- loadingContext, uri = original_resolve_and_validate_document(loadingContext, workflowobj, uri, preprocess_only, skip_schemas)
- if loadingContext.do_update in (True, None):
- fileuri = urllib.parse.urldefrag(uri)[0]
- def update_index(pr):
- loadingContext.loader.idx[pr["id"]] = pr
- visit_class(loadingContext.loader.idx[fileuri], ("CommandLineTool", "Workflow", "ExpressionTool"), update_index)
- return loadingContext, uri
-cwltool.load_tool.resolve_and_validate_document = wrapped_resolve_and_validate_document
-### end monkey patch ###
-
from schema_salad.sourceline import SourceLine
import schema_salad.validate as validate
import cwltool.main
import logging
from schema_salad.sourceline import SourceLine, cmap
+import schema_salad.ref_resolver
from cwltool.pack import pack
-from cwltool.load_tool import fetch_document
+from cwltool.load_tool import fetch_document, resolve_and_validate_document
from cwltool.process import shortname
from cwltool.workflow import Workflow, WorkflowException, WorkflowStep
from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
with SourceLine(self.tool, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
if "id" not in self.tool:
raise WorkflowException("%s object must have 'id'" % (self.tool["class"]))
- document_loader, workflowobj, uri = (self.doc_loader, self.doc_loader.fetch(self.tool["id"]), self.tool["id"])
discover_secondary_files(self.arvrunner.fs_access, builder,
self.tool["inputs"], joborder)
with Perf(metrics, "subworkflow upload_deps"):
upload_dependencies(self.arvrunner,
os.path.basename(joborder.get("id", "#")),
- document_loader,
+ self.doc_loader,
joborder,
joborder.get("id", "#"),
False)
if self.wf_pdh is None:
- workflowobj["requirements"] = dedup_reqs(self.requirements)
- workflowobj["hints"] = dedup_reqs(self.hints)
+ packed = pack(self.loadingContext, self.tool["id"], loader=self.doc_loader)
- packed = pack(document_loader, workflowobj, uri, self.metadata)
+ for p in packed["$graph"]:
+ if p["id"] == "#main":
+ p["requirements"] = dedup_reqs(self.requirements)
+ p["hints"] = dedup_reqs(self.hints)
def visit(item):
+ if "requirements" in item:
+ item["requirements"] = [i for i in item["requirements"] if i["class"] != "DockerRequirement"]
for t in ("hints", "requirements"):
if t not in item:
continue
raise WorkflowException("Non-top-level ResourceRequirement in single container cannot have expressions")
if not dyn:
self.static_resource_req.append(req)
- if req["class"] == "DockerRequirement":
- if "http://arvados.org/cwl#dockerCollectionPDH" in req:
- del req["http://arvados.org/cwl#dockerCollectionPDH"]
visit_class(packed["$graph"], ("Workflow", "CommandLineTool"), visit)
upload_dependencies(self.arvrunner,
runtimeContext.name,
- document_loader,
+ self.doc_loader,
packed,
- uri,
+ self.tool["id"],
False)
# Discover files/directories referenced by the
if job_res_reqs[0].get("ramMin", 1024) < 128:
job_res_reqs[0]["ramMin"] = 128
+ arguments = ["--no-container", "--move-outputs", "--preserve-entire-environment", "workflow.cwl", "cwl.input.yml"]
+ if runtimeContext.debug:
+ arguments.insert(0, '--debug')
+
wf_runner = cmap({
"class": "CommandLineTool",
"baseCommand": "cwltool",
}]
}],
"hints": self.hints,
- "arguments": ["--no-container", "--move-outputs", "--preserve-entire-environment", "workflow.cwl#main", "cwl.input.yml"],
+ "arguments": arguments,
"id": "#"
})
return ArvadosCommandTool(self.arvrunner, wf_runner, self.loadingContext).job(joborder_resolved, output_callback, runtimeContext)
A "packed" workflow is one where all the components have been combined into a single document."""
rewrites = {}
- packed = pack(tool.doc_loader, tool.doc_loader.fetch(tool.tool["id"]),
- tool.tool["id"], tool.metadata, rewrite_out=rewrites)
+ packed = pack(arvrunner.loadingContext, tool.tool["id"],
+ rewrite_out=rewrites,
+ loader=tool.doc_loader)
rewrite_to_orig = {v: k for k,v in viewitems(rewrites)}
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==1.0.20190831161204',
- 'schema-salad==4.5.20190815125611',
- 'typing >= 3.6.4',
- 'ruamel.yaml >=0.15.54, <=0.15.77',
+ 'cwltool==3.0.20200317203547',
+ 'schema-salad==5.0.20200302192450',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
- 'ciso8601 >= 2.0.0',
- 'networkx < 2.3'
+ 'ciso8601 >= 2.0.0'
],
extras_require={
':os.name=="posix" and python_version<"3"': ['subprocess32 >= 3.5.1'],
data_files=[
('share/doc/arvados-cwl-runner', ['LICENSE-2.0.txt', 'README.rst']),
],
+ python_requires=">=3.5, <4",
classifiers=[
- 'Programming Language :: Python :: 2',
'Programming Language :: Python :: 3',
],
test_suite='tests',
reset_container=1
leave_running=0
config=dev
+devcwl=0
tag="latest"
-pythoncmd=python
+pythoncmd=python3
suite=conformance
runapi=containers
build=1
shift
;;
+ --devcwl)
+ devcwl=1
+ shift
+ ;;
--pythoncmd)
pythoncmd=$2
shift ; shift
shift ; shift
;;
-h|--help)
- echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag] [--build] [--pythoncmd python(2|3)] [--suite (integration|conformance-v1.0|conformance-v1.1)]"
+ echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag] [--build] [--pythoncmd python(2|3)] [--suite (integration|conformance-v1.0|conformance-*)]"
exit
;;
*)
git clone https://github.com/common-workflow-language/common-workflow-language.git
fi
cd common-workflow-language
-elif [[ "$suite" = "conformance-v1.1" ]] ; then
- if ! test -d cwl-v1.1 ; then
- git clone https://github.com/common-workflow-language/cwl-v1.1.git
+elif [[ "$suite" =~ conformance-(.*) ]] ; then
+ version=\${BASH_REMATCH[1]}
+ if ! test -d cwl-\${version} ; then
+ git clone https://github.com/common-workflow-language/cwl-\${version}.git
fi
- cd cwl-v1.1
+ cd cwl-\${version}
fi
if [[ "$suite" != "integration" ]] ; then
EOF2
chmod +x /tmp/cwltest/arv-cwl-containers
+EXTRA=--compute-checksum
+
+if [[ $devcwl == 1 ]] ; then
+ EXTRA="\$EXTRA --enable-dev"
+fi
+
env
if [[ "$suite" = "integration" ]] ; then
cd /usr/src/arvados/sdk/cwl/tests
exec ./arvados-tests.sh $@
else
- exec ./run_test.sh RUNNER=/tmp/cwltest/arv-cwl-${runapi} EXTRA=--compute-checksum $@
+ exec ./run_test.sh RUNNER=/tmp/cwltest/arv-cwl-${runapi} EXTRA="\$EXTRA" $@
fi
EOF
import functools
import cwltool.process
import cwltool.secrets
+from cwltool.update import INTERNAL_VERSION
from schema_salad.ref_resolver import Loader
from schema_salad.sourceline import cmap
cwltool.process._names = set()
def helper(self, runner, enable_reuse=True):
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
"basedir": "",
"make_fs_access": make_fs_access,
"loader": Loader({}),
- "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
+ "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
runtimeContext = arvados_cwl.context.ArvRuntimeContext(
{"work_api": "containers",
"basedir": "",
runner.api.collections().get().execute.return_value = {
"portable_data_hash": "99999999999999999999999999999993+99"}
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
tool = cmap({
"inputs": [],
cwltool.process._names = set()
def helper(self, runner, enable_reuse=True):
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
"basedir": "",
"make_fs_access": make_fs_access,
"loader": document_loader,
- "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
+ "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
"construct_tool_object": runner.arv_make_tool})
runtimeContext = arvados_cwl.context.ArvRuntimeContext(
{"work_api": "containers",
"--no-container",
"--move-outputs",
"--preserve-entire-environment",
- "workflow.cwl#main",
+ "workflow.cwl",
"cwl.input.yml"
],
"container_image": "99999999999999999999999999999993+99",
u'--no-container',
u'--move-outputs',
u'--preserve-entire-environment',
- u'workflow.cwl#main',
+ u'workflow.cwl',
u'cwl.input.yml'
],
'use_existing': True,
import sys
import unittest
import cwltool.process
+import re
from io import BytesIO
self.assertEqual(exited, 1)
self.assertRegexpMatches(
- capture_stderr.getvalue(),
+ re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
finally:
cwltool_logger.removeHandler(stderr_logger)
{
"$graph": [
{
+ "$namespaces": {
+ "arv": "http://arvados.org/cwl#"
+ },
"class": "Workflow",
- "cwlVersion": "v1.1",
+ "cwlVersion": "v1.0",
"hints": [],
"id": "#main",
"inputs": [
"run": {
"baseCommand": "sleep",
"class": "CommandLineTool",
- "id": "#main/sleep1/run/subtool",
+ "id": "#main/sleep1/subtool",
"inputs": [
{
- "id": "#main/sleep1/run/subtool/sleeptime",
+ "id": "#main/sleep1/subtool/sleeptime",
"inputBinding": {
"position": 1
},
],
"outputs": [
{
- "id": "#main/sleep1/run/subtool/out",
+ "id": "#main/sleep1/subtool/out",
"outputBinding": {
"outputEval": "out"
},
]
}
],
- "cwlVersion": "v1.1"
-}
+ "cwlVersion": "v1.0"
+}
\ No newline at end of file
'google-api-python-client >=1.6.2, <1.7',
'httplib2 >=0.9.2',
'pycurl >=7.19.5.1',
- 'ruamel.yaml >=0.15.54, <=0.15.77',
+ 'ruamel.yaml >=0.15.54, <=0.16.5',
'setuptools',
'ws4py >=0.4.2',
],
projects / arvados.git / commitdiff