From: Peter Amstutz Date: Tue, 23 Apr 2019 21:32:33 +0000 (-0400) Subject: 15028: Fixing 1.0 conformance tests X-Git-Tag: 2.0.0~317^2~21 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/09fcc3cf5ba690aa75c36dfef911c730f48f05ab 15028: Fixing 1.0 conformance tests Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- diff --git a/build/build-dev-docker-jobs-image.sh b/build/build-dev-docker-jobs-image.sh index 2e4c457382..52df80f582 100755 --- a/build/build-dev-docker-jobs-image.sh +++ b/build/build-dev-docker-jobs-image.sh @@ -37,8 +37,12 @@ fi cd "$WORKSPACE" py=python +pipcmd=pip if [[ -n "$PYCMD" ]] ; then - py="$PYCMD" ; + py="$PYCMD" + if [[ $py = python3 ]] ; then + pipcmd=pip3 + fi fi (cd sdk/python && python setup.py sdist) @@ -75,6 +79,6 @@ if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then cwl_runner_version=$(cd sdk/python && nohash_version_from_git 1.0) fi -docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk" +docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py --build-arg pipcmd=$pipcmd -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk" echo arv-keepdocker arvados/jobs $cwl_runner_version arv-keepdocker arvados/jobs $cwl_runner_version diff --git a/sdk/cwl/arvados_cwl/done.py b/sdk/cwl/arvados_cwl/done.py index 9b26ad7064..c0e3e0de2a 100644 --- a/sdk/cwl/arvados_cwl/done.py +++ b/sdk/cwl/arvados_cwl/done.py @@ -52,7 +52,7 @@ def done(self, record, tmpdir, outdir, keepdir): def done_outputs(self, record, tmpdir, outdir, keepdir): self.builder.outdir = outdir self.builder.pathmapper.keepdir = keepdir - return self.collect_outputs("keep:" + record["output"]) + return self.collect_outputs("keep:" + record["output"], record["exit_code"]) crunchstat_re = re.compile(r"^\d{4}-\d\d-\d\d_\d\d:\d\d:\d\d [a-z0-9]{5}-8i9sb-[a-z0-9]{15} \d+ \d+ stderr crunchstat:") timestamp_re = re.compile(r"^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d+Z) (.*)") diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py index 2875dbc222..20cff60600 100644 --- a/sdk/cwl/arvados_cwl/executor.py +++ b/sdk/cwl/arvados_cwl/executor.py @@ -45,6 +45,7 @@ from ._version import __version__ from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing, visit_class from cwltool.command_line_tool import compute_checksums +from cwltool.load_tool import load_tool logger = logging.getLogger('arvados.cwl-runner') metrics = logging.getLogger('arvados.cwl-runner.metrics') @@ -672,6 +673,11 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods "state": "RunningOnClient"}).execute(num_retries=self.num_retries) logger.info("Pipeline instance %s", self.pipeline["uuid"]) + if not isinstance(tool, Runner): + loadingContext.do_update = True + tool = load_tool(tool.doc_loader.idx[tool.tool["id"]], + loadingContext) + if runtimeContext.cwl_runner_job is not None: self.uuid = runtimeContext.cwl_runner_job.get('uuid') diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py index 3ccf49f9e8..79c5fc2090 100644 --- a/sdk/cwl/setup.py +++ b/sdk/cwl/setup.py @@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner', # Note that arvados/build/run-build-packages.sh looks at this # file to determine what version of cwltool and schema-salad to build. install_requires=[ - 'cwltool==1.0.20190422193858', + 'cwltool==1.0.20190423203253', 'schema-salad==4.1.20190305210046', 'typing >= 3.6.4', 'ruamel.yaml >=0.15.54, <=0.15.77', diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile index f9e370794d..dd067e9778 100644 --- a/sdk/dev-jobs.dockerfile +++ b/sdk/dev-jobs.dockerfile @@ -13,22 +13,19 @@ # (This dockerfile file must be located in the arvados/sdk/ directory because # of the docker build root.) -FROM debian:jessie +FROM debian:9 MAINTAINER Ward Vandewege ENV DEBIAN_FRONTEND noninteractive ARG pythoncmd=python +ARG pipcmd=pip RUN apt-get update -q && apt-get install -qy --no-install-recommends \ git ${pythoncmd}-pip ${pythoncmd}-virtualenv ${pythoncmd}-dev libcurl4-gnutls-dev \ libgnutls28-dev nodejs ${pythoncmd}-pyasn1-modules build-essential -RUN if [ "$pythoncmd" = "python3" ]; then \ - pip3 install -U setuptools six requests ; \ - else \ - pip install -U setuptools six requests ; \ - fi +RUN $pipcmd install -U setuptools six requests ARG sdk ARG runner @@ -40,10 +37,10 @@ ADD cwl/salad_dist/$salad /tmp/ ADD cwl/cwltool_dist/$cwltool /tmp/ ADD cwl/dist/$runner /tmp/ -RUN cd /tmp/arvados-python-client-* && $pythoncmd setup.py install -RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && $pythoncmd setup.py install ; fi -RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && $pythoncmd setup.py install ; fi -RUN cd /tmp/arvados-cwl-runner-* && $pythoncmd setup.py install +RUN cd /tmp/arvados-python-client-* && $pipcmd install . +RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && $pipcmd install . ; fi +RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && $pipcmd install networkx==2.2 && $pipcmd install . ; fi +RUN cd /tmp/arvados-cwl-runner-* && $pipcmd install . # Install dependencies and set up system. RUN /usr/sbin/adduser --disabled-password \