From e32de15f4de447dae7b3b6b91efff07ba53b91ed Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 16 Nov 2018 10:28:24 -0500 Subject: [PATCH] 14198: Update version deps for schema-salad, cwltool, ruamel.yaml, requests Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- build/build.list | 4 ++-- sdk/cwl/arvados_cwl/arvworkflow.py | 17 ++++++++++++++++- sdk/cwl/arvados_cwl/fsaccess.py | 6 ++++++ sdk/cwl/setup.py | 8 +++----- sdk/dev-jobs.dockerfile | 2 +- sdk/python/setup.py | 2 +- 6 files changed, 29 insertions(+), 10 deletions(-) diff --git a/build/build.list b/build/build.list index 4c3d740b0b..3b2e17fa85 100644 --- a/build/build.list +++ b/build/build.list @@ -33,7 +33,7 @@ debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|docker-py|1.7.2|2|pytho debian8,debian9,centos7|six|1.10.0|2|python3|all debian8,debian9,ubuntu1404,centos7|requests|2.12.4|2|python3|all debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|websocket-client|0.37.0|2|python3|all -ubuntu1404|requests|2.4.3|2|python|all +ubuntu1404|requests|2.6.1|2|python|all centos7|contextlib2|0.5.4|2|python|all centos7|isodate|0.5.4|2|python|all centos7|python-daemon|2.1.2|1|python|all @@ -44,7 +44,7 @@ centos7|networkx|1.11|0|python|all centos7|psutil|5.0.1|0|python|all debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|lockfile|0.12.2|2|python|all|--epoch 1 debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|subprocess32|3.5.1|2|python|all -all|ruamel.yaml|0.14.12|2|python|amd64|--python-setup-py-arguments --single-version-externally-managed +all|ruamel.yaml|0.15.77|2|python|amd64|--python-setup-py-arguments --single-version-externally-managed all|cwltest|1.0.20180518074130|4|python|all|--depends 'python-futures >= 3.0.5' --depends 'python-subprocess32 >= 3.5.0' all|junit-xml|1.8|3|python|all all|rdflib-jsonld|0.4.0|2|python|all diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py index e80c06f00d..8a7fb7f60e 100644 --- a/sdk/cwl/arvados_cwl/arvworkflow.py +++ b/sdk/cwl/arvados_cwl/arvworkflow.py @@ -12,7 +12,7 @@ from schema_salad.sourceline import SourceLine, cmap from cwltool.pack import pack from cwltool.load_tool import fetch_document from cwltool.process import shortname -from cwltool.workflow import Workflow, WorkflowException +from cwltool.workflow import Workflow, WorkflowException, WorkflowStep from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class from cwltool.builder import Builder from cwltool.context import LoadingContext @@ -118,6 +118,12 @@ def get_overall_res_req(res_reqs): overall_res_req["class"] = "ResourceRequirement" return cmap(overall_res_req) +class ArvadosWorkflowStep(WorkflowStep): + def job(self, joborder, output_callback, runtimeContext): + builder = self._init_job(joborder, runtimeContext) + check_cluster_target(self, builder, runtimeContext) + return super(ArvadosWorkflowStep, self).job(joborder, output_callback, runtimeContext) + class ArvadosWorkflow(Workflow): """Wrap cwltool Workflow to override selected methods.""" @@ -282,3 +288,12 @@ class ArvadosWorkflow(Workflow): "id": "#" }) return ArvadosCommandTool(self.arvrunner, wf_runner, self.loadingContext).job(joborder_resolved, output_callback, runtimeContext) + + def make_workflow_step(self, + toolpath_object, # type: Dict[Text, Any] + pos, # type: int + loadingContext, # type: LoadingContext + parentworkflowProv=None # type: Optional[CreateProvProfile] + ): + # (...) -> WorkflowStep + return WorkflowStep(toolpath_object, pos, loadingContext, parentworkflowProv) diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py index 316a652529..b22b2ffd6f 100644 --- a/sdk/cwl/arvados_cwl/fsaccess.py +++ b/sdk/cwl/arvados_cwl/fsaccess.py @@ -265,6 +265,12 @@ class CollectionFetcher(DefaultFetcher): return super(CollectionFetcher, self).urljoin(base_url, url) + schemes = [u"file", u"http", u"https", u"mailto", u"keep"] + + def supported_schemes(self): # type: () -> List[Text] + return self.schemes + + workflow_uuid_pattern = re.compile(r'[a-z0-9]{5}-7fd4e-[a-z0-9]{15}') pipeline_template_uuid_pattern = re.compile(r'[a-z0-9]{5}-p5p6p-[a-z0-9]{15}') diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py index 91a03ed865..36aed32171 100644 --- a/sdk/cwl/setup.py +++ b/sdk/cwl/setup.py @@ -33,12 +33,10 @@ setup(name='arvados-cwl-runner', # Note that arvados/build/run-build-packages.sh looks at this # file to determine what version of cwltool and schema-salad to build. install_requires=[ - 'cwltool==1.0.20181114114354', - 'schema-salad==2.7.20181017120439', + 'cwltool==1.0.20181116032456', + 'schema-salad==2.7.20181116024232', 'typing >= 3.6.4', - # Need to limit ruamel.yaml version to 0.15.26 because of bug - # https://bitbucket.org/ruamel/yaml/issues/227/regression-parsing-flow-mapping - 'ruamel.yaml >=0.13.11, <= 0.15.26', + 'ruamel.yaml >=0.13.11, <= 0.15.77', 'arvados-python-client>=1.1.4.20180607143841', 'setuptools', 'ciso8601 >=1.0.6, <2.0.0', diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile index aa1f18052f..d33956ccc3 100644 --- a/sdk/dev-jobs.dockerfile +++ b/sdk/dev-jobs.dockerfile @@ -20,7 +20,7 @@ ENV DEBIAN_FRONTEND noninteractive RUN apt-get update -q && apt-get install -qy git python-pip python-virtualenv python-dev libcurl4-gnutls-dev libgnutls28-dev nodejs python-pyasn1-modules -RUN pip install -U setuptools six +RUN pip install -U setuptools six requests ARG sdk ARG runner diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 8f576196bc..e5126c12b0 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -51,7 +51,7 @@ setup(name='arvados-python-client', 'google-api-python-client >=1.6.2, <1.7', 'httplib2 >=0.9.2', 'pycurl >=7.19.5.1', - 'ruamel.yaml >=0.13.11, <= 0.15.26', + 'ruamel.yaml >=0.15.54', 'setuptools', 'ws4py >=0.4.2', 'subprocess32 >=3.5.1', -- 2.30.2