14198: Update version deps for schema-salad, cwltool, ruamel.yaml, requests

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/build/build.list b/build/build.list
index 4c3d740b0b82c21a71e274bfc0db1311bdcd3e43..3b2e17fa85252bd24664d30bc11292deb0fdb0d5 100644 (file)
--- a/build/build.list
+++ b/build/build.list
@@ -33,7 +33,7 @@ debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|docker-py|1.7.2|2|pytho
 debian8,debian9,centos7|six|1.10.0|2|python3|all
 debian8,debian9,ubuntu1404,centos7|requests|2.12.4|2|python3|all
 debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|websocket-client|0.37.0|2|python3|all
-ubuntu1404|requests|2.4.3|2|python|all
+ubuntu1404|requests|2.6.1|2|python|all
 centos7|contextlib2|0.5.4|2|python|all
 centos7|isodate|0.5.4|2|python|all
 centos7|python-daemon|2.1.2|1|python|all
@@ -44,7 +44,7 @@ centos7|networkx|1.11|0|python|all
 centos7|psutil|5.0.1|0|python|all
 debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|lockfile|0.12.2|2|python|all|--epoch 1
 debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|subprocess32|3.5.1|2|python|all
-all|ruamel.yaml|0.14.12|2|python|amd64|--python-setup-py-arguments --single-version-externally-managed
+all|ruamel.yaml|0.15.77|2|python|amd64|--python-setup-py-arguments --single-version-externally-managed
 all|cwltest|1.0.20180518074130|4|python|all|--depends 'python-futures >= 3.0.5' --depends 'python-subprocess32 >= 3.5.0'
 all|junit-xml|1.8|3|python|all
 all|rdflib-jsonld|0.4.0|2|python|all

diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index e80c06f00d283ef503eb9e1d171636a697ff99cd..8a7fb7f60eba64e86b1474ac9276e6eb411d41ac 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -12,7 +12,7 @@ from schema_salad.sourceline import SourceLine, cmap
 from cwltool.pack import pack
 from cwltool.load_tool import fetch_document
 from cwltool.process import shortname
-from cwltool.workflow import Workflow, WorkflowException
+from cwltool.workflow import Workflow, WorkflowException, WorkflowStep
 from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
 from cwltool.builder import Builder
 from cwltool.context import LoadingContext
@@ -118,6 +118,12 @@ def get_overall_res_req(res_reqs):
             overall_res_req["class"] = "ResourceRequirement"
         return cmap(overall_res_req)
 
+class ArvadosWorkflowStep(WorkflowStep):
+    def job(self, joborder, output_callback, runtimeContext):
+        builder = self._init_job(joborder, runtimeContext)
+        check_cluster_target(self, builder, runtimeContext)
+        return super(ArvadosWorkflowStep, self).job(joborder, output_callback, runtimeContext)
+
 class ArvadosWorkflow(Workflow):
     """Wrap cwltool Workflow to override selected methods."""
 
@@ -282,3 +288,12 @@ class ArvadosWorkflow(Workflow):
             "id": "#"
         })
         return ArvadosCommandTool(self.arvrunner, wf_runner, self.loadingContext).job(joborder_resolved, output_callback, runtimeContext)
+
+    def make_workflow_step(self,
+                           toolpath_object,      # type: Dict[Text, Any]
+                           pos,                  # type: int
+                           loadingContext,       # type: LoadingContext
+                           parentworkflowProv=None  # type: Optional[CreateProvProfile]
+    ):
+        # (...) -> WorkflowStep
+        return WorkflowStep(toolpath_object, pos, loadingContext, parentworkflowProv)

diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index 316a652529b384205661827e2c46d056025d5506..b22b2ffd6f4574ea6a774064d782fbfd07c7a94d 100644 (file)
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -265,6 +265,12 @@ class CollectionFetcher(DefaultFetcher):
 
         return super(CollectionFetcher, self).urljoin(base_url, url)
 
+    schemes = [u"file", u"http", u"https", u"mailto", u"keep"]
+
+    def supported_schemes(self):  # type: () -> List[Text]
+        return self.schemes
+
+
 workflow_uuid_pattern = re.compile(r'[a-z0-9]{5}-7fd4e-[a-z0-9]{15}')
 pipeline_template_uuid_pattern = re.compile(r'[a-z0-9]{5}-p5p6p-[a-z0-9]{15}')
 

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 91a03ed86559e184c4a8dfc927dde28db39c21f4..36aed32171d89f887ec20463d7162852589ad6ac 100644 (file)
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,12 +33,10 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20181114114354',
-          'schema-salad==2.7.20181017120439',
+          'cwltool==1.0.20181116032456',
+          'schema-salad==2.7.20181116024232',
           'typing >= 3.6.4',
-          # Need to limit ruamel.yaml version to 0.15.26 because of bug
-          # https://bitbucket.org/ruamel/yaml/issues/227/regression-parsing-flow-mapping
-          'ruamel.yaml >=0.13.11, <= 0.15.26',
+          'ruamel.yaml >=0.13.11, <= 0.15.77',
           'arvados-python-client>=1.1.4.20180607143841',
           'setuptools',
           'ciso8601 >=1.0.6, <2.0.0',

diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile
index aa1f18052f8afcbe289da18d597b6e66d62d3db6..d33956ccc3f74caa7d6b64958b4c9863f09bbd70 100644 (file)
--- a/sdk/dev-jobs.dockerfile
+++ b/sdk/dev-jobs.dockerfile
@@ -20,7 +20,7 @@ ENV DEBIAN_FRONTEND noninteractive
 
 RUN apt-get update -q && apt-get install -qy git python-pip python-virtualenv python-dev libcurl4-gnutls-dev libgnutls28-dev nodejs python-pyasn1-modules
 
-RUN pip install -U setuptools six
+RUN pip install -U setuptools six requests
 
 ARG sdk
 ARG runner

diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index 8f576196bc4cd623076ed59c4166e4f40a48f369..e5126c12b0f567ba64f251a483e0a1ddf055d8ef 100644 (file)
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -51,7 +51,7 @@ setup(name='arvados-python-client',
           'google-api-python-client >=1.6.2, <1.7',
           'httplib2 >=0.9.2',
           'pycurl >=7.19.5.1',
-          'ruamel.yaml >=0.13.11, <= 0.15.26',
+          'ruamel.yaml >=0.15.54',
           'setuptools',
           'ws4py >=0.4.2',
           'subprocess32 >=3.5.1',