Merge branch '11968-ws-startup-crash'

closes #11968

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curoverse.com>

diff --git a/AUTHORS b/AUTHORS
index ad72aea0aa98486b498465b51e4f1cfd9211658a..ea9fa4c7092e8c2069a2105d8eafb25e6107d3ab 100644 (file)
--- a/AUTHORS
+++ b/AUTHORS
@@ -16,3 +16,4 @@ Guillermo Carrasco <guille.ch.88@gmail.com>
 Joshua Randall <joshua.randall@sanger.ac.uk>
 President and Fellows of Harvard College <*@harvard.edu>
 Thomas Mooney <tmooney@genome.wustl.edu>
+Chen Chen <aflyhorse@gmail.com>

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index d642cbd243a2727c7eb65f87562714428c575601..695597f839c1ceee5ccbb67f3391218d3d1a8e34 100644 (file)
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -89,7 +89,7 @@ class ArvCwlRunner(object):
         self.collection_cache = CollectionCache(self.api, self.keep_client, self.num_retries)
 
         self.work_api = None
-        expected_api = ["containers", "jobs"]
+        expected_api = ["jobs", "containers"]
         for api in expected_api:
             try:
                 methods = self.api._rootDesc.get('resources')[api]['methods']
@@ -426,12 +426,6 @@ class ArvCwlRunner(object):
             if self.work_api == "containers":
                 if tool.tool["class"] == "CommandLineTool" and kwargs.get("wait"):
                     kwargs["runnerjob"] = tool.tool["id"]
-                    upload_dependencies(self,
-                                        kwargs["name"],
-                                        tool.doc_loader,
-                                        tool.tool,
-                                        tool.tool["id"],
-                                        False)
                     runnerjob = tool.job(job_order,
                                          self.output_callback,
                                          **kwargs).next()
@@ -632,7 +626,7 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     parser.add_argument("--api", type=str,
                         default=None, dest="work_api",
                         choices=("jobs", "containers"),
-                        help="Select work submission API.  Default is 'containers' if that API is available, otherwise 'jobs'.")
+                        help="Select work submission API.  Default is 'jobs' if that API is available, otherwise 'containers'.")
 
     parser.add_argument("--compute-checksum", action="store_true", default=False,
                         help="Compute checksum of contents while collecting outputs",
@@ -756,6 +750,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None):
     arvargs.use_container = True
     arvargs.relax_path_checks = True
     arvargs.validate = None
+    arvargs.print_supported_versions = False
 
     make_fs_access = partial(CollectionFsAccess,
                            collection_cache=runner.collection_cache)

diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index 8ab86c173f42ba16cc28699cba9495ec48e8281b..35a068f91be7b28b7eebd0f32538e9222367d8a1 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -15,7 +15,7 @@ class ArvadosCommandTool(CommandLineTool):
         self.arvrunner = arvrunner
         self.work_api = kwargs["work_api"]
 
-    def makeJobRunner(self):
+    def makeJobRunner(self, use_container=True):
         if self.work_api == "containers":
             return ArvadosContainer(self.arvrunner)
         elif self.work_api == "jobs":

diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index 5582d63cfc6362acd0477f7b5508346772d1d6d3..93e2819084601d784a973c33ca30bbbdb2d6db49 100644 (file)
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -95,7 +95,7 @@ class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
 
     def exists(self, fn):
         collection, rest = self.get_collection(fn)
-        if collection:
+        if collection is not None:
             if rest:
                 return collection.exists(rest)
             else:

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 683f548c48cc835cd5ff7664cd0306cb0ea7f17e..087fed3e16e72cb26c95500b4ccb03a83bf71806 100644 (file)
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -18,7 +18,7 @@ from cwltool.draft2tool import CommandLineTool
 import cwltool.workflow
 from cwltool.process import get_feature, scandeps, UnsupportedRequirement, normalizeFilesDirs, shortname
 from cwltool.load_tool import fetch_document
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs
+from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
 from cwltool.utils import aslist
 from cwltool.builder import substitute
 from cwltool.pack import pack
@@ -46,6 +46,17 @@ def trim_anonymous_location(obj):
     if obj.get("location", "").startswith("_:"):
         del obj["location"]
 
+def find_defaults(d, op):
+    if isinstance(d, list):
+        for i in d:
+            find_defaults(i, op)
+    elif isinstance(d, dict):
+        if "default" in d:
+            op(d)
+        else:
+            for i in d.itervalues():
+                find_defaults(i, op)
+
 def upload_dependencies(arvrunner, name, document_loader,
                         workflowobj, uri, loadref_run, include_primary=True):
     """Upload the dependencies of the workflowobj document to Keep.
@@ -101,6 +112,23 @@ def upload_dependencies(arvrunner, name, document_loader,
         for s in workflowobj["$schemas"]:
             sc.append({"class": "File", "location": s})
 
+    def capture_default(obj):
+        remove = [False]
+        def add_default(f):
+            if "location" not in f and "path" in f:
+                f["location"] = f["path"]
+                del f["path"]
+            if "location" in f and not arvrunner.fs_access.exists(f["location"]):
+                # Remove from sc
+                sc[:] = [x for x in sc if x["location"] != f["location"]]
+                # Delete "default" from workflowobj
+                remove[0] = True
+        visit_class(obj["default"], ("File", "Directory"), add_default)
+        if remove[0]:
+            del obj["default"]
+
+    find_defaults(workflowobj, capture_default)
+
     mapper = ArvPathMapper(arvrunner, sc, "",
                            "keep:%s",
                            "keep:%s/%s",

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 55987ca1fc1796fb5f84215bdf3d60b5bdf00544..375485e62735effea3c935fde640b6db5dbc170e 100644 (file)
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -51,8 +51,8 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20170525215327',
-          'schema-salad==2.5.20170428142041',
+          'cwltool==1.0.20170707200431',
+          'schema-salad==2.6.20170630075932',
           'typing==3.5.3.0',
           'ruamel.yaml==0.13.7',
           'arvados-python-client>=0.1.20170526013812',

diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 89f4677e3b34f84ad4b5b0601f22fa2c9bc6d286..5d140ba6b9d56a052f3a75eb0f4697f29b8dad79 100644 (file)
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -315,7 +315,7 @@ class TestWorkflow(unittest.TestCase):
         api._rootDesc = get_rootDesc()
 
         runner = arvados_cwl.ArvCwlRunner(api)
-        runner.work_api = 'jobs'
+        self.assertEqual(runner.work_api, 'jobs')
 
         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
         runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}

diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index eec37ef62aeeaf3adbf6f06d52f6c3da1024ac1a..3d6b91536a2f2732ad2e97d5c172bf41e2ee60e7 100644 (file)
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -79,7 +79,12 @@ def stubs(func):
                 if uuid in (v["uuid"], v["portable_data_hash"]):
                     return CollectionExecute(v)
 
-        created_collections = {}
+        created_collections = {
+            "99999999999999999999999999999998+99": {
+                "uuid": "",
+                "portable_data_hash": "99999999999999999999999999999998+99",
+                "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
+            }}
         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 
@@ -142,10 +147,22 @@ def stubs(func):
                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
                     'script_parameters': {
                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
-                        'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'}},
+                        'x': {"value": {
+                            'basename': 'blorp.txt',
+                            'class': 'File',
+                            'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
+                            "nameext": ".txt",
+                            "nameroot": "blorp"
+                        }},
                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
                               'listing': [
-                                  {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
+                                  {
+                                      'basename': 'renamed.txt',
+                                      'class': 'File', 'location':
+                                      'keep:99999999999999999999999999999998+99/file1.txt',
+                                      "nameext": ".txt",
+                                      "nameroot": "renamed"
+                                  }
                               ]}},
                         'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
                         'arv:enable_reuse': True,
@@ -191,10 +208,24 @@ def stubs(func):
                 '/var/lib/cwl/cwl.input.json': {
                     'kind': 'json',
                     'content': {
-                        'y': {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'},
-                        'x': {'basename': u'blorp.txt', 'class': 'File', 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'},
+                        'y': {
+                            'basename': '99999999999999999999999999999998+99',
+                            'location': 'keep:99999999999999999999999999999998+99',
+                            'class': 'Directory'},
+                        'x': {
+                            'basename': u'blorp.txt',
+                            'class': 'File',
+                            'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
+                            "nameext": ".txt",
+                            "nameroot": "blorp"
+                        },
                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
-                            {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
+                            {'basename': 'renamed.txt',
+                             'class': 'File',
+                             'location': 'keep:99999999999999999999999999999998+99/file1.txt',
+                             "nameext": ".txt",
+                             "nameroot": "renamed"
+                            }
                         ]}
                     },
                     'kind': 'json'
@@ -319,7 +350,7 @@ class TestSubmit(unittest.TestCase):
     def test_submit_runner_ram(self, stubs, tm):
         capture_stdout = cStringIO.StringIO()
         exited = arvados_cwl.main(
-            ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-ram=2048",
+            ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
             capture_stdout, sys.stderr, api_client=stubs.api)
         self.assertEqual(exited, 0)
@@ -338,7 +369,7 @@ class TestSubmit(unittest.TestCase):
     def test_submit_invalid_runner_ram(self, stubs, tm):
         capture_stdout = cStringIO.StringIO()
         exited = arvados_cwl.main(
-            ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-ram=-2048",
+            ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
             capture_stdout, sys.stderr, api_client=stubs.api)
         self.assertEqual(exited, 1)
@@ -350,7 +381,7 @@ class TestSubmit(unittest.TestCase):
 
         capture_stdout = cStringIO.StringIO()
         exited = arvados_cwl.main(
-            ["--submit", "--no-wait", "--api=jobs", "--debug", "--output-name", output_name,
+            ["--submit", "--no-wait", "--debug", "--output-name", output_name,
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
             capture_stdout, sys.stderr, api_client=stubs.api)
         self.assertEqual(exited, 0)
@@ -369,7 +400,7 @@ class TestSubmit(unittest.TestCase):
     def test_submit_pipeline_name(self, stubs, tm):
         capture_stdout = cStringIO.StringIO()
         exited = arvados_cwl.main(
-            ["--submit", "--no-wait", "--api=jobs", "--debug", "--name=hello job 123",
+            ["--submit", "--no-wait", "--debug", "--name=hello job 123",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
             capture_stdout, sys.stderr, api_client=stubs.api)
         self.assertEqual(exited, 0)
@@ -389,7 +420,7 @@ class TestSubmit(unittest.TestCase):
 
         capture_stdout = cStringIO.StringIO()
         exited = arvados_cwl.main(
-            ["--submit", "--no-wait", "--api=jobs", "--debug", "--output-tags", output_tags,
+            ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
             capture_stdout, sys.stderr, api_client=stubs.api)
         self.assertEqual(exited, 0)
@@ -408,7 +439,7 @@ class TestSubmit(unittest.TestCase):
         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 
         exited = arvados_cwl.main(
-            ["--submit", "--no-wait", "--api=jobs",
+            ["--submit", "--no-wait",
              "--project-uuid", project_uuid,
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
             sys.stdout, sys.stderr, api_client=stubs.api)
@@ -424,7 +455,7 @@ class TestSubmit(unittest.TestCase):
         capture_stdout = cStringIO.StringIO()
         try:
             exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--debug",
+                ["--submit", "--no-wait", "--api=containers", "--debug",
                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
             self.assertEqual(exited, 0)
@@ -895,7 +926,7 @@ class TestSubmit(unittest.TestCase):
                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
 
 class TestCreateTemplate(unittest.TestCase):
-    existing_template_uuid = "zzzzz-p5p6p-validworkfloyml"
+    existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
 
     def _adjust_script_params(self, expect_component):
         expect_component['script_parameters']['x'] = {
@@ -1237,7 +1268,7 @@ class TestTemplateInputs(unittest.TestCase):
     @stubs
     def test_inputs_empty(self, stubs):
         exited = arvados_cwl.main(
-            ["--create-template", "--api=jobs",
+            ["--create-template",
              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
         self.assertEqual(exited, 0)
@@ -1248,7 +1279,7 @@ class TestTemplateInputs(unittest.TestCase):
     @stubs
     def test_inputs(self, stubs):
         exited = arvados_cwl.main(
-            ["--create-template", "--api=jobs",
+            ["--create-template",
              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
         self.assertEqual(exited, 0)

diff --git a/sdk/cwl/tests/wf/expect_packed.cwl b/sdk/cwl/tests/wf/expect_packed.cwl
index 561daf40ff186841dd28645373877aac89ba7d63..32a9255e90d442a475394036ea54dd55a797e6fa 100644 (file)
--- a/sdk/cwl/tests/wf/expect_packed.cwl
+++ b/sdk/cwl/tests/wf/expect_packed.cwl
@@ -24,7 +24,7 @@ $graph:
   - id: '#main/x'
     type: File
     default: {class: File, location: 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
-      basename: blorp.txt}
+      basename: blorp.txt, nameroot: blorp, nameext: .txt}
   - id: '#main/y'
     type: Directory
     default: {class: Directory, location: 'keep:99999999999999999999999999999998+99',
@@ -32,7 +32,8 @@ $graph:
   - id: '#main/z'
     type: Directory
     default: {class: Directory, basename: anonymous, listing: [{basename: renamed.txt,
-          class: File, location: 'keep:99999999999999999999999999999998+99/file1.txt'}]}
+          class: File, location: 'keep:99999999999999999999999999999998+99/file1.txt',
+          nameroot: renamed, nameext: .txt}]}
   outputs: []
   steps:
   - id: '#main/step1'