Merge branch 'master' into 10194-cwl-version-skew

Conflicts:
	sdk/cwl/tests/test_job.py

diff --cc sdk/cwl/arvados_cwl/__init__.py
Simple merge

diff --cc sdk/cwl/arvados_cwl/arvjob.py
index c8492e51abab033841ae20cc77dcd9a5b1c3933b,a97453151b0568ab57f79ca5e30e932fa704d1fa..8228387f95cadccb774a534b9552d0350a95c95f
--- 1/sdk/cwl/arvados_cwl/arvjob.py
--- 2/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@@ -257,19 -256,10 +257,19 @@@ class RunnerJob(Runner)
      def run(self, *args, **kwargs):
          job_spec = self.arvados_job_spec(*args, **kwargs)
  
 +        job_spec.setdefault("owner_uuid", self.arvrunner.project_uuid)
 +
 +        job = self.arvrunner.api.jobs().create(
 +            body=job_spec,
 +            find_or_create=self.enable_reuse
 +        ).execute(num_retries=self.arvrunner.num_retries)
 +
          for k,v in job_spec["script_parameters"].items():
-             if isinstance(v, dict):
+             if v is False or v is None or isinstance(v, dict):
                  job_spec["script_parameters"][k] = {"value": v}
  
 +        del job_spec["owner_uuid"]
 +        job_spec["job"] = job
          self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().create(
              body={
                  "owner_uuid": self.arvrunner.project_uuid,

diff --cc sdk/cwl/arvados_cwl/runner.py
Simple merge

diff --cc sdk/cwl/tests/test_job.py
index 2b8240518901bc22168908923cf1c6cdc56922f7,7abc5676cc2f04efd0a74b8bac8271b7abab8b71..93b5d39ffa6eb7e45efba47c64b69fb7a95a33d8
--- 1/sdk/cwl/tests/test_job.py
--- 2/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@@ -220,67 -213,73 +221,77 @@@ class TestWorkflow(unittest.TestCase)
      # The test passes no builder.resources
      # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
      @mock.patch("arvados.collection.Collection")
 -    def test_run(self, mockcollection):
 +    @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 +    def test_run(self, list_images_in_arv, mockcollection):
-         try:
-             arvados_cwl.add_arv_hints()
+         arvados_cwl.add_arv_hints()
  
-             runner = arvados_cwl.ArvCwlRunner(mock.MagicMock())
-             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
-             runner.ignore_docker_for_reuse = False
-             runner.num_retries = 0
-             document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+         api = mock.MagicMock()
+         api._rootDesc = arvados.api('v1')._rootDesc
+         runner = arvados_cwl.ArvCwlRunner(api)
+         self.assertEqual(runner.work_api, 'jobs')
  
-             list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
-             runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
++        list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
++        runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
 +
-             tool, metadata = document_loader.resolve_ref("tests/wf/scatter2.cwl")
-             metadata["cwlVersion"] = tool["cwlVersion"]
+         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+         runner.ignore_docker_for_reuse = False
+         runner.num_retries = 0
+         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  
-             mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
+         tool, metadata = document_loader.resolve_ref("tests/wf/scatter2.cwl")
+         metadata["cwlVersion"] = tool["cwlVersion"]
  
-             make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
-             arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names,
-                                                   basedir="", make_fs_access=make_fs_access, loader=document_loader,
-                                                   makeTool=runner.arv_make_tool, metadata=metadata)
-             arvtool.formatgraph = None
-             it = arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access)
-             it.next().run()
-             it.next().run()
+         mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
  
-             with open("tests/wf/scatter2_subwf.cwl") as f:
-                 subwf = f.read()
+         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
+         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names,
+                                               basedir="", make_fs_access=make_fs_access, loader=document_loader,
+                                               makeTool=runner.arv_make_tool, metadata=metadata)
+         arvtool.formatgraph = None
+         it = arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access)
+         it.next().run()
+         it.next().run()
  
-             mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
-             mockcollection().open().__enter__().write.assert_has_calls([mock.call('{sleeptime: 5}')])
+         with open("tests/wf/scatter2_subwf.cwl") as f:
+             subwf = f.read()
  
-             runner.api.jobs().create.assert_called_with(
-                 body={
-                     'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
-                     'repository': 'arvados',
-                     'script_version': 'master',
-                     'script': 'crunchrunner',
-                     'script_parameters': {
-                         'tasks': [{'task.env': {
-                             'HOME': '$(task.outdir)',
-                             'TMPDIR': '$(task.tmpdir)'},
-                                    'task.vwd': {
-                                        'workflow.cwl': '$(task.keep)/99999999999999999999999999999999+118/workflow.cwl',
-                                        'cwl.input.yml': '$(task.keep)/99999999999999999999999999999999+118/cwl.input.yml'
-                                    },
-                         'command': [u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml'],
-                         'task.stdout': 'cwl.output.json'}]},
-                     'runtime_constraints': {
-                         'min_scratch_mb_per_node': 2048,
-                         'min_cores_per_node': 1,
-                         'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
-                         'min_ram_mb_per_node': 1024
-                     },
-                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'},
-                 filters=[['repository', '=', 'arvados'],
-                          ['script', '=', 'crunchrunner'],
-                          ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
-                          ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]],
-                 find_or_create=True)
-         except:
-             logging.exception("")
 -        mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
 -        mockcollection().open().__enter__().write.assert_has_calls([mock.call('{sleeptime: 5}')])
 -
+         runner.api.jobs().create.assert_called_with(
+             body={
+                 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
+                 'repository': 'arvados',
+                 'script_version': 'master',
+                 'script': 'crunchrunner',
+                 'script_parameters': {
+                     'tasks': [{'task.env': {
+                         'HOME': '$(task.outdir)',
+                         'TMPDIR': '$(task.tmpdir)'},
+                                'task.vwd': {
+                                    'workflow.cwl': '$(task.keep)/99999999999999999999999999999999+118/workflow.cwl',
+                                    'cwl.input.yml': '$(task.keep)/99999999999999999999999999999999+118/cwl.input.yml'
+                                },
+                     'command': [u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml'],
+                     'task.stdout': 'cwl.output.json'}]},
+                 'runtime_constraints': {
+                     'min_scratch_mb_per_node': 2048,
+                     'min_cores_per_node': 1,
 -                    'docker_image': 'arvados/jobs',
++                    'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+                     'min_ram_mb_per_node': 1024
+                 },
+                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'},
+             filters=[['repository', '=', 'arvados'],
+                      ['script', '=', 'crunchrunner'],
+                      ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
 -                     ['docker_image_locator', 'in docker', 'arvados/jobs']],
++                     ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]],
+             find_or_create=True)
+ 
++        mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
++        mockcollection().open().__enter__().write.assert_has_calls([mock.call('{sleeptime: 5}')])
++
+     def test_default_work_api(self):
+         arvados_cwl.add_arv_hints()
+ 
+         api = mock.MagicMock()
+         api._rootDesc = arvados.api('v1')._rootDesc
+         del api._rootDesc.get('resources')['jobs']['methods']['create']
+         runner = arvados_cwl.ArvCwlRunner(api)
+         self.assertEqual(runner.work_api, 'containers')

diff --cc sdk/cwl/tests/test_submit.py
Simple merge