Merge branch 'master' into 10576-cwl-keep-fetcher

Conflicts:
	sdk/cwl/arvados_cwl/runner.py
	sdk/cwl/tests/test_submit.py

diff --cc sdk/cwl/arvados_cwl/__init__.py
Simple merge

diff --cc sdk/cwl/arvados_cwl/arvjob.py
Simple merge

diff --cc sdk/cwl/arvados_cwl/runner.py
index 1152b7a632fbdfae2192fe99e47ca90603daa0ae,2d13e6640b1c57b3ccfb22c9c3c9bac79b6ecda8..dea47567a767b34dd0252781a1d1031b9990deec
--- 1/sdk/cwl/arvados_cwl/runner.py
--- 2/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@@ -184,13 -187,8 +187,14 @@@ class Runner(object)
          pass
  
      def arvados_job_spec(self, *args, **kwargs):
-         self.name = self.tool.tool.get("label") or os.path.basename(self.tool.tool["id"])
+         if self.name is None:
 -            self.name = os.path.basename(self.tool.tool["id"])
++            self.name = self.tool.tool.get("label") or os.path.basename(self.tool.tool["id"])
 +
 +        # Need to filter this out, gets added by cwltool when providing
 +        # parameters on the command line.
 +        if "job_order" in self.job_order:
 +            del self.job_order["job_order"]
 +
          workflowmapper = upload_instance(self.arvrunner, self.name, self.tool, self.job_order)
          adjustDirObjs(self.job_order, trim_listing)
          return workflowmapper

diff --cc sdk/cwl/tests/test_submit.py
index f082c7149fa52f4fb65232b183d4356c439154fb,d917aef57a8ea2378b43193e7ff01f066a73f1d2..4418ee3fd153ac1d3ddf92a798ce7a0b28ace361
--- 1/sdk/cwl/tests/test_submit.py
--- 2/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@@ -497,165 -508,26 +516,185 @@@ class TestSubmit(unittest.TestCase)
          self.assertEqual(capture_stdout.getvalue(),
                           stubs.expect_container_request_uuid + '\n')
  
- 
 +    @mock.patch("arvados.collection.CollectionReader")
 +    @mock.patch("time.sleep")
 +    @stubs
 +    def test_submit_file_keepref(self, stubs, tm, collectionReader):
 +        capture_stdout = cStringIO.StringIO()
 +        exited = arvados_cwl.main(
 +            ["--submit", "--no-wait", "--api=containers", "--debug",
 +             "tests/wf/submit_keepref_wf.cwl"],
 +            capture_stdout, sys.stderr, api_client=stubs.api)
 +        self.assertEqual(exited, 0)
 +
 +
 +    @mock.patch("arvados.collection.CollectionReader")
 +    @mock.patch("time.sleep")
 +    @stubs
 +    def test_submit_keepref(self, stubs, tm, reader):
 +        capture_stdout = cStringIO.StringIO()
 +
 +        with open("tests/wf/expect_arvworkflow.cwl") as f:
 +            reader().open().__enter__().read.return_value = f.read()
 +
 +        exited = arvados_cwl.main(
 +            ["--submit", "--no-wait", "--api=containers", "--debug",
 +             "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 +            capture_stdout, sys.stderr, api_client=stubs.api)
 +        self.assertEqual(exited, 0)
 +
 +        expect_container = {
 +            'priority': 1,
 +            'mounts': {
 +                '/var/spool/cwl': {
 +                    'writable': True,
 +                    'kind': 'collection'
 +                },
 +                'stdout': {
 +                    'path': '/var/spool/cwl/cwl.output.json',
 +                    'kind': 'file'
 +                },
 +                '/var/lib/cwl/workflow': {
 +                    'portable_data_hash': '99999999999999999999999999999994+99',
 +                    'kind': 'collection'
 +                },
 +                '/var/lib/cwl/cwl.input.json': {
 +                    'content': {
 +                        'x': 'XxX'
 +                    },
 +                    'kind': 'json'
 +                }
 +            }, 'state': 'Committed',
 +            'owner_uuid': None,
 +            'output_path': '/var/spool/cwl',
 +            'name': 'expect_arvworkflow.cwl#main',
 +            'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 +            'command': ['arvados-cwl-runner', '--local', '--api=containers', '--enable-reuse', '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 +            'cwd': '/var/spool/cwl',
 +            'runtime_constraints': {
 +                'API': True,
 +                'vcpus': 1,
 +                'ram': 1073741824
 +            },
 +            "properties": {}
 +        }
 +
 +        stubs.api.container_requests().create.assert_called_with(
 +            body=JsonDiffMatcher(expect_container))
 +        self.assertEqual(capture_stdout.getvalue(),
 +                         stubs.expect_container_request_uuid + '\n')
 +
 +    @mock.patch("time.sleep")
 +    @stubs
 +    def test_submit_arvworkflow(self, stubs, tm):
 +        capture_stdout = cStringIO.StringIO()
 +
 +        with open("tests/wf/expect_arvworkflow.cwl") as f:
 +            stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 +
 +        exited = arvados_cwl.main(
 +            ["--submit", "--no-wait", "--api=containers", "--debug",
 +             "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 +            capture_stdout, sys.stderr, api_client=stubs.api)
 +        self.assertEqual(exited, 0)
 +
 +        expect_container = {
 +            'priority': 1,
 +            'mounts': {
 +                '/var/spool/cwl': {
 +                    'writable': True,
 +                    'kind': 'collection'
 +                },
 +                'stdout': {
 +                    'path': '/var/spool/cwl/cwl.output.json',
 +                    'kind': 'file'
 +                },
 +                '/var/lib/cwl/workflow.json': {
 +                    'kind': 'json',
 +                    'json': {
 +                        'cwlVersion': 'v1.0',
 +                        '$graph': [
 +                            {
 +                                'inputs': [
 +                                    {
 +                                        'inputBinding': {'position': 1},
 +                                        'type': 'string',
 +                                        'id': '#submit_tool.cwl/x'}
 +                                ],
 +                                'requirements': [
 +                                    {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
 +                                ],
 +                                'id': '#submit_tool.cwl',
 +                                'outputs': [],
 +                                'baseCommand': 'cat',
 +                                'class': 'CommandLineTool'
 +                            }, {
 +                                'id': '#main',
 +                                'inputs': [
 +                                    {'type': 'string', 'id': '#main/x'}
 +                                ],
 +                                'steps': [
 +                                    {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 +                                     'run': '#submit_tool.cwl',
 +                                     'id': '#main/step1',
 +                                     'out': []}
 +                                ],
 +                                'class': 'Workflow',
 +                                'outputs': []
 +                            }
 +                        ]
 +                    }
 +                },
 +                '/var/lib/cwl/cwl.input.json': {
 +                    'content': {
 +                        'x': 'XxX'
 +                    },
 +                    'kind': 'json'
 +                }
 +            }, 'state': 'Committed',
 +            'owner_uuid': None,
 +            'output_path': '/var/spool/cwl',
 +            'name': 'a test workflow',
 +            'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 +            'command': ['arvados-cwl-runner', '--local', '--api=containers', '--enable-reuse', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 +            'cwd': '/var/spool/cwl',
 +            'runtime_constraints': {
 +                'API': True,
 +                'vcpus': 1,
 +                'ram': 1073741824
 +            },
 +            "properties": {
 +                "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 +            }
 +        }
 +
 +        stubs.api.container_requests().create.assert_called_with(
 +            body=JsonDiffMatcher(expect_container))
 +        self.assertEqual(capture_stdout.getvalue(),
 +                         stubs.expect_container_request_uuid + '\n')
 +
 +
+     @stubs
+     def test_submit_container_name(self, stubs):
+         capture_stdout = cStringIO.StringIO()
+         try:
+             exited = arvados_cwl.main(
+                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
+                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+             self.assertEqual(exited, 0)
+         except:
+             logging.exception("")
+ 
+         stubs.expect_container_spec["name"] = "hello container 123"
+ 
+         expect_container = copy.deepcopy(stubs.expect_container_spec)
+         stubs.api.container_requests().create.assert_called_with(
+             body=expect_container)
+         self.assertEqual(capture_stdout.getvalue(),
+                          stubs.expect_container_request_uuid + '\n')
+ 
++
      @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
      @mock.patch("cwltool.docker.get_image")
      @mock.patch("arvados.api")