8460: Return recent events if last_log_id given.
[arvados.git] / sdk / cwl / tests / test_submit.py
1 import copy
2 import cStringIO
3 import functools
4 import hashlib
5 import json
6 import logging
7 import mock
8 import sys
9 import unittest
10
11 import arvados
12 import arvados.collection
13 import arvados_cwl
14 import arvados_cwl.runner
15 import arvados.keep
16
17 from .matcher import JsonDiffMatcher
18 from .mock_discovery import get_rootDesc
19
20 _rootDesc = None
21
22 def stubs(func):
23     @functools.wraps(func)
24     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
25     @mock.patch("arvados.collection.KeepClient")
26     @mock.patch("arvados.keep.KeepClient")
27     @mock.patch("arvados.events.subscribe")
28     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
29         class Stubs:
30             pass
31         stubs = Stubs()
32         stubs.events = events
33         stubs.keepdocker = keepdocker
34
35
36         def putstub(p, **kwargs):
37             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
38         keep_client1().put.side_effect = putstub
39         keep_client1.put.side_effect = putstub
40         keep_client2().put.side_effect = putstub
41         keep_client2.put.side_effect = putstub
42
43         stubs.keep_client = keep_client2
44         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
45         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
46
47         stubs.api = mock.MagicMock()
48         stubs.api._rootDesc = get_rootDesc()
49
50         stubs.api.users().current().execute.return_value = {
51             "uuid": stubs.fake_user_uuid,
52         }
53         stubs.api.collections().list().execute.return_value = {"items": []}
54         stubs.api.collections().create().execute.side_effect = ({
55             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
56             "portable_data_hash": "99999999999999999999999999999991+99",
57             "manifest_text": ""
58         }, {
59             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
60             "portable_data_hash": "99999999999999999999999999999992+99",
61             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
62         },
63         {
64             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
65             "portable_data_hash": "99999999999999999999999999999994+99",
66             "manifest_text": ""
67         },
68         {
69             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
70             "portable_data_hash": "99999999999999999999999999999995+99",
71             "manifest_text": ""
72         },
73         {
74             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
75             "portable_data_hash": "99999999999999999999999999999996+99",
76             "manifest_text": ""
77         }
78         )
79         stubs.api.collections().get().execute.return_value = {
80             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
81
82         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
83         stubs.api.jobs().create().execute.return_value = {
84             "uuid": stubs.expect_job_uuid,
85             "state": "Queued",
86         }
87
88         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
89         stubs.api.container_requests().create().execute.return_value = {
90             "uuid": stubs.expect_container_request_uuid,
91             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
92             "state": "Queued"
93         }
94
95         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
96         stubs.api.pipeline_templates().create().execute.return_value = {
97             "uuid": stubs.expect_pipeline_template_uuid,
98         }
99         stubs.expect_job_spec = {
100             'runtime_constraints': {
101                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__
102             },
103             'script_parameters': {
104                 'x': {
105                     'basename': 'blorp.txt',
106                     'location': 'keep:99999999999999999999999999999994+99/blorp.txt',
107                     'class': 'File'
108                 },
109                 'y': {
110                     'basename': '99999999999999999999999999999998+99',
111                     'location': 'keep:99999999999999999999999999999998+99',
112                     'class': 'Directory'
113                 },
114                 'z': {
115                     'basename': 'anonymous',
116                     "listing": [{
117                         "basename": "renamed.txt",
118                         "class": "File",
119                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
120                     }],
121                     'class': 'Directory'
122                 },
123                 'cwl:tool':
124                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
125             },
126             'repository': 'arvados',
127             'script_version': arvados_cwl.__version__,
128             'script': 'cwl-runner'
129         }
130         stubs.pipeline_component = stubs.expect_job_spec.copy()
131         stubs.expect_pipeline_instance = {
132             'name': 'submit_wf.cwl',
133             'state': 'RunningOnServer',
134             "components": {
135                 "cwl-runner": {
136                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__},
137                     'script_parameters': {
138                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
139                         'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999994+99/blorp.txt'}},
140                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
141                               'listing': [
142                                   {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
143                               ]}},
144                         'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl',
145                         'arv:enable_reuse': True
146                     },
147                     'repository': 'arvados',
148                     'script_version': arvados_cwl.__version__,
149                     'script': 'cwl-runner',
150                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
151                 }
152             }
153         }
154         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
155         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
156         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
157         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
158         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
159             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
160             "state": "Queued"
161         }
162         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
163         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
164
165         stubs.expect_container_spec = {
166             'priority': 1,
167             'mounts': {
168                 '/var/spool/cwl': {
169                     'writable': True,
170                     'kind': 'collection'
171                 },
172                 '/var/lib/cwl/workflow': {
173                     'portable_data_hash': '99999999999999999999999999999991+99',
174                     'kind': 'collection'
175                 },
176                 'stdout': {
177                     'path': '/var/spool/cwl/cwl.output.json',
178                     'kind': 'file'
179                 },
180                 '/var/lib/cwl/job/cwl.input.json': {
181                     'portable_data_hash': 'd20d7cddd1984f105dd3702c7f125afb+60/cwl.input.json',
182                     'kind': 'collection'
183                 }
184             },
185             'state': 'Committed',
186             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
187             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--enable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
188             'name': 'submit_wf.cwl',
189             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
190             'output_path': '/var/spool/cwl',
191             'cwd': '/var/spool/cwl',
192             'runtime_constraints': {
193                 'API': True,
194                 'vcpus': 1,
195                 'ram': 268435456
196             }
197         }
198
199         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
200         stubs.api.workflows().create().execute.return_value = {
201             "uuid": stubs.expect_workflow_uuid,
202         }
203
204         return func(self, stubs, *args, **kwargs)
205     return wrapped
206
207
208 class TestSubmit(unittest.TestCase):
209     @mock.patch("time.sleep")
210     @stubs
211     def test_submit(self, stubs, tm):
212         capture_stdout = cStringIO.StringIO()
213         exited = arvados_cwl.main(
214             ["--submit", "--no-wait", "--debug",
215              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
216             capture_stdout, sys.stderr, api_client=stubs.api)
217         self.assertEqual(exited, 0)
218
219         stubs.api.collections().create.assert_has_calls([
220             mock.call(),
221             mock.call(body={
222                 'manifest_text':
223                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
224                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
225                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
226                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
227                 'name': 'submit_wf.cwl',
228             }, ensure_unique_name=True),
229             mock.call().execute(),
230             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
231                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
232                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
233                             'replication_desired': None,
234                             'name': 'New collection'
235             }, ensure_unique_name=True),
236             mock.call().execute(num_retries=4),
237             mock.call(body={
238                 'manifest_text':
239                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
240                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
241                 'name': '#',
242             }, ensure_unique_name=True),
243             mock.call().execute()])
244
245         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
246         expect_pipeline["owner_uuid"] = stubs.fake_user_uuid
247         stubs.api.pipeline_instances().create.assert_called_with(
248             body=expect_pipeline)
249         self.assertEqual(capture_stdout.getvalue(),
250                          stubs.expect_pipeline_uuid + '\n')
251
252
253     @mock.patch("time.sleep")
254     @stubs
255     def test_submit_no_reuse(self, stubs, tm):
256         capture_stdout = cStringIO.StringIO()
257         exited = arvados_cwl.main(
258             ["--submit", "--no-wait", "--debug", "--disable-reuse",
259              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
260             capture_stdout, sys.stderr, api_client=stubs.api)
261         self.assertEqual(exited, 0)
262
263         stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
264
265         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
266         expect_pipeline["owner_uuid"] = stubs.fake_user_uuid
267         stubs.api.pipeline_instances().create.assert_called_with(
268             body=expect_pipeline)
269         self.assertEqual(capture_stdout.getvalue(),
270                          stubs.expect_pipeline_uuid + '\n')
271
272     @mock.patch("time.sleep")
273     @stubs
274     def test_submit_with_project_uuid(self, stubs, tm):
275         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
276
277         exited = arvados_cwl.main(
278             ["--submit", "--no-wait",
279              "--project-uuid", project_uuid,
280              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
281             sys.stdout, sys.stderr, api_client=stubs.api)
282         self.assertEqual(exited, 0)
283
284         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
285         expect_pipeline["owner_uuid"] = project_uuid
286         stubs.api.pipeline_instances().create.assert_called_with(
287             body=expect_pipeline)
288
289     @stubs
290     def test_submit_container(self, stubs):
291         capture_stdout = cStringIO.StringIO()
292         try:
293             exited = arvados_cwl.main(
294                 ["--submit", "--no-wait", "--api=containers", "--debug",
295                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
296                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
297             self.assertEqual(exited, 0)
298         except:
299             logging.exception("")
300
301         stubs.api.collections().create.assert_has_calls([
302             mock.call(),
303             mock.call(body={
304                 'manifest_text':
305                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
306                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
307                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
308                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
309                 'name': 'submit_wf.cwl',
310             }, ensure_unique_name=True),
311             mock.call().execute(),
312             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
313                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
314                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
315                             'name': 'New collection',
316                             'replication_desired': None,
317             }, ensure_unique_name=True),
318             mock.call().execute(num_retries=4),
319             mock.call(body={
320                 'manifest_text':
321                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
322                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
323                 'name': '#',
324             }, ensure_unique_name=True),
325             mock.call().execute()])
326
327         expect_container = copy.deepcopy(stubs.expect_container_spec)
328         expect_container["owner_uuid"] = stubs.fake_user_uuid
329         stubs.api.container_requests().create.assert_called_with(
330             body=expect_container)
331         self.assertEqual(capture_stdout.getvalue(),
332                          stubs.expect_container_request_uuid + '\n')
333
334     @stubs
335     def test_submit_container_no_reuse(self, stubs):
336         capture_stdout = cStringIO.StringIO()
337         try:
338             exited = arvados_cwl.main(
339                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
340                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
341                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
342             self.assertEqual(exited, 0)
343         except:
344             logging.exception("")
345
346         stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--disable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json']
347
348         expect_container = copy.deepcopy(stubs.expect_container_spec)
349         expect_container["owner_uuid"] = stubs.fake_user_uuid
350         stubs.api.container_requests().create.assert_called_with(
351             body=expect_container)
352         self.assertEqual(capture_stdout.getvalue(),
353                          stubs.expect_container_request_uuid + '\n')
354
355     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
356     @mock.patch("cwltool.docker.get_image")
357     @mock.patch("arvados.api")
358     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
359         arvrunner = mock.MagicMock()
360         arvrunner.project_uuid = ""
361         api.return_value = mock.MagicMock()
362         arvrunner.api = api.return_value
363         arvrunner.api.links().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
364                                                             {"items": [], "items_available": 0, "offset": 0},
365                                                             {"items": [], "items_available": 0, "offset": 0},
366                                                             {"items": [{"created_at": "",
367                                                                         "head_uuid": "",
368                                                                         "link_class": "docker_image_hash",
369                                                                         "name": "123456",
370                                                                         "owner_uuid": "",
371                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
372                                                             {"items": [], "items_available": 0, "offset": 0},
373                                                             {"items": [{"created_at": "",
374                                                                         "head_uuid": "",
375                                                                         "link_class": "docker_image_repo+tag",
376                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
377                                                                         "owner_uuid": "",
378                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
379                                                             {"items": [{"created_at": "",
380                                                                         "head_uuid": "",
381                                                                         "link_class": "docker_image_hash",
382                                                                         "name": "123456",
383                                                                         "owner_uuid": "",
384                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}                                                            ,
385         )
386         find_one_image_hash.return_value = "123456"
387
388         arvrunner.api.collections().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
389                                                                   {"items": [{"uuid": "",
390                                                                               "owner_uuid": "",
391                                                                               "manifest_text": "",
392                                                                               "properties": ""
393                                                                           }], "items_available": 1, "offset": 0},
394                                                                   {"items": [{"uuid": ""}], "items_available": 1, "offset": 0})
395         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
396         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner))
397
398 class TestCreateTemplate(unittest.TestCase):
399     @stubs
400     def test_create(self, stubs):
401         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
402
403         capture_stdout = cStringIO.StringIO()
404
405         exited = arvados_cwl.main(
406             ["--create-template", "--debug",
407              "--project-uuid", project_uuid,
408              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
409             capture_stdout, sys.stderr, api_client=stubs.api)
410         self.assertEqual(exited, 0)
411
412         stubs.api.pipeline_instances().create.refute_called()
413         stubs.api.jobs().create.refute_called()
414
415         expect_component = copy.deepcopy(stubs.expect_job_spec)
416         expect_component['script_parameters']['x'] = {
417             'dataclass': 'File',
418             'required': True,
419             'type': 'File',
420             'value': '99999999999999999999999999999994+99/blorp.txt',
421         }
422         expect_component['script_parameters']['y'] = {
423             'dataclass': 'Collection',
424             'required': True,
425             'type': 'Directory',
426             'value': '99999999999999999999999999999998+99',
427         }
428         expect_component['script_parameters']['z'] = {
429             'dataclass': 'Collection',
430             'required': True,
431             'type': 'Directory',
432         }
433         expect_template = {
434             "components": {
435                 "submit_wf.cwl": expect_component,
436             },
437             "name": "submit_wf.cwl",
438             "owner_uuid": project_uuid,
439         }
440         stubs.api.pipeline_templates().create.assert_called_with(
441             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
442
443         self.assertEqual(capture_stdout.getvalue(),
444                          stubs.expect_pipeline_template_uuid + '\n')
445
446
447 class TestCreateWorkflow(unittest.TestCase):
448     @stubs
449     def test_create(self, stubs):
450         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
451
452         capture_stdout = cStringIO.StringIO()
453
454         exited = arvados_cwl.main(
455             ["--create-workflow", "--debug",
456              "--project-uuid", project_uuid,
457              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
458             capture_stdout, sys.stderr, api_client=stubs.api)
459         self.assertEqual(exited, 0)
460
461         stubs.api.pipeline_templates().create.refute_called()
462         stubs.api.container_requests().create.refute_called()
463
464         with open("tests/wf/expect_packed.cwl") as f:
465             expect_workflow = f.read()
466
467         body = {
468             "workflow": {
469                 "owner_uuid": project_uuid,
470                 "name": "submit_wf.cwl",
471                 "description": "",
472                 "definition": expect_workflow
473                 }
474         }
475         stubs.api.workflows().create.assert_called_with(
476             body=JsonDiffMatcher(body))
477
478         self.assertEqual(capture_stdout.getvalue(),
479                          stubs.expect_workflow_uuid + '\n')
480
481
482 class TestTemplateInputs(unittest.TestCase):
483     expect_template = {
484         "components": {
485             "inputs_test.cwl": {
486                 'runtime_constraints': {
487                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
488                 },
489                 'script_parameters': {
490                     'cwl:tool':
491                     '99999999999999999999999999999991+99/'
492                     'wf/inputs_test.cwl',
493                     'optionalFloatInput': None,
494                     'fileInput': {
495                         'type': 'File',
496                         'dataclass': 'File',
497                         'required': True,
498                         'title': "It's a file; we expect to find some characters in it.",
499                         'description': 'If there were anything further to say, it would be said here,\nor here.'
500                     },
501                     'floatInput': {
502                         'type': 'float',
503                         'dataclass': 'number',
504                         'required': True,
505                         'title': 'Floats like a duck',
506                         'default': 0.1,
507                         'value': 0.1,
508                     },
509                     'optionalFloatInput': {
510                         'type': ['null', 'float'],
511                         'dataclass': 'number',
512                         'required': False,
513                     },
514                     'boolInput': {
515                         'type': 'boolean',
516                         'dataclass': 'boolean',
517                         'required': True,
518                         'title': 'True or false?',
519                     },
520                 },
521                 'repository': 'arvados',
522                 'script_version': arvados_cwl.__version__,
523                 'script': 'cwl-runner',
524             },
525         },
526         "name": "inputs_test.cwl",
527     }
528
529     @stubs
530     def test_inputs_empty(self, stubs):
531         exited = arvados_cwl.main(
532             ["--create-template", "--no-wait",
533              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
534             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
535         self.assertEqual(exited, 0)
536
537         expect_template = copy.deepcopy(self.expect_template)
538         expect_template["owner_uuid"] = stubs.fake_user_uuid
539
540         stubs.api.pipeline_templates().create.assert_called_with(
541             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
542
543     @stubs
544     def test_inputs(self, stubs):
545         exited = arvados_cwl.main(
546             ["--create-template", "--no-wait",
547              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
548             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
549         self.assertEqual(exited, 0)
550
551         self.expect_template["owner_uuid"] = stubs.fake_user_uuid
552
553         expect_template = copy.deepcopy(self.expect_template)
554         expect_template["owner_uuid"] = stubs.fake_user_uuid
555         params = expect_template[
556             "components"]["inputs_test.cwl"]["script_parameters"]
557         params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
558         params["floatInput"]["value"] = 1.234
559         params["boolInput"]["value"] = True
560
561         stubs.api.pipeline_templates().create.assert_called_with(
562             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)