projects / arvados.git / blobdiff
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
Merge branch '2411-check-copyright'
[arvados.git] / sdk / cwl / tests / test_container.py
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index b06eae8105aad06d35ece8611f3b7a5103ab838c..c516d7b35340b15ac1f4eaac1a7fcbc645b95eda 100644 (file)
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
 import arvados_cwl
 from arvados_cwl.arvdocker import arv_docker_clear_cache
 import logging
@@ -28,6 +32,7 @@ class TestContainer(unittest.TestCase):
             runner = mock.MagicMock()
             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
             runner.ignore_docker_for_reuse = False
+            runner.intermediate_output_ttl = 0
 
             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
             runner.api.collections().get().execute.return_value = {
@@ -63,12 +68,16 @@ class TestContainer(unittest.TestCase):
                         'use_existing': enable_reuse,
                         'priority': 1,
                         'mounts': {
-                            '/tmp': {'kind': 'tmp'},
-                            '/var/spool/cwl': {'kind': 'tmp'}
+                            '/tmp': {'kind': 'tmp',
+                                     "capacity": 1073741824
+                                 },
+                            '/var/spool/cwl': {'kind': 'tmp',
+                                               "capacity": 1073741824 }
                         },
                         'state': 'Committed',
                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                         'output_path': '/var/spool/cwl',
+                        'output_ttl': 0,
                         'container_image': 'arvados/jobs',
                         'command': ['ls', '/var/spool/cwl'],
                         'cwd': '/var/spool/cwl',
@@ -84,6 +93,7 @@ class TestContainer(unittest.TestCase):
         runner = mock.MagicMock()
         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
+        runner.intermediate_output_ttl = 3600
         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
@@ -97,7 +107,8 @@ class TestContainer(unittest.TestCase):
                 "class": "ResourceRequirement",
                 "coresMin": 3,
                 "ramMin": 3000,
-                "tmpdirMin": 4000
+                "tmpdirMin": 4000,
+                "outdirMin": 5000
             }, {
                 "class": "http://arvados.org/cwl#RuntimeConstraints",
                 "keep_cache": 512
@@ -106,6 +117,12 @@ class TestContainer(unittest.TestCase):
             }, {
                 "class": "http://arvados.org/cwl#PartitionRequirement",
                 "partition": "blurb"
+            }, {
+                "class": "http://arvados.org/cwl#IntermediateOutput",
+                "outputTTL": 7200
+            }, {
+                "class": "http://arvados.org/cwl#ReuseRequirement",
+                "enableReuse": False
             }],
             "baseCommand": "ls"
         })
@@ -117,38 +134,41 @@ class TestContainer(unittest.TestCase):
         arvtool.formatgraph = None
         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
                              make_fs_access=make_fs_access, tmpdir="/tmp"):
-            j.run()
+            j.run(enable_reuse=True)
 
         call_args, call_kwargs = runner.api.container_requests().create.call_args
 
         call_body_expected = {
-                'environment': {
-                    'HOME': '/var/spool/cwl',
-                    'TMPDIR': '/tmp'
-                },
-                'name': 'test_resource_requirements',
-                'runtime_constraints': {
-                    'vcpus': 3,
-                    'ram': 3145728000,
-                    'keep_cache_ram': 536870912,
-                    'API': True
-                },
-                'use_existing': True,
-                'priority': 1,
-                'mounts': {
-                    '/tmp': {'kind': 'tmp'},
-                    '/var/spool/cwl': {'kind': 'tmp'}
-                },
-                'state': 'Committed',
-                'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
-                'output_path': '/var/spool/cwl',
-                'container_image': 'arvados/jobs',
-                'command': ['ls'],
-                'cwd': '/var/spool/cwl',
-                'scheduling_parameters': {
-                    'partitions': ['blurb']
-                },
-                'properties': {}
+            'environment': {
+                'HOME': '/var/spool/cwl',
+                'TMPDIR': '/tmp'
+            },
+            'name': 'test_resource_requirements',
+            'runtime_constraints': {
+                'vcpus': 3,
+                'ram': 3145728000,
+                'keep_cache_ram': 536870912,
+                'API': True
+            },
+            'use_existing': False,
+            'priority': 1,
+            'mounts': {
+                '/tmp': {'kind': 'tmp',
+                         "capacity": 4194304000 },
+                '/var/spool/cwl': {'kind': 'tmp',
+                                   "capacity": 5242880000 }
+            },
+            'state': 'Committed',
+            'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+            'output_path': '/var/spool/cwl',
+            'output_ttl': 7200,
+            'container_image': 'arvados/jobs',
+            'command': ['ls'],
+            'cwd': '/var/spool/cwl',
+            'scheduling_parameters': {
+                'partitions': ['blurb']
+            },
+            'properties': {}
         }
 
         call_body = call_kwargs.get('body', None)
@@ -166,6 +186,7 @@ class TestContainer(unittest.TestCase):
         runner = mock.MagicMock()
         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
+        runner.intermediate_output_ttl = 0
         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
@@ -230,50 +251,53 @@ class TestContainer(unittest.TestCase):
         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 
         call_body_expected = {
-                'environment': {
-                    'HOME': '/var/spool/cwl',
-                    'TMPDIR': '/tmp'
-                },
-                'name': 'test_initial_work_dir',
-                'runtime_constraints': {
-                    'vcpus': 1,
-                    'ram': 1073741824
+            'environment': {
+                'HOME': '/var/spool/cwl',
+                'TMPDIR': '/tmp'
+            },
+            'name': 'test_initial_work_dir',
+            'runtime_constraints': {
+                'vcpus': 1,
+                'ram': 1073741824
+            },
+            'use_existing': True,
+            'priority': 1,
+            'mounts': {
+                '/tmp': {'kind': 'tmp',
+                         "capacity": 1073741824 },
+                '/var/spool/cwl': {'kind': 'tmp',
+                                   "capacity": 1073741824 },
+                '/var/spool/cwl/foo': {
+                    'kind': 'collection',
+                    'path': 'foo',
+                    'portable_data_hash': '99999999999999999999999999999996+99'
                 },
-                'use_existing': True,
-                'priority': 1,
-                'mounts': {
-                    '/tmp': {'kind': 'tmp'},
-                    '/var/spool/cwl': {'kind': 'tmp'},
-                    '/var/spool/cwl/foo': {
-                        'kind': 'collection',
-                        'path': 'foo',
-                        'portable_data_hash': '99999999999999999999999999999996+99'
-                    },
-                    '/var/spool/cwl/foo2': {
-                        'kind': 'collection',
-                        'path': 'foo2',
-                        'portable_data_hash': '99999999999999999999999999999996+99'
-                    },
-                    '/var/spool/cwl/filename': {
-                        'kind': 'collection',
-                        'path': 'filename',
-                        'portable_data_hash': '99999999999999999999999999999996+99'
-                    },
-                    '/var/spool/cwl/subdir': {
-                        'kind': 'collection',
-                        'path': 'subdir',
-                        'portable_data_hash': '99999999999999999999999999999996+99'
-                    }
+                '/var/spool/cwl/foo2': {
+                    'kind': 'collection',
+                    'path': 'foo2',
+                    'portable_data_hash': '99999999999999999999999999999996+99'
                 },
-                'state': 'Committed',
-                'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
-                'output_path': '/var/spool/cwl',
-                'container_image': 'arvados/jobs',
-                'command': ['ls'],
-                'cwd': '/var/spool/cwl',
-                'scheduling_parameters': {
+                '/var/spool/cwl/filename': {
+                    'kind': 'collection',
+                    'path': 'filename',
+                    'portable_data_hash': '99999999999999999999999999999996+99'
                 },
-                'properties': {}
+                '/var/spool/cwl/subdir': {
+                    'kind': 'collection',
+                    'path': 'subdir',
+                    'portable_data_hash': '99999999999999999999999999999996+99'
+                }
+            },
+            'state': 'Committed',
+            'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+            'output_path': '/var/spool/cwl',
+            'output_ttl': 0,
+            'container_image': 'arvados/jobs',
+            'command': ['ls'],
+            'cwd': '/var/spool/cwl',
+            'scheduling_parameters': {
+            },
+            'properties': {}
         }
 
         call_body = call_kwargs.get('body', None)
@@ -290,6 +314,7 @@ class TestContainer(unittest.TestCase):
         runner = mock.MagicMock()
         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
+        runner.intermediate_output_ttl = 0
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -328,8 +353,10 @@ class TestContainer(unittest.TestCase):
                     'use_existing': True,
                     'priority': 1,
                     'mounts': {
-                        '/tmp': {'kind': 'tmp'},
-                        '/var/spool/cwl': {'kind': 'tmp'},
+                        '/tmp': {'kind': 'tmp',
+                                 "capacity": 1073741824 },
+                        '/var/spool/cwl': {'kind': 'tmp',
+                                           "capacity": 1073741824 },
                         "stderr": {
                             "kind": "file",
                             "path": "/var/spool/cwl/stderr.txt"
@@ -347,6 +374,7 @@ class TestContainer(unittest.TestCase):
                     'state': 'Committed',
                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                     'output_path': '/var/spool/cwl',
+                    'output_ttl': 0,
                     'container_image': 'arvados/jobs',
                     'command': ['ls', '/var/spool/cwl'],
                     'cwd': '/var/spool/cwl',
@@ -363,6 +391,7 @@ class TestContainer(unittest.TestCase):
         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.num_retries = 0
         runner.ignore_docker_for_reuse = False
+        runner.intermediate_output_ttl = 0
 
         runner.api.containers().get().execute.return_value = {"state":"Complete",
                                                               "output": "abc+123",
@@ -377,6 +406,7 @@ class TestContainer(unittest.TestCase):
         arvjob.collect_outputs = mock.MagicMock()
         arvjob.successCodes = [0]
         arvjob.outdir = "/var/spool/cwl"
+        arvjob.output_ttl = 3600
 
         arvjob.collect_outputs.return_value = {"out": "stuff"}
 
@@ -385,13 +415,15 @@ class TestContainer(unittest.TestCase):
             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
-            "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+            "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
+            "modified_at": "2017-05-26T12:01:22Z"
         })
 
         self.assertFalse(api.collections().create.called)
 
         arvjob.collect_outputs.assert_called_with("keep:abc+123")
         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
+        runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 
     # The test passes no builder.resources
     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@@ -402,6 +434,7 @@ class TestContainer(unittest.TestCase):
         runner = mock.MagicMock()
         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
+        runner.intermediate_output_ttl = 0
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -460,12 +493,15 @@ class TestContainer(unittest.TestCase):
                             "kind": "collection",
                             "portable_data_hash": "99999999999999999999999999999994+44"
                         },
-                        '/tmp': {'kind': 'tmp'},
-                        '/var/spool/cwl': {'kind': 'tmp'}
+                        '/tmp': {'kind': 'tmp',
+                                 "capacity": 1073741824 },
+                        '/var/spool/cwl': {'kind': 'tmp',
+                                           "capacity": 1073741824 }
                     },
                     'state': 'Committed',
                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                     'output_path': '/var/spool/cwl',
+                    'output_ttl': 0,
                     'container_image': 'arvados/jobs',
                     'command': ['ls', '/var/spool/cwl'],
                     'cwd': '/var/spool/cwl',