sdk/python/tests/test_arv_keepdocker.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados
   6 import collections
   7 import collections.abc
   8 import copy
   9 import hashlib
  10 import logging
  11 import mock
  12 import os
  13 import subprocess
  14 import sys
  15 import tempfile
  16 import unittest
  17 from pathlib import Path
  18
  19 import parameterized
  20
  21 import arvados.commands.keepdocker as arv_keepdocker
  22 from . import arvados_testutil as tutil
  23
  24 class StopTest(Exception):
  25     pass
  26
  27
  28 class ArvKeepdockerTestCase(unittest.TestCase, tutil.VersionChecker):
  29     def run_arv_keepdocker(self, args, err, **kwargs):
  30         sys.argv = ['arv-keepdocker'] + args
  31         log_handler = logging.StreamHandler(err)
  32         arv_keepdocker.logger.addHandler(log_handler)
  33         try:
  34             return arv_keepdocker.main(**kwargs)
  35         finally:
  36             arv_keepdocker.logger.removeHandler(log_handler)
  37
  38     def test_unsupported_arg(self):
  39         out = tutil.StringIO()
  40         with tutil.redirected_streams(stdout=out, stderr=out), \
  41              self.assertRaises(SystemExit):
  42             self.run_arv_keepdocker(['-x=unknown'], sys.stderr)
  43         self.assertRegex(out.getvalue(), 'unrecognized arguments')
  44
  45     def test_version_argument(self):
  46         with tutil.redirected_streams(
  47                 stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err):
  48             with self.assertRaises(SystemExit):
  49                 self.run_arv_keepdocker(['--version'], sys.stderr)
  50         self.assertVersionOutput(out, err)
  51
  52     @mock.patch('arvados.commands.keepdocker.list_images_in_arv',
  53                 return_value=[])
  54     @mock.patch('arvados.commands.keepdocker.find_image_hashes',
  55                 return_value=['abc123'])
  56     @mock.patch('arvados.commands.keepdocker.find_one_image_hash',
  57                 return_value='abc123')
  58     def test_image_format_compatibility(self, _1, _2, _3):
  59         old_id = hashlib.sha256(b'old').hexdigest()
  60         new_id = 'sha256:'+hashlib.sha256(b'new').hexdigest()
  61         for supported, img_id, expect_ok in [
  62                 (['v1'], old_id, True),
  63                 (['v1'], new_id, False),
  64                 (None, old_id, False),
  65                 ([], old_id, False),
  66                 ([], new_id, False),
  67                 (['v1', 'v2'], new_id, True),
  68                 (['v1'], new_id, False),
  69                 (['v2'], new_id, True)]:
  70
  71             fakeDD = arvados.api('v1')._rootDesc
  72             if supported is None:
  73                 del fakeDD['dockerImageFormats']
  74             else:
  75                 fakeDD['dockerImageFormats'] = supported
  76
  77             err = tutil.StringIO()
  78             out = tutil.StringIO()
  79
  80             with tutil.redirected_streams(stdout=out), \
  81                  mock.patch('arvados.api') as api, \
  82                  mock.patch('arvados.commands.keepdocker.popen_docker',
  83                             return_value=subprocess.Popen(
  84                                 ['echo', img_id],
  85                                 stdout=subprocess.PIPE)), \
  86                  mock.patch('arvados.commands.keepdocker.prep_image_file',
  87                             side_effect=StopTest), \
  88                  self.assertRaises(StopTest if expect_ok else SystemExit):
  89
  90                 api()._rootDesc = fakeDD
  91                 self.run_arv_keepdocker(['--force', 'testimage'], err)
  92
  93             self.assertEqual(out.getvalue(), '')
  94             if expect_ok:
  95                 self.assertNotRegex(
  96                     err.getvalue(), "refusing to store",
  97                     msg=repr((supported, img_id)))
  98             else:
  99                 self.assertRegex(
 100                     err.getvalue(), "refusing to store",
 101                     msg=repr((supported, img_id)))
 102             if not supported:
 103                 self.assertRegex(
 104                     err.getvalue(),
 105                     "server does not specify supported image formats",
 106                     msg=repr((supported, img_id)))
 107
 108         fakeDD = arvados.api('v1')._rootDesc
 109         fakeDD['dockerImageFormats'] = ['v1']
 110         err = tutil.StringIO()
 111         out = tutil.StringIO()
 112         with tutil.redirected_streams(stdout=out), \
 113              mock.patch('arvados.api') as api, \
 114              mock.patch('arvados.commands.keepdocker.popen_docker',
 115                         return_value=subprocess.Popen(
 116                             ['echo', new_id],
 117                             stdout=subprocess.PIPE)), \
 118              mock.patch('arvados.commands.keepdocker.prep_image_file',
 119                         side_effect=StopTest), \
 120              self.assertRaises(StopTest):
 121             api()._rootDesc = fakeDD
 122             self.run_arv_keepdocker(
 123                 ['--force', '--force-image-format', 'testimage'], err)
 124         self.assertRegex(err.getvalue(), "forcing incompatible image")
 125
 126     def test_tag_given_twice(self):
 127         with tutil.redirected_streams(stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err):
 128             with self.assertRaises(SystemExit):
 129                 self.run_arv_keepdocker(['myrepo:mytag', 'extratag'], sys.stderr)
 130             self.assertRegex(err.getvalue(), "cannot add tag argument 'extratag'")
 131
 132     def test_image_given_as_repo_colon_tag(self):
 133         with self.assertRaises(StopTest), \
 134              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 135                         side_effect=StopTest) as find_image_mock:
 136             self.run_arv_keepdocker(['repo:tag'], sys.stderr)
 137         find_image_mock.assert_called_with('repo', 'tag')
 138
 139     def test_image_given_as_registry_repo_colon_tag(self):
 140         with self.assertRaises(StopTest), \
 141              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 142                         side_effect=StopTest) as find_image_mock:
 143             self.run_arv_keepdocker(['myreg.example:8888/repo/img:tag'], sys.stderr)
 144         find_image_mock.assert_called_with('myreg.example:8888/repo/img', 'tag')
 145
 146         with self.assertRaises(StopTest), \
 147              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 148                         side_effect=StopTest) as find_image_mock:
 149             self.run_arv_keepdocker(['registry.hub.docker.com:443/library/debian:bullseye-slim'], sys.stderr)
 150         find_image_mock.assert_called_with('registry.hub.docker.com/library/debian', 'bullseye-slim')
 151
 152     def test_image_has_colons(self):
 153         with self.assertRaises(StopTest), \
 154              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 155                         side_effect=StopTest) as find_image_mock:
 156             self.run_arv_keepdocker(['[::1]:8888/repo/img'], sys.stderr)
 157         find_image_mock.assert_called_with('[::1]:8888/repo/img', 'latest')
 158
 159         with self.assertRaises(StopTest), \
 160              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 161                         side_effect=StopTest) as find_image_mock:
 162             self.run_arv_keepdocker(['[::1]/repo/img'], sys.stderr)
 163         find_image_mock.assert_called_with('[::1]/repo/img', 'latest')
 164
 165         with self.assertRaises(StopTest), \
 166              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 167                         side_effect=StopTest) as find_image_mock:
 168             self.run_arv_keepdocker(['[::1]:8888/repo/img:tag'], sys.stderr)
 169         find_image_mock.assert_called_with('[::1]:8888/repo/img', 'tag')
 170
 171     def test_list_images_with_host_and_port(self):
 172         api = arvados.api('v1')
 173         taglink = api.links().create(body={'link': {
 174             'link_class': 'docker_image_repo+tag',
 175             'name': 'registry.example:1234/repo:latest',
 176             'head_uuid': 'zzzzz-4zz18-1v45jub259sjjgb',
 177         }}).execute()
 178         try:
 179             out = tutil.StringIO()
 180             with self.assertRaises(SystemExit):
 181                 self.run_arv_keepdocker([], sys.stderr, stdout=out)
 182             self.assertRegex(out.getvalue(), '\nregistry.example:1234/repo +latest ')
 183         finally:
 184             api.links().delete(uuid=taglink['uuid']).execute()
 185
 186     @mock.patch('arvados.commands.keepdocker.list_images_in_arv',
 187                 return_value=[])
 188     @mock.patch('arvados.commands.keepdocker.find_image_hashes',
 189                 return_value=['abc123'])
 190     @mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 191                 return_value='abc123')
 192     def test_collection_property_update(self, _1, _2, _3):
 193         image_id = 'sha256:'+hashlib.sha256(b'image').hexdigest()
 194         fakeDD = arvados.api('v1')._rootDesc
 195         fakeDD['dockerImageFormats'] = ['v2']
 196
 197         err = tutil.StringIO()
 198         out = tutil.StringIO()
 199         File = collections.namedtuple('File', ['name'])
 200         mocked_file = File(name='docker_image')
 201         mocked_collection = {
 202             'uuid': 'new-collection-uuid',
 203             'properties': {
 204                 'responsible_person_uuid': 'person_uuid',
 205             }
 206         }
 207
 208         with tutil.redirected_streams(stdout=out), \
 209              mock.patch('arvados.api') as api, \
 210              mock.patch('arvados.commands.keepdocker.popen_docker',
 211                         return_value=subprocess.Popen(
 212                             ['echo', image_id],
 213                             stdout=subprocess.PIPE)), \
 214              mock.patch('arvados.commands.keepdocker.prep_image_file',
 215                         return_value=(mocked_file, False)), \
 216              mock.patch('arvados.commands.put.main',
 217                         return_value='new-collection-uuid'), \
 218              self.assertRaises(StopTest):
 219
 220             api()._rootDesc = fakeDD
 221             api().collections().get().execute.return_value = copy.deepcopy(mocked_collection)
 222             api().collections().update().execute.side_effect = StopTest
 223             self.run_arv_keepdocker(['--force', 'testimage'], err)
 224
 225         updated_properties = mocked_collection['properties']
 226         updated_properties.update({'docker-image-repo-tag': 'testimage:latest'})
 227         api().collections().update.assert_called_with(
 228             uuid=mocked_collection['uuid'],
 229             body={'properties': updated_properties})
 230
 231
 232 @parameterized.parameterized_class(('filename',), [
 233     ('hello-world-ManifestV2.tar',),
 234     ('hello-world-ManifestV2-OCILayout.tar',),
 235 ])
 236 class ImageMetadataTestCase(unittest.TestCase):
 237     DATA_PATH = Path(__file__).parent / 'data'
 238
 239     @classmethod
 240     def setUpClass(cls):
 241         cls.image_file = (cls.DATA_PATH / cls.filename).open('rb')
 242
 243     @classmethod
 244     def tearDownClass(cls):
 245         cls.image_file.close()
 246
 247     def setUp(self):
 248         self.manifest, self.config = arv_keepdocker.load_image_metadata(self.image_file)
 249
 250     def test_image_manifest(self):
 251         self.assertIsInstance(self.manifest, collections.abc.Mapping)
 252         self.assertEqual(self.manifest.get('RepoTags'), ['hello-world:latest'])
 253
 254     def test_image_config(self):
 255         self.assertIsInstance(self.config, collections.abc.Mapping)
 256         self.assertEqual(self.config.get('created'), '2023-05-02T16:49:27Z')