sdk/python/tests/test_arv_keepdocker.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados
   6 import collections
   7 import collections.abc
   8 import copy
   9 import hashlib
  10 import logging
  11 import os
  12 import subprocess
  13 import sys
  14 import tempfile
  15 import unittest
  16
  17 from pathlib import Path
  18 from unittest import mock
  19
  20 import parameterized
  21 import pytest
  22
  23 import arvados.commands.keepdocker as arv_keepdocker
  24 from . import arvados_testutil as tutil
  25
  26 class StopTest(Exception):
  27     pass
  28
  29
  30 class ArvKeepdockerTestCase(unittest.TestCase, tutil.VersionChecker):
  31     def run_arv_keepdocker(self, args, err, **kwargs):
  32         sys.argv = ['arv-keepdocker'] + args
  33         log_handler = logging.StreamHandler(err)
  34         arv_keepdocker.logger.addHandler(log_handler)
  35         try:
  36             return arv_keepdocker.main(**kwargs)
  37         finally:
  38             arv_keepdocker.logger.removeHandler(log_handler)
  39
  40     def test_unsupported_arg(self):
  41         out = tutil.StringIO()
  42         with tutil.redirected_streams(stdout=out, stderr=out), \
  43              self.assertRaises(SystemExit):
  44             self.run_arv_keepdocker(['-x=unknown'], sys.stderr)
  45         self.assertRegex(out.getvalue(), r'unrecognized arguments')
  46
  47     def test_version_argument(self):
  48         with tutil.redirected_streams(
  49                 stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err):
  50             with self.assertRaises(SystemExit):
  51                 self.run_arv_keepdocker(['--version'], sys.stderr)
  52         self.assertVersionOutput(out, err)
  53
  54     @mock.patch('arvados.commands.keepdocker.list_images_in_arv',
  55                 return_value=[])
  56     @mock.patch('arvados.commands.keepdocker.find_image_hashes',
  57                 return_value=['abc123'])
  58     @mock.patch('arvados.commands.keepdocker.find_one_image_hash',
  59                 return_value='abc123')
  60     def test_image_format_compatibility(self, _1, _2, _3):
  61         old_id = hashlib.sha256(b'old').hexdigest()
  62         new_id = 'sha256:'+hashlib.sha256(b'new').hexdigest()
  63         for supported, img_id, expect_ok in [
  64                 (['v1'], old_id, True),
  65                 (['v1'], new_id, False),
  66                 (None, old_id, False),
  67                 ([], old_id, False),
  68                 ([], new_id, False),
  69                 (['v1', 'v2'], new_id, True),
  70                 (['v1'], new_id, False),
  71                 (['v2'], new_id, True)]:
  72
  73             fakeDD = arvados.api('v1')._rootDesc
  74             if supported is None:
  75                 del fakeDD['dockerImageFormats']
  76             else:
  77                 fakeDD['dockerImageFormats'] = supported
  78
  79             err = tutil.StringIO()
  80             out = tutil.StringIO()
  81
  82             with tutil.redirected_streams(stdout=out), \
  83                  mock.patch('arvados.api') as api, \
  84                  mock.patch('arvados.commands.keepdocker.popen_docker',
  85                             return_value=subprocess.Popen(
  86                                 ['echo', img_id],
  87                                 stdout=subprocess.PIPE)), \
  88                  mock.patch('arvados.commands.keepdocker.prep_image_file',
  89                             side_effect=StopTest), \
  90                  self.assertRaises(StopTest if expect_ok else SystemExit):
  91
  92                 api()._rootDesc = fakeDD
  93                 self.run_arv_keepdocker(['--force', 'testimage'], err)
  94
  95             self.assertEqual(out.getvalue(), '')
  96             if expect_ok:
  97                 self.assertNotRegex(
  98                     err.getvalue(), r"refusing to store",
  99                     msg=repr((supported, img_id)))
 100             else:
 101                 self.assertRegex(
 102                     err.getvalue(), r"refusing to store",
 103                     msg=repr((supported, img_id)))
 104             if not supported:
 105                 self.assertRegex(
 106                     err.getvalue(),
 107                     r"server does not specify supported image formats",
 108                     msg=repr((supported, img_id)))
 109
 110         fakeDD = arvados.api('v1')._rootDesc
 111         fakeDD['dockerImageFormats'] = ['v1']
 112         err = tutil.StringIO()
 113         out = tutil.StringIO()
 114         with tutil.redirected_streams(stdout=out), \
 115              mock.patch('arvados.api') as api, \
 116              mock.patch('arvados.commands.keepdocker.popen_docker',
 117                         return_value=subprocess.Popen(
 118                             ['echo', new_id],
 119                             stdout=subprocess.PIPE)), \
 120              mock.patch('arvados.commands.keepdocker.prep_image_file',
 121                         side_effect=StopTest), \
 122              self.assertRaises(StopTest):
 123             api()._rootDesc = fakeDD
 124             self.run_arv_keepdocker(
 125                 ['--force', '--force-image-format', 'testimage'], err)
 126         self.assertRegex(err.getvalue(), r"forcing incompatible image")
 127
 128     def test_tag_given_twice(self):
 129         with tutil.redirected_streams(stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err):
 130             with self.assertRaises(SystemExit):
 131                 self.run_arv_keepdocker(['myrepo:mytag', 'extratag'], sys.stderr)
 132             self.assertRegex(err.getvalue(), r"cannot add tag argument 'extratag'")
 133
 134     def test_image_given_as_repo_colon_tag(self):
 135         with self.assertRaises(StopTest), \
 136              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 137                         side_effect=StopTest) as find_image_mock:
 138             self.run_arv_keepdocker(['repo:tag'], sys.stderr)
 139         find_image_mock.assert_called_with('repo', 'tag')
 140
 141     def test_image_given_as_registry_repo_colon_tag(self):
 142         with self.assertRaises(StopTest), \
 143              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 144                         side_effect=StopTest) as find_image_mock:
 145             self.run_arv_keepdocker(['myreg.example:8888/repo/img:tag'], sys.stderr)
 146         find_image_mock.assert_called_with('myreg.example:8888/repo/img', 'tag')
 147
 148         with self.assertRaises(StopTest), \
 149              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 150                         side_effect=StopTest) as find_image_mock:
 151             self.run_arv_keepdocker(['registry.hub.docker.com:443/library/debian:bullseye-slim'], sys.stderr)
 152         find_image_mock.assert_called_with('registry.hub.docker.com/library/debian', 'bullseye-slim')
 153
 154     def test_image_has_colons(self):
 155         with self.assertRaises(StopTest), \
 156              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 157                         side_effect=StopTest) as find_image_mock:
 158             self.run_arv_keepdocker(['[::1]:8888/repo/img'], sys.stderr)
 159         find_image_mock.assert_called_with('[::1]:8888/repo/img', 'latest')
 160
 161         with self.assertRaises(StopTest), \
 162              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 163                         side_effect=StopTest) as find_image_mock:
 164             self.run_arv_keepdocker(['[::1]/repo/img'], sys.stderr)
 165         find_image_mock.assert_called_with('[::1]/repo/img', 'latest')
 166
 167         with self.assertRaises(StopTest), \
 168              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 169                         side_effect=StopTest) as find_image_mock:
 170             self.run_arv_keepdocker(['[::1]:8888/repo/img:tag'], sys.stderr)
 171         find_image_mock.assert_called_with('[::1]:8888/repo/img', 'tag')
 172
 173     def test_list_images_with_host_and_port(self):
 174         api = arvados.api('v1')
 175         taglink = api.links().create(body={'link': {
 176             'link_class': 'docker_image_repo+tag',
 177             'name': 'registry.example:1234/repo:latest',
 178             'head_uuid': 'zzzzz-4zz18-1v45jub259sjjgb',
 179         }}).execute()
 180         try:
 181             out = tutil.StringIO()
 182             with self.assertRaises(SystemExit):
 183                 self.run_arv_keepdocker([], sys.stderr, stdout=out)
 184             self.assertRegex(out.getvalue(), r'\nregistry.example:1234/repo +latest ')
 185         finally:
 186             api.links().delete(uuid=taglink['uuid']).execute()
 187
 188     @mock.patch('arvados.commands.keepdocker.list_images_in_arv',
 189                 return_value=[])
 190     @mock.patch('arvados.commands.keepdocker.find_image_hashes',
 191                 return_value=['abc123'])
 192     @mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 193                 return_value='abc123')
 194     def test_collection_property_update(self, _1, _2, _3):
 195         image_id = 'sha256:'+hashlib.sha256(b'image').hexdigest()
 196         fakeDD = arvados.api('v1')._rootDesc
 197         fakeDD['dockerImageFormats'] = ['v2']
 198
 199         err = tutil.StringIO()
 200         out = tutil.StringIO()
 201         File = collections.namedtuple('File', ['name'])
 202         mocked_file = File(name='docker_image')
 203         mocked_collection = {
 204             'uuid': 'new-collection-uuid',
 205             'properties': {
 206                 'responsible_person_uuid': 'person_uuid',
 207             }
 208         }
 209
 210         with tutil.redirected_streams(stdout=out), \
 211              mock.patch('arvados.api') as api, \
 212              mock.patch('arvados.commands.keepdocker.popen_docker',
 213                         return_value=subprocess.Popen(
 214                             ['echo', image_id],
 215                             stdout=subprocess.PIPE)), \
 216              mock.patch('arvados.commands.keepdocker.prep_image_file',
 217                         return_value=(mocked_file, False)), \
 218              mock.patch('arvados.commands.put.main',
 219                         return_value='new-collection-uuid'), \
 220              self.assertRaises(StopTest):
 221
 222             api()._rootDesc = fakeDD
 223             api().collections().get().execute.return_value = copy.deepcopy(mocked_collection)
 224             api().collections().update().execute.side_effect = StopTest
 225             self.run_arv_keepdocker(['--force', 'testimage'], err)
 226
 227         updated_properties = mocked_collection['properties']
 228         updated_properties.update({'docker-image-repo-tag': 'testimage:latest'})
 229         api().collections().update.assert_called_with(
 230             uuid=mocked_collection['uuid'],
 231             body={'properties': updated_properties})
 232
 233
 234 @parameterized.parameterized_class(('filename',), [
 235     ('hello-world-ManifestV2.tar',),
 236     ('hello-world-ManifestV2-OCILayout.tar',),
 237 ])
 238 class ImageMetadataTestCase(unittest.TestCase):
 239     DATA_PATH = Path(__file__).parent / 'data'
 240
 241     @classmethod
 242     def setUpClass(cls):
 243         cls.image_file = (cls.DATA_PATH / cls.filename).open('rb')
 244
 245     @classmethod
 246     def tearDownClass(cls):
 247         cls.image_file.close()
 248
 249     def setUp(self):
 250         self.manifest, self.config = arv_keepdocker.load_image_metadata(self.image_file)
 251
 252     def test_image_manifest(self):
 253         self.assertIsInstance(self.manifest, collections.abc.Mapping)
 254         self.assertEqual(self.manifest.get('RepoTags'), ['hello-world:latest'])
 255
 256     def test_image_config(self):
 257         self.assertIsInstance(self.config, collections.abc.Mapping)
 258         self.assertEqual(self.config.get('created'), '2023-05-02T16:49:27Z')
 259
 260
 261 def test_get_cache_dir(tmp_path):
 262     actual = arv_keepdocker.get_cache_dir(lambda: tmp_path)
 263     assert isinstance(actual, str)
 264     actual = Path(actual)
 265     assert actual.is_dir()
 266     assert actual.name == 'docker'
 267     assert actual.parent == tmp_path