sdk/python/tests/test_arv_keepdocker.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados
   6 import collections
   7 import collections.abc
   8 import copy
   9 import hashlib
  10 import logging
  11 import os
  12 import subprocess
  13 import sys
  14 import tempfile
  15 import unittest
  16
  17 from pathlib import Path
  18 from unittest import mock
  19
  20 import parameterized
  21
  22 import arvados.commands.keepdocker as arv_keepdocker
  23 from . import arvados_testutil as tutil
  24
  25 class StopTest(Exception):
  26     pass
  27
  28
  29 class ArvKeepdockerTestCase(unittest.TestCase, tutil.VersionChecker):
  30     def run_arv_keepdocker(self, args, err, **kwargs):
  31         sys.argv = ['arv-keepdocker'] + args
  32         log_handler = logging.StreamHandler(err)
  33         arv_keepdocker.logger.addHandler(log_handler)
  34         try:
  35             return arv_keepdocker.main(**kwargs)
  36         finally:
  37             arv_keepdocker.logger.removeHandler(log_handler)
  38
  39     def test_unsupported_arg(self):
  40         out = tutil.StringIO()
  41         with tutil.redirected_streams(stdout=out, stderr=out), \
  42              self.assertRaises(SystemExit):
  43             self.run_arv_keepdocker(['-x=unknown'], sys.stderr)
  44         self.assertRegex(out.getvalue(), 'unrecognized arguments')
  45
  46     def test_version_argument(self):
  47         with tutil.redirected_streams(
  48                 stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err):
  49             with self.assertRaises(SystemExit):
  50                 self.run_arv_keepdocker(['--version'], sys.stderr)
  51         self.assertVersionOutput(out, err)
  52
  53     @mock.patch('arvados.commands.keepdocker.list_images_in_arv',
  54                 return_value=[])
  55     @mock.patch('arvados.commands.keepdocker.find_image_hashes',
  56                 return_value=['abc123'])
  57     @mock.patch('arvados.commands.keepdocker.find_one_image_hash',
  58                 return_value='abc123')
  59     def test_image_format_compatibility(self, _1, _2, _3):
  60         old_id = hashlib.sha256(b'old').hexdigest()
  61         new_id = 'sha256:'+hashlib.sha256(b'new').hexdigest()
  62         for supported, img_id, expect_ok in [
  63                 (['v1'], old_id, True),
  64                 (['v1'], new_id, False),
  65                 (None, old_id, False),
  66                 ([], old_id, False),
  67                 ([], new_id, False),
  68                 (['v1', 'v2'], new_id, True),
  69                 (['v1'], new_id, False),
  70                 (['v2'], new_id, True)]:
  71
  72             fakeDD = arvados.api('v1')._rootDesc
  73             if supported is None:
  74                 del fakeDD['dockerImageFormats']
  75             else:
  76                 fakeDD['dockerImageFormats'] = supported
  77
  78             err = tutil.StringIO()
  79             out = tutil.StringIO()
  80
  81             with tutil.redirected_streams(stdout=out), \
  82                  mock.patch('arvados.api') as api, \
  83                  mock.patch('arvados.commands.keepdocker.popen_docker',
  84                             return_value=subprocess.Popen(
  85                                 ['echo', img_id],
  86                                 stdout=subprocess.PIPE)), \
  87                  mock.patch('arvados.commands.keepdocker.prep_image_file',
  88                             side_effect=StopTest), \
  89                  self.assertRaises(StopTest if expect_ok else SystemExit):
  90
  91                 api()._rootDesc = fakeDD
  92                 self.run_arv_keepdocker(['--force', 'testimage'], err)
  93
  94             self.assertEqual(out.getvalue(), '')
  95             if expect_ok:
  96                 self.assertNotRegex(
  97                     err.getvalue(), "refusing to store",
  98                     msg=repr((supported, img_id)))
  99             else:
 100                 self.assertRegex(
 101                     err.getvalue(), "refusing to store",
 102                     msg=repr((supported, img_id)))
 103             if not supported:
 104                 self.assertRegex(
 105                     err.getvalue(),
 106                     "server does not specify supported image formats",
 107                     msg=repr((supported, img_id)))
 108
 109         fakeDD = arvados.api('v1')._rootDesc
 110         fakeDD['dockerImageFormats'] = ['v1']
 111         err = tutil.StringIO()
 112         out = tutil.StringIO()
 113         with tutil.redirected_streams(stdout=out), \
 114              mock.patch('arvados.api') as api, \
 115              mock.patch('arvados.commands.keepdocker.popen_docker',
 116                         return_value=subprocess.Popen(
 117                             ['echo', new_id],
 118                             stdout=subprocess.PIPE)), \
 119              mock.patch('arvados.commands.keepdocker.prep_image_file',
 120                         side_effect=StopTest), \
 121              self.assertRaises(StopTest):
 122             api()._rootDesc = fakeDD
 123             self.run_arv_keepdocker(
 124                 ['--force', '--force-image-format', 'testimage'], err)
 125         self.assertRegex(err.getvalue(), "forcing incompatible image")
 126
 127     def test_tag_given_twice(self):
 128         with tutil.redirected_streams(stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err):
 129             with self.assertRaises(SystemExit):
 130                 self.run_arv_keepdocker(['myrepo:mytag', 'extratag'], sys.stderr)
 131             self.assertRegex(err.getvalue(), "cannot add tag argument 'extratag'")
 132
 133     def test_image_given_as_repo_colon_tag(self):
 134         with self.assertRaises(StopTest), \
 135              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 136                         side_effect=StopTest) as find_image_mock:
 137             self.run_arv_keepdocker(['repo:tag'], sys.stderr)
 138         find_image_mock.assert_called_with('repo', 'tag')
 139
 140     def test_image_given_as_registry_repo_colon_tag(self):
 141         with self.assertRaises(StopTest), \
 142              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 143                         side_effect=StopTest) as find_image_mock:
 144             self.run_arv_keepdocker(['myreg.example:8888/repo/img:tag'], sys.stderr)
 145         find_image_mock.assert_called_with('myreg.example:8888/repo/img', 'tag')
 146
 147         with self.assertRaises(StopTest), \
 148              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 149                         side_effect=StopTest) as find_image_mock:
 150             self.run_arv_keepdocker(['registry.hub.docker.com:443/library/debian:bullseye-slim'], sys.stderr)
 151         find_image_mock.assert_called_with('registry.hub.docker.com/library/debian', 'bullseye-slim')
 152
 153     def test_image_has_colons(self):
 154         with self.assertRaises(StopTest), \
 155              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 156                         side_effect=StopTest) as find_image_mock:
 157             self.run_arv_keepdocker(['[::1]:8888/repo/img'], sys.stderr)
 158         find_image_mock.assert_called_with('[::1]:8888/repo/img', 'latest')
 159
 160         with self.assertRaises(StopTest), \
 161              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 162                         side_effect=StopTest) as find_image_mock:
 163             self.run_arv_keepdocker(['[::1]/repo/img'], sys.stderr)
 164         find_image_mock.assert_called_with('[::1]/repo/img', 'latest')
 165
 166         with self.assertRaises(StopTest), \
 167              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 168                         side_effect=StopTest) as find_image_mock:
 169             self.run_arv_keepdocker(['[::1]:8888/repo/img:tag'], sys.stderr)
 170         find_image_mock.assert_called_with('[::1]:8888/repo/img', 'tag')
 171
 172     def test_list_images_with_host_and_port(self):
 173         api = arvados.api('v1')
 174         taglink = api.links().create(body={'link': {
 175             'link_class': 'docker_image_repo+tag',
 176             'name': 'registry.example:1234/repo:latest',
 177             'head_uuid': 'zzzzz-4zz18-1v45jub259sjjgb',
 178         }}).execute()
 179         try:
 180             out = tutil.StringIO()
 181             with self.assertRaises(SystemExit):
 182                 self.run_arv_keepdocker([], sys.stderr, stdout=out)
 183             self.assertRegex(out.getvalue(), '\nregistry.example:1234/repo +latest ')
 184         finally:
 185             api.links().delete(uuid=taglink['uuid']).execute()
 186
 187     @mock.patch('arvados.commands.keepdocker.list_images_in_arv',
 188                 return_value=[])
 189     @mock.patch('arvados.commands.keepdocker.find_image_hashes',
 190                 return_value=['abc123'])
 191     @mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 192                 return_value='abc123')
 193     def test_collection_property_update(self, _1, _2, _3):
 194         image_id = 'sha256:'+hashlib.sha256(b'image').hexdigest()
 195         fakeDD = arvados.api('v1')._rootDesc
 196         fakeDD['dockerImageFormats'] = ['v2']
 197
 198         err = tutil.StringIO()
 199         out = tutil.StringIO()
 200         File = collections.namedtuple('File', ['name'])
 201         mocked_file = File(name='docker_image')
 202         mocked_collection = {
 203             'uuid': 'new-collection-uuid',
 204             'properties': {
 205                 'responsible_person_uuid': 'person_uuid',
 206             }
 207         }
 208
 209         with tutil.redirected_streams(stdout=out), \
 210              mock.patch('arvados.api') as api, \
 211              mock.patch('arvados.commands.keepdocker.popen_docker',
 212                         return_value=subprocess.Popen(
 213                             ['echo', image_id],
 214                             stdout=subprocess.PIPE)), \
 215              mock.patch('arvados.commands.keepdocker.prep_image_file',
 216                         return_value=(mocked_file, False)), \
 217              mock.patch('arvados.commands.put.main',
 218                         return_value='new-collection-uuid'), \
 219              self.assertRaises(StopTest):
 220
 221             api()._rootDesc = fakeDD
 222             api().collections().get().execute.return_value = copy.deepcopy(mocked_collection)
 223             api().collections().update().execute.side_effect = StopTest
 224             self.run_arv_keepdocker(['--force', 'testimage'], err)
 225
 226         updated_properties = mocked_collection['properties']
 227         updated_properties.update({'docker-image-repo-tag': 'testimage:latest'})
 228         api().collections().update.assert_called_with(
 229             uuid=mocked_collection['uuid'],
 230             body={'properties': updated_properties})
 231
 232
 233 @parameterized.parameterized_class(('filename',), [
 234     ('hello-world-ManifestV2.tar',),
 235     ('hello-world-ManifestV2-OCILayout.tar',),
 236 ])
 237 class ImageMetadataTestCase(unittest.TestCase):
 238     DATA_PATH = Path(__file__).parent / 'data'
 239
 240     @classmethod
 241     def setUpClass(cls):
 242         cls.image_file = (cls.DATA_PATH / cls.filename).open('rb')
 243
 244     @classmethod
 245     def tearDownClass(cls):
 246         cls.image_file.close()
 247
 248     def setUp(self):
 249         self.manifest, self.config = arv_keepdocker.load_image_metadata(self.image_file)
 250
 251     def test_image_manifest(self):
 252         self.assertIsInstance(self.manifest, collections.abc.Mapping)
 253         self.assertEqual(self.manifest.get('RepoTags'), ['hello-world:latest'])
 254
 255     def test_image_config(self):
 256         self.assertIsInstance(self.config, collections.abc.Mapping)
 257         self.assertEqual(self.config.get('created'), '2023-05-02T16:49:27Z')