sdk/python/tests/test_arv_keepdocker.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from __future__ import absolute_import
   6 import arvados
   7 import collections
   8 import copy
   9 import hashlib
  10 import mock
  11 import os
  12 import subprocess
  13 import sys
  14 import tempfile
  15 import unittest
  16 import logging
  17
  18 import arvados.commands.keepdocker as arv_keepdocker
  19 from . import arvados_testutil as tutil
  20 from . import run_test_server
  21
  22
  23 class StopTest(Exception):
  24     pass
  25
  26
  27 class ArvKeepdockerTestCase(unittest.TestCase, tutil.VersionChecker):
  28     def run_arv_keepdocker(self, args, err, **kwargs):
  29         sys.argv = ['arv-keepdocker'] + args
  30         log_handler = logging.StreamHandler(err)
  31         arv_keepdocker.logger.addHandler(log_handler)
  32         try:
  33             return arv_keepdocker.main(**kwargs)
  34         finally:
  35             arv_keepdocker.logger.removeHandler(log_handler)
  36
  37     def test_unsupported_arg(self):
  38         out = tutil.StringIO()
  39         with tutil.redirected_streams(stdout=out, stderr=out), \
  40              self.assertRaises(SystemExit):
  41             self.run_arv_keepdocker(['-x=unknown'], sys.stderr)
  42         self.assertRegex(out.getvalue(), 'unrecognized arguments')
  43
  44     def test_version_argument(self):
  45         with tutil.redirected_streams(
  46                 stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err):
  47             with self.assertRaises(SystemExit):
  48                 self.run_arv_keepdocker(['--version'], sys.stderr)
  49         self.assertVersionOutput(out, err)
  50
  51     @mock.patch('arvados.commands.keepdocker.list_images_in_arv',
  52                 return_value=[])
  53     @mock.patch('arvados.commands.keepdocker.find_image_hashes',
  54                 return_value=['abc123'])
  55     @mock.patch('arvados.commands.keepdocker.find_one_image_hash',
  56                 return_value='abc123')
  57     def test_image_format_compatibility(self, _1, _2, _3):
  58         old_id = hashlib.sha256(b'old').hexdigest()
  59         new_id = 'sha256:'+hashlib.sha256(b'new').hexdigest()
  60         for supported, img_id, expect_ok in [
  61                 (['v1'], old_id, True),
  62                 (['v1'], new_id, False),
  63                 (None, old_id, False),
  64                 ([], old_id, False),
  65                 ([], new_id, False),
  66                 (['v1', 'v2'], new_id, True),
  67                 (['v1'], new_id, False),
  68                 (['v2'], new_id, True)]:
  69
  70             fakeDD = arvados.api('v1')._rootDesc
  71             if supported is None:
  72                 del fakeDD['dockerImageFormats']
  73             else:
  74                 fakeDD['dockerImageFormats'] = supported
  75
  76             err = tutil.StringIO()
  77             out = tutil.StringIO()
  78
  79             with tutil.redirected_streams(stdout=out), \
  80                  mock.patch('arvados.api') as api, \
  81                  mock.patch('arvados.commands.keepdocker.popen_docker',
  82                             return_value=subprocess.Popen(
  83                                 ['echo', img_id],
  84                                 stdout=subprocess.PIPE)), \
  85                  mock.patch('arvados.commands.keepdocker.prep_image_file',
  86                             side_effect=StopTest), \
  87                  self.assertRaises(StopTest if expect_ok else SystemExit):
  88
  89                 api()._rootDesc = fakeDD
  90                 self.run_arv_keepdocker(['--force', 'testimage'], err)
  91
  92             self.assertEqual(out.getvalue(), '')
  93             if expect_ok:
  94                 self.assertNotRegex(
  95                     err.getvalue(), "refusing to store",
  96                     msg=repr((supported, img_id)))
  97             else:
  98                 self.assertRegex(
  99                     err.getvalue(), "refusing to store",
 100                     msg=repr((supported, img_id)))
 101             if not supported:
 102                 self.assertRegex(
 103                     err.getvalue(),
 104                     "server does not specify supported image formats",
 105                     msg=repr((supported, img_id)))
 106
 107         fakeDD = arvados.api('v1')._rootDesc
 108         fakeDD['dockerImageFormats'] = ['v1']
 109         err = tutil.StringIO()
 110         out = tutil.StringIO()
 111         with tutil.redirected_streams(stdout=out), \
 112              mock.patch('arvados.api') as api, \
 113              mock.patch('arvados.commands.keepdocker.popen_docker',
 114                         return_value=subprocess.Popen(
 115                             ['echo', new_id],
 116                             stdout=subprocess.PIPE)), \
 117              mock.patch('arvados.commands.keepdocker.prep_image_file',
 118                         side_effect=StopTest), \
 119              self.assertRaises(StopTest):
 120             api()._rootDesc = fakeDD
 121             self.run_arv_keepdocker(
 122                 ['--force', '--force-image-format', 'testimage'], err)
 123         self.assertRegex(err.getvalue(), "forcing incompatible image")
 124
 125     def test_tag_given_twice(self):
 126         with tutil.redirected_streams(stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err):
 127             with self.assertRaises(SystemExit):
 128                 self.run_arv_keepdocker(['myrepo:mytag', 'extratag'], sys.stderr)
 129             self.assertRegex(err.getvalue(), "cannot add tag argument 'extratag'")
 130
 131     def test_image_given_as_repo_colon_tag(self):
 132         with self.assertRaises(StopTest), \
 133              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 134                         side_effect=StopTest) as find_image_mock:
 135             self.run_arv_keepdocker(['repo:tag'], sys.stderr)
 136         find_image_mock.assert_called_with('repo', 'tag')
 137
 138     def test_image_given_as_registry_repo_colon_tag(self):
 139         with self.assertRaises(StopTest), \
 140              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 141                         side_effect=StopTest) as find_image_mock:
 142             self.run_arv_keepdocker(['myreg.example:8888/repo/img:tag'], sys.stderr)
 143         find_image_mock.assert_called_with('myreg.example:8888/repo/img', 'tag')
 144
 145         with self.assertRaises(StopTest), \
 146              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 147                         side_effect=StopTest) as find_image_mock:
 148             self.run_arv_keepdocker(['registry.hub.docker.com:443/library/debian:bullseye-slim'], sys.stderr)
 149         find_image_mock.assert_called_with('registry.hub.docker.com/library/debian', 'bullseye-slim')
 150
 151     def test_image_has_colons(self):
 152         with self.assertRaises(StopTest), \
 153              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 154                         side_effect=StopTest) as find_image_mock:
 155             self.run_arv_keepdocker(['[::1]:8888/repo/img'], sys.stderr)
 156         find_image_mock.assert_called_with('[::1]:8888/repo/img', 'latest')
 157
 158         with self.assertRaises(StopTest), \
 159              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 160                         side_effect=StopTest) as find_image_mock:
 161             self.run_arv_keepdocker(['[::1]/repo/img'], sys.stderr)
 162         find_image_mock.assert_called_with('[::1]/repo/img', 'latest')
 163
 164         with self.assertRaises(StopTest), \
 165              mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 166                         side_effect=StopTest) as find_image_mock:
 167             self.run_arv_keepdocker(['[::1]:8888/repo/img:tag'], sys.stderr)
 168         find_image_mock.assert_called_with('[::1]:8888/repo/img', 'tag')
 169
 170     def test_list_images_with_host_and_port(self):
 171         api = arvados.api('v1')
 172         taglink = api.links().create(body={'link': {
 173             'link_class': 'docker_image_repo+tag',
 174             'name': 'registry.example:1234/repo:latest',
 175             'head_uuid': 'zzzzz-4zz18-1v45jub259sjjgb',
 176         }}).execute()
 177         try:
 178             out = tutil.StringIO()
 179             with self.assertRaises(SystemExit):
 180                 self.run_arv_keepdocker([], sys.stderr, stdout=out)
 181             self.assertRegex(out.getvalue(), '\nregistry.example:1234/repo +latest ')
 182         finally:
 183             api.links().delete(uuid=taglink['uuid']).execute()
 184
 185     @mock.patch('arvados.commands.keepdocker.list_images_in_arv',
 186                 return_value=[])
 187     @mock.patch('arvados.commands.keepdocker.find_image_hashes',
 188                 return_value=['abc123'])
 189     @mock.patch('arvados.commands.keepdocker.find_one_image_hash',
 190                 return_value='abc123')
 191     def test_collection_property_update(self, _1, _2, _3):
 192         image_id = 'sha256:'+hashlib.sha256(b'image').hexdigest()
 193         fakeDD = arvados.api('v1')._rootDesc
 194         fakeDD['dockerImageFormats'] = ['v2']
 195
 196         err = tutil.StringIO()
 197         out = tutil.StringIO()
 198         File = collections.namedtuple('File', ['name'])
 199         mocked_file = File(name='docker_image')
 200         mocked_collection = {
 201             'uuid': 'new-collection-uuid',
 202             'properties': {
 203                 'responsible_person_uuid': 'person_uuid',
 204             }
 205         }
 206
 207         with tutil.redirected_streams(stdout=out), \
 208              mock.patch('arvados.api') as api, \
 209              mock.patch('arvados.commands.keepdocker.popen_docker',
 210                         return_value=subprocess.Popen(
 211                             ['echo', image_id],
 212                             stdout=subprocess.PIPE)), \
 213              mock.patch('arvados.commands.keepdocker.prep_image_file',
 214                         return_value=(mocked_file, False)), \
 215              mock.patch('arvados.commands.put.main',
 216                         return_value='new-collection-uuid'), \
 217              self.assertRaises(StopTest):
 218
 219             api()._rootDesc = fakeDD
 220             api().collections().get().execute.return_value = copy.deepcopy(mocked_collection)
 221             api().collections().update().execute.side_effect = StopTest
 222             self.run_arv_keepdocker(['--force', 'testimage'], err)
 223
 224         updated_properties = mocked_collection['properties']
 225         updated_properties.update({'docker-image-repo-tag': 'testimage:latest'})
 226         api().collections().update.assert_called_with(
 227             uuid=mocked_collection['uuid'],
 228             body={'properties': updated_properties})