sdk/python/tests/test_s3_to_keep.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import copy
   6 import io
   7 import functools
   8 import hashlib
   9 import json
  10 import logging
  11 import sys
  12 import unittest
  13 import datetime
  14
  15 from unittest import mock
  16
  17 import arvados
  18 import arvados.collection
  19 import arvados.keep
  20
  21 from arvados._internal import s3_to_keep
  22 import boto3.s3.transfer
  23
  24 class TestS3ToKeep(unittest.TestCase):
  25
  26     @mock.patch("arvados.collection.Collection")
  27     def test_s3_get(self, collectionmock):
  28         api = mock.MagicMock()
  29
  30         api.collections().list().execute.return_value = {
  31             "items": []
  32         }
  33
  34         cm = mock.MagicMock()
  35         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz3"
  36         cm.portable_data_hash.return_value = "99999999999999999999999999999998+99"
  37         collectionmock.return_value = cm
  38
  39         mockfile = mock.MagicMock()
  40         cm.open.return_value = mockfile
  41
  42         mockboto = mock.MagicMock()
  43         mockbotoclient = mock.MagicMock()
  44         mockboto.client.return_value = mockbotoclient
  45
  46         mockbotoclient.head_object.return_value = {
  47             'ResponseMetadata': {
  48                 'HTTPStatusCode': 200,
  49                 'HTTPHeaders': {
  50                     "Content-Length": 123
  51                 }
  52             }
  53         }
  54
  55         utcnow = mock.MagicMock()
  56         utcnow.return_value = datetime.datetime(2018, 5, 15)
  57
  58         r = s3_to_keep.s3_to_keep(api, mockboto, None, "s3://examplebucket/file1.txt", utcnow=utcnow)
  59         self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt",
  60                              'zzzzz-4zz18-zzzzzzzzzzzzzz3', 's3://examplebucket/file1.txt',
  61                              datetime.datetime(2018, 5, 15, 0, 0)))
  62
  63         cm.open.assert_called_with("file1.txt", "wb")
  64         cm.save_new.assert_called_with(name="Downloaded from s3%3A%2F%2Fexamplebucket%2Ffile1.txt",
  65                                        owner_uuid=None, ensure_unique_name=True)
  66
  67         api.collections().update.assert_has_calls([
  68             mock.call(uuid=cm.manifest_locator(),
  69                       body={"collection":{"properties": {'s3://examplebucket/file1.txt': {'Content-Length': 123, 'Date': 'Tue, 15 May 2018 00:00:00 GMT'}}}})
  70         ])
  71
  72         kall = mockbotoclient.download_fileobj.call_args
  73         assert kall.kwargs['Bucket'] == 'examplebucket'
  74         assert kall.kwargs['Key'] == 'file1.txt'
  75         assert kall.kwargs['Fileobj'] is mockfile