sdk/cwl/tests/test_http.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7
   8 import copy
   9 import io
  10 import functools
  11 import hashlib
  12 import json
  13 import logging
  14 import mock
  15 import sys
  16 import unittest
  17 import datetime
  18
  19 import arvados
  20 import arvados.collection
  21 import arvados_cwl
  22 import arvados_cwl.runner
  23 import arvados.keep
  24
  25 from .matcher import JsonDiffMatcher, StripYAMLComments
  26 from .mock_discovery import get_rootDesc
  27
  28 import arvados_cwl.http
  29
  30 import ruamel.yaml as yaml
  31
  32
  33 class TestHttpToKeep(unittest.TestCase):
  34
  35     @mock.patch("requests.get")
  36     @mock.patch("arvados.collection.Collection")
  37     def test_http_get(self, collectionmock, getmock):
  38         api = mock.MagicMock()
  39
  40         api.collections().list().execute.return_value = {
  41             "items": []
  42         }
  43
  44         cm = mock.MagicMock()
  45         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz3"
  46         cm.portable_data_hash.return_value = "99999999999999999999999999999998+99"
  47         collectionmock.return_value = cm
  48
  49         req = mock.MagicMock()
  50         req.status_code = 200
  51         req.headers = {}
  52         req.iter_content.return_value = ["abc"]
  53         getmock.return_value = req
  54
  55         utcnow = mock.MagicMock()
  56         utcnow.return_value = datetime.datetime(2018, 5, 15)
  57
  58         r = arvados_cwl.http.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
  59         self.assertEqual(r, "keep:99999999999999999999999999999998+99/file1.txt")
  60
  61         getmock.assert_called_with("http://example.com/file1.txt", stream=True, allow_redirects=True)
  62
  63         cm.open.assert_called_with("file1.txt", "wb")
  64         cm.save_new.assert_called_with(name="Downloaded from http://example.com/file1.txt",
  65                                        owner_uuid=None, ensure_unique_name=True)
  66
  67         api.collections().update.assert_has_calls([
  68             mock.call(uuid=cm.manifest_locator(),
  69                       body={"collection":{"properties": {'http://example.com/file1.txt': {'Date': 'Tue, 15 May 2018 00:00:00 GMT'}}}})
  70         ])
  71
  72
  73     @mock.patch("requests.get")
  74     @mock.patch("arvados.collection.CollectionReader")
  75     def test_http_expires(self, collectionmock, getmock):
  76         api = mock.MagicMock()
  77
  78         api.collections().list().execute.return_value = {
  79             "items": [{
  80                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz3",
  81                 "portable_data_hash": "99999999999999999999999999999998+99",
  82                 "properties": {
  83                     'http://example.com/file1.txt': {
  84                         'Date': 'Tue, 15 May 2018 00:00:00 GMT',
  85                         'Expires': 'Tue, 17 May 2018 00:00:00 GMT'
  86                     }
  87                 }
  88             }]
  89         }
  90
  91         cm = mock.MagicMock()
  92         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz3"
  93         cm.portable_data_hash.return_value = "99999999999999999999999999999998+99"
  94         cm.keys.return_value = ["file1.txt"]
  95         collectionmock.return_value = cm
  96
  97         req = mock.MagicMock()
  98         req.status_code = 200
  99         req.headers = {}
 100         req.iter_content.return_value = ["abc"]
 101         getmock.return_value = req
 102
 103         utcnow = mock.MagicMock()
 104         utcnow.return_value = datetime.datetime(2018, 5, 16)
 105
 106         r = arvados_cwl.http.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
 107         self.assertEqual(r, "keep:99999999999999999999999999999998+99/file1.txt")
 108
 109         getmock.assert_not_called()
 110
 111
 112     @mock.patch("requests.get")
 113     @mock.patch("arvados.collection.CollectionReader")
 114     def test_http_cache_control(self, collectionmock, getmock):
 115         api = mock.MagicMock()
 116
 117         api.collections().list().execute.return_value = {
 118             "items": [{
 119                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz3",
 120                 "portable_data_hash": "99999999999999999999999999999998+99",
 121                 "properties": {
 122                     'http://example.com/file1.txt': {
 123                         'Date': 'Tue, 15 May 2018 00:00:00 GMT',
 124                         'Cache-Control': 'max-age=172800'
 125                     }
 126                 }
 127             }]
 128         }
 129
 130         cm = mock.MagicMock()
 131         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz3"
 132         cm.portable_data_hash.return_value = "99999999999999999999999999999998+99"
 133         cm.keys.return_value = ["file1.txt"]
 134         collectionmock.return_value = cm
 135
 136         req = mock.MagicMock()
 137         req.status_code = 200
 138         req.headers = {}
 139         req.iter_content.return_value = ["abc"]
 140         getmock.return_value = req
 141
 142         utcnow = mock.MagicMock()
 143         utcnow.return_value = datetime.datetime(2018, 5, 16)
 144
 145         r = arvados_cwl.http.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
 146         self.assertEqual(r, "keep:99999999999999999999999999999998+99/file1.txt")
 147
 148         getmock.assert_not_called()
 149
 150
 151     @mock.patch("requests.get")
 152     @mock.patch("requests.head")
 153     @mock.patch("arvados.collection.Collection")
 154     def test_http_expired(self, collectionmock, headmock, getmock):
 155         api = mock.MagicMock()
 156
 157         api.collections().list().execute.return_value = {
 158             "items": [{
 159                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz3",
 160                 "portable_data_hash": "99999999999999999999999999999998+99",
 161                 "properties": {
 162                     'http://example.com/file1.txt': {
 163                         'Date': 'Tue, 15 May 2018 00:00:00 GMT',
 164                         'Expires': 'Tue, 16 May 2018 00:00:00 GMT'
 165                     }
 166                 }
 167             }]
 168         }
 169
 170         cm = mock.MagicMock()
 171         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz4"
 172         cm.portable_data_hash.return_value = "99999999999999999999999999999997+99"
 173         cm.keys.return_value = ["file1.txt"]
 174         collectionmock.return_value = cm
 175
 176         req = mock.MagicMock()
 177         req.status_code = 200
 178         req.headers = {'Date': 'Tue, 17 May 2018 00:00:00 GMT'}
 179         req.iter_content.return_value = ["def"]
 180         getmock.return_value = req
 181         headmock.return_value = req
 182
 183         utcnow = mock.MagicMock()
 184         utcnow.return_value = datetime.datetime(2018, 5, 17)
 185
 186         r = arvados_cwl.http.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
 187         self.assertEqual(r, "keep:99999999999999999999999999999997+99/file1.txt")
 188
 189         getmock.assert_called_with("http://example.com/file1.txt", stream=True, allow_redirects=True)
 190
 191         cm.open.assert_called_with("file1.txt", "wb")
 192         cm.save_new.assert_called_with(name="Downloaded from http://example.com/file1.txt",
 193                                        owner_uuid=None, ensure_unique_name=True)
 194
 195         api.collections().update.assert_has_calls([
 196             mock.call(uuid=cm.manifest_locator(),
 197                       body={"collection":{"properties": {'http://example.com/file1.txt': {'Date': 'Tue, 17 May 2018 00:00:00 GMT'}}}})
 198         ])
 199
 200
 201     @mock.patch("requests.get")
 202     @mock.patch("requests.head")
 203     @mock.patch("arvados.collection.CollectionReader")
 204     def test_http_etag(self, collectionmock, headmock, getmock):
 205         api = mock.MagicMock()
 206
 207         api.collections().list().execute.return_value = {
 208             "items": [{
 209                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz3",
 210                 "portable_data_hash": "99999999999999999999999999999998+99",
 211                 "properties": {
 212                     'http://example.com/file1.txt': {
 213                         'Date': 'Tue, 15 May 2018 00:00:00 GMT',
 214                         'Expires': 'Tue, 16 May 2018 00:00:00 GMT',
 215                         'ETag': '123456'
 216                     }
 217                 }
 218             }]
 219         }
 220
 221         cm = mock.MagicMock()
 222         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz3"
 223         cm.portable_data_hash.return_value = "99999999999999999999999999999998+99"
 224         cm.keys.return_value = ["file1.txt"]
 225         collectionmock.return_value = cm
 226
 227         req = mock.MagicMock()
 228         req.status_code = 200
 229         req.headers = {
 230             'Date': 'Tue, 17 May 2018 00:00:00 GMT',
 231             'Expires': 'Tue, 19 May 2018 00:00:00 GMT',
 232             'ETag': '123456'
 233         }
 234         headmock.return_value = req
 235
 236         utcnow = mock.MagicMock()
 237         utcnow.return_value = datetime.datetime(2018, 5, 17)
 238
 239         r = arvados_cwl.http.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
 240         self.assertEqual(r, "keep:99999999999999999999999999999998+99/file1.txt")
 241
 242         getmock.assert_not_called()
 243         cm.open.assert_not_called()
 244
 245         api.collections().update.assert_has_calls([
 246             mock.call(uuid=cm.manifest_locator(),
 247                       body={"collection":{"properties": {'http://example.com/file1.txt': {
 248                           'Date': 'Tue, 17 May 2018 00:00:00 GMT',
 249                           'Expires': 'Tue, 19 May 2018 00:00:00 GMT',
 250                           'ETag': '123456'
 251                       }}}})
 252                       ])
 253
 254     @mock.patch("requests.get")
 255     @mock.patch("arvados.collection.Collection")
 256     def test_http_content_disp(self, collectionmock, getmock):
 257         api = mock.MagicMock()
 258
 259         api.collections().list().execute.return_value = {
 260             "items": []
 261         }
 262
 263         cm = mock.MagicMock()
 264         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz3"
 265         cm.portable_data_hash.return_value = "99999999999999999999999999999998+99"
 266         collectionmock.return_value = cm
 267
 268         req = mock.MagicMock()
 269         req.status_code = 200
 270         req.headers = {"Content-Disposition": "attachment; filename=file1.txt"}
 271         req.iter_content.return_value = ["abc"]
 272         getmock.return_value = req
 273
 274         utcnow = mock.MagicMock()
 275         utcnow.return_value = datetime.datetime(2018, 5, 15)
 276
 277         r = arvados_cwl.http.http_to_keep(api, None, "http://example.com/download?fn=/file1.txt", utcnow=utcnow)
 278         self.assertEqual(r, "keep:99999999999999999999999999999998+99/file1.txt")
 279
 280         getmock.assert_called_with("http://example.com/download?fn=/file1.txt", stream=True, allow_redirects=True)
 281
 282         cm.open.assert_called_with("file1.txt", "wb")
 283         cm.save_new.assert_called_with(name="Downloaded from http://example.com/download?fn=/file1.txt",
 284                                        owner_uuid=None, ensure_unique_name=True)
 285
 286         api.collections().update.assert_has_calls([
 287             mock.call(uuid=cm.manifest_locator(),
 288                       body={"collection":{"properties": {"http://example.com/download?fn=/file1.txt": {'Date': 'Tue, 15 May 2018 00:00:00 GMT'}}}})
 289         ])