sdk/cwl/tests/test_http.py

   1 from future import standard_library
   2 standard_library.install_aliases()
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: Apache-2.0
   6
   7 import copy
   8 import io
   9 import functools
  10 import hashlib
  11 import json
  12 import logging
  13 import mock
  14 import sys
  15 import unittest
  16 import datetime
  17
  18 import arvados
  19 import arvados.collection
  20 import arvados_cwl
  21 import arvados_cwl.runner
  22 import arvados.keep
  23
  24 from .matcher import JsonDiffMatcher, StripYAMLComments
  25 from .mock_discovery import get_rootDesc
  26
  27 import arvados_cwl.http
  28
  29 import ruamel.yaml as yaml
  30
  31
  32 class TestHttpToKeep(unittest.TestCase):
  33
  34     @mock.patch("requests.get")
  35     @mock.patch("arvados.collection.Collection")
  36     def test_http_get(self, collectionmock, getmock):
  37         api = mock.MagicMock()
  38
  39         api.collections().list().execute.return_value = {
  40             "items": []
  41         }
  42
  43         cm = mock.MagicMock()
  44         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz3"
  45         cm.portable_data_hash.return_value = "99999999999999999999999999999998+99"
  46         collectionmock.return_value = cm
  47
  48         req = mock.MagicMock()
  49         req.status_code = 200
  50         req.headers = {}
  51         req.iter_content.return_value = ["abc"]
  52         getmock.return_value = req
  53
  54         utcnow = mock.MagicMock()
  55         utcnow.return_value = datetime.datetime(2018, 5, 15)
  56
  57         r = arvados_cwl.http.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
  58         self.assertEqual(r, "keep:99999999999999999999999999999998+99/file1.txt")
  59
  60         getmock.assert_called_with("http://example.com/file1.txt", stream=True, allow_redirects=True)
  61
  62         cm.open.assert_called_with("file1.txt", "w")
  63         cm.save_new.assert_called_with(name="Downloaded from http://example.com/file1.txt",
  64                                        owner_uuid=None, ensure_unique_name=True)
  65
  66         api.collections().update.assert_has_calls([
  67             mock.call(uuid=cm.manifest_locator(),
  68                       body={"collection":{"properties": {'http://example.com/file1.txt': {'Date': 'Tue, 15 May 2018 00:00:00 GMT'}}}})
  69         ])
  70
  71
  72     @mock.patch("requests.get")
  73     @mock.patch("arvados.collection.CollectionReader")
  74     def test_http_expires(self, collectionmock, getmock):
  75         api = mock.MagicMock()
  76
  77         api.collections().list().execute.return_value = {
  78             "items": [{
  79                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz3",
  80                 "portable_data_hash": "99999999999999999999999999999998+99",
  81                 "properties": {
  82                     'http://example.com/file1.txt': {
  83                         'Date': 'Tue, 15 May 2018 00:00:00 GMT',
  84                         'Expires': 'Tue, 17 May 2018 00:00:00 GMT'
  85                     }
  86                 }
  87             }]
  88         }
  89
  90         cm = mock.MagicMock()
  91         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz3"
  92         cm.portable_data_hash.return_value = "99999999999999999999999999999998+99"
  93         cm.keys.return_value = ["file1.txt"]
  94         collectionmock.return_value = cm
  95
  96         req = mock.MagicMock()
  97         req.status_code = 200
  98         req.headers = {}
  99         req.iter_content.return_value = ["abc"]
 100         getmock.return_value = req
 101
 102         utcnow = mock.MagicMock()
 103         utcnow.return_value = datetime.datetime(2018, 5, 16)
 104
 105         r = arvados_cwl.http.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
 106         self.assertEqual(r, "keep:99999999999999999999999999999998+99/file1.txt")
 107
 108         getmock.assert_not_called()
 109
 110
 111     @mock.patch("requests.get")
 112     @mock.patch("arvados.collection.CollectionReader")
 113     def test_http_cache_control(self, collectionmock, getmock):
 114         api = mock.MagicMock()
 115
 116         api.collections().list().execute.return_value = {
 117             "items": [{
 118                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz3",
 119                 "portable_data_hash": "99999999999999999999999999999998+99",
 120                 "properties": {
 121                     'http://example.com/file1.txt': {
 122                         'Date': 'Tue, 15 May 2018 00:00:00 GMT',
 123                         'Cache-Control': 'max-age=172800'
 124                     }
 125                 }
 126             }]
 127         }
 128
 129         cm = mock.MagicMock()
 130         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz3"
 131         cm.portable_data_hash.return_value = "99999999999999999999999999999998+99"
 132         cm.keys.return_value = ["file1.txt"]
 133         collectionmock.return_value = cm
 134
 135         req = mock.MagicMock()
 136         req.status_code = 200
 137         req.headers = {}
 138         req.iter_content.return_value = ["abc"]
 139         getmock.return_value = req
 140
 141         utcnow = mock.MagicMock()
 142         utcnow.return_value = datetime.datetime(2018, 5, 16)
 143
 144         r = arvados_cwl.http.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
 145         self.assertEqual(r, "keep:99999999999999999999999999999998+99/file1.txt")
 146
 147         getmock.assert_not_called()
 148
 149
 150     @mock.patch("requests.get")
 151     @mock.patch("requests.head")
 152     @mock.patch("arvados.collection.Collection")
 153     def test_http_expired(self, collectionmock, headmock, getmock):
 154         api = mock.MagicMock()
 155
 156         api.collections().list().execute.return_value = {
 157             "items": [{
 158                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz3",
 159                 "portable_data_hash": "99999999999999999999999999999998+99",
 160                 "properties": {
 161                     'http://example.com/file1.txt': {
 162                         'Date': 'Tue, 15 May 2018 00:00:00 GMT',
 163                         'Expires': 'Tue, 16 May 2018 00:00:00 GMT'
 164                     }
 165                 }
 166             }]
 167         }
 168
 169         cm = mock.MagicMock()
 170         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz4"
 171         cm.portable_data_hash.return_value = "99999999999999999999999999999997+99"
 172         cm.keys.return_value = ["file1.txt"]
 173         collectionmock.return_value = cm
 174
 175         req = mock.MagicMock()
 176         req.status_code = 200
 177         req.headers = {'Date': 'Tue, 17 May 2018 00:00:00 GMT'}
 178         req.iter_content.return_value = ["def"]
 179         getmock.return_value = req
 180         headmock.return_value = req
 181
 182         utcnow = mock.MagicMock()
 183         utcnow.return_value = datetime.datetime(2018, 5, 17)
 184
 185         r = arvados_cwl.http.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
 186         self.assertEqual(r, "keep:99999999999999999999999999999997+99/file1.txt")
 187
 188         getmock.assert_called_with("http://example.com/file1.txt", stream=True, allow_redirects=True)
 189
 190         cm.open.assert_called_with("file1.txt", "w")
 191         cm.save_new.assert_called_with(name="Downloaded from http://example.com/file1.txt",
 192                                        owner_uuid=None, ensure_unique_name=True)
 193
 194         api.collections().update.assert_has_calls([
 195             mock.call(uuid=cm.manifest_locator(),
 196                       body={"collection":{"properties": {'http://example.com/file1.txt': {'Date': 'Tue, 17 May 2018 00:00:00 GMT'}}}})
 197         ])
 198
 199
 200     @mock.patch("requests.get")
 201     @mock.patch("requests.head")
 202     @mock.patch("arvados.collection.CollectionReader")
 203     def test_http_etag(self, collectionmock, headmock, getmock):
 204         api = mock.MagicMock()
 205
 206         api.collections().list().execute.return_value = {
 207             "items": [{
 208                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz3",
 209                 "portable_data_hash": "99999999999999999999999999999998+99",
 210                 "properties": {
 211                     'http://example.com/file1.txt': {
 212                         'Date': 'Tue, 15 May 2018 00:00:00 GMT',
 213                         'Expires': 'Tue, 16 May 2018 00:00:00 GMT',
 214                         'ETag': '123456'
 215                     }
 216                 }
 217             }]
 218         }
 219
 220         cm = mock.MagicMock()
 221         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz3"
 222         cm.portable_data_hash.return_value = "99999999999999999999999999999998+99"
 223         cm.keys.return_value = ["file1.txt"]
 224         collectionmock.return_value = cm
 225
 226         req = mock.MagicMock()
 227         req.status_code = 200
 228         req.headers = {
 229             'Date': 'Tue, 17 May 2018 00:00:00 GMT',
 230             'Expires': 'Tue, 19 May 2018 00:00:00 GMT',
 231             'ETag': '123456'
 232         }
 233         headmock.return_value = req
 234
 235         utcnow = mock.MagicMock()
 236         utcnow.return_value = datetime.datetime(2018, 5, 17)
 237
 238         r = arvados_cwl.http.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
 239         self.assertEqual(r, "keep:99999999999999999999999999999998+99/file1.txt")
 240
 241         getmock.assert_not_called()
 242         cm.open.assert_not_called()
 243
 244         api.collections().update.assert_has_calls([
 245             mock.call(uuid=cm.manifest_locator(),
 246                       body={"collection":{"properties": {'http://example.com/file1.txt': {
 247                           'Date': 'Tue, 17 May 2018 00:00:00 GMT',
 248                           'Expires': 'Tue, 19 May 2018 00:00:00 GMT',
 249                           'ETag': '123456'
 250                       }}}})
 251                       ])
 252
 253     @mock.patch("requests.get")
 254     @mock.patch("arvados.collection.Collection")
 255     def test_http_content_disp(self, collectionmock, getmock):
 256         api = mock.MagicMock()
 257
 258         api.collections().list().execute.return_value = {
 259             "items": []
 260         }
 261
 262         cm = mock.MagicMock()
 263         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz3"
 264         cm.portable_data_hash.return_value = "99999999999999999999999999999998+99"
 265         collectionmock.return_value = cm
 266
 267         req = mock.MagicMock()
 268         req.status_code = 200
 269         req.headers = {"Content-Disposition": "attachment; filename=file1.txt"}
 270         req.iter_content.return_value = ["abc"]
 271         getmock.return_value = req
 272
 273         utcnow = mock.MagicMock()
 274         utcnow.return_value = datetime.datetime(2018, 5, 15)
 275
 276         r = arvados_cwl.http.http_to_keep(api, None, "http://example.com/download?fn=/file1.txt", utcnow=utcnow)
 277         self.assertEqual(r, "keep:99999999999999999999999999999998+99/file1.txt")
 278
 279         getmock.assert_called_with("http://example.com/download?fn=/file1.txt", stream=True, allow_redirects=True)
 280
 281         cm.open.assert_called_with("file1.txt", "w")
 282         cm.save_new.assert_called_with(name="Downloaded from http://example.com/download?fn=/file1.txt",
 283                                        owner_uuid=None, ensure_unique_name=True)
 284
 285         api.collections().update.assert_has_calls([
 286             mock.call(uuid=cm.manifest_locator(),
 287                       body={"collection":{"properties": {"http://example.com/download?fn=/file1.txt": {'Date': 'Tue, 15 May 2018 00:00:00 GMT'}}}})
 288         ])