sdk/cwl/tests/test_http.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import json
  10 import logging
  11 import mock
  12 import sys
  13 import unittest
  14 import datetime
  15
  16 import arvados
  17 import arvados.collection
  18 import arvados_cwl
  19 import arvados_cwl.runner
  20 import arvados.keep
  21
  22 from .matcher import JsonDiffMatcher, StripYAMLComments
  23 from .mock_discovery import get_rootDesc
  24
  25 import arvados_cwl.http
  26
  27 import ruamel.yaml as yaml
  28
  29
  30 class TestHttpToKeep(unittest.TestCase):
  31
  32     @mock.patch("requests.get")
  33     @mock.patch("arvados.collection.Collection")
  34     def test_http_get(self, collectionmock, getmock):
  35         api = mock.MagicMock()
  36
  37         api.collections().list().execute.return_value = {
  38             "items": []
  39         }
  40
  41         cm = mock.MagicMock()
  42         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz3"
  43         cm.portable_data_hash.return_value = "99999999999999999999999999999998+99"
  44         collectionmock.return_value = cm
  45
  46         req = mock.MagicMock()
  47         req.status_code = 200
  48         req.headers = {}
  49         req.iter_content.return_value = ["abc"]
  50         getmock.return_value = req
  51
  52         utcnow = mock.MagicMock()
  53         utcnow.return_value = datetime.datetime(2018, 5, 15)
  54
  55         r = arvados_cwl.http.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
  56         self.assertEqual(r, "keep:99999999999999999999999999999998+99/file1.txt")
  57
  58         getmock.assert_called_with("http://example.com/file1.txt", stream=True, allow_redirects=True)
  59
  60         cm.open.assert_called_with("file1.txt", "w")
  61         cm.save_new.assert_called_with(name="Downloaded from http://example.com/file1.txt",
  62                                        owner_uuid=None, ensure_unique_name=True)
  63
  64         api.collections().update.assert_has_calls([
  65             mock.call(uuid=cm.manifest_locator(),
  66                       body={"collection":{"properties": {'http://example.com/file1.txt': {'Date': 'Tue, 15 May 2018 00:00:00 GMT'}}}})
  67         ])
  68
  69
  70     @mock.patch("requests.get")
  71     @mock.patch("arvados.collection.CollectionReader")
  72     def test_http_expires(self, collectionmock, getmock):
  73         api = mock.MagicMock()
  74
  75         api.collections().list().execute.return_value = {
  76             "items": [{
  77                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz3",
  78                 "portable_data_hash": "99999999999999999999999999999998+99",
  79                 "properties": {
  80                     'http://example.com/file1.txt': {
  81                         'Date': 'Tue, 15 May 2018 00:00:00 GMT',
  82                         'Expires': 'Tue, 17 May 2018 00:00:00 GMT'
  83                     }
  84                 }
  85             }]
  86         }
  87
  88         cm = mock.MagicMock()
  89         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz3"
  90         cm.portable_data_hash.return_value = "99999999999999999999999999999998+99"
  91         cm.keys.return_value = ["file1.txt"]
  92         collectionmock.return_value = cm
  93
  94         req = mock.MagicMock()
  95         req.status_code = 200
  96         req.headers = {}
  97         req.iter_content.return_value = ["abc"]
  98         getmock.return_value = req
  99
 100         utcnow = mock.MagicMock()
 101         utcnow.return_value = datetime.datetime(2018, 5, 16)
 102
 103         r = arvados_cwl.http.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
 104         self.assertEqual(r, "keep:99999999999999999999999999999998+99/file1.txt")
 105
 106         getmock.assert_not_called()
 107
 108
 109     @mock.patch("requests.get")
 110     @mock.patch("arvados.collection.CollectionReader")
 111     def test_http_cache_control(self, collectionmock, getmock):
 112         api = mock.MagicMock()
 113
 114         api.collections().list().execute.return_value = {
 115             "items": [{
 116                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz3",
 117                 "portable_data_hash": "99999999999999999999999999999998+99",
 118                 "properties": {
 119                     'http://example.com/file1.txt': {
 120                         'Date': 'Tue, 15 May 2018 00:00:00 GMT',
 121                         'Cache-Control': 'max-age=172800'
 122                     }
 123                 }
 124             }]
 125         }
 126
 127         cm = mock.MagicMock()
 128         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz3"
 129         cm.portable_data_hash.return_value = "99999999999999999999999999999998+99"
 130         cm.keys.return_value = ["file1.txt"]
 131         collectionmock.return_value = cm
 132
 133         req = mock.MagicMock()
 134         req.status_code = 200
 135         req.headers = {}
 136         req.iter_content.return_value = ["abc"]
 137         getmock.return_value = req
 138
 139         utcnow = mock.MagicMock()
 140         utcnow.return_value = datetime.datetime(2018, 5, 16)
 141
 142         r = arvados_cwl.http.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
 143         self.assertEqual(r, "keep:99999999999999999999999999999998+99/file1.txt")
 144
 145         getmock.assert_not_called()
 146
 147
 148     @mock.patch("requests.get")
 149     @mock.patch("requests.head")
 150     @mock.patch("arvados.collection.Collection")
 151     def test_http_expired(self, collectionmock, headmock, getmock):
 152         api = mock.MagicMock()
 153
 154         api.collections().list().execute.return_value = {
 155             "items": [{
 156                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz3",
 157                 "portable_data_hash": "99999999999999999999999999999998+99",
 158                 "properties": {
 159                     'http://example.com/file1.txt': {
 160                         'Date': 'Tue, 15 May 2018 00:00:00 GMT',
 161                         'Expires': 'Tue, 16 May 2018 00:00:00 GMT'
 162                     }
 163                 }
 164             }]
 165         }
 166
 167         cm = mock.MagicMock()
 168         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz4"
 169         cm.portable_data_hash.return_value = "99999999999999999999999999999997+99"
 170         cm.keys.return_value = ["file1.txt"]
 171         collectionmock.return_value = cm
 172
 173         req = mock.MagicMock()
 174         req.status_code = 200
 175         req.headers = {'Date': 'Tue, 17 May 2018 00:00:00 GMT'}
 176         req.iter_content.return_value = ["def"]
 177         getmock.return_value = req
 178         headmock.return_value = req
 179
 180         utcnow = mock.MagicMock()
 181         utcnow.return_value = datetime.datetime(2018, 5, 17)
 182
 183         r = arvados_cwl.http.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
 184         self.assertEqual(r, "keep:99999999999999999999999999999997+99/file1.txt")
 185
 186         getmock.assert_called_with("http://example.com/file1.txt", stream=True, allow_redirects=True)
 187
 188         cm.open.assert_called_with("file1.txt", "w")
 189         cm.save_new.assert_called_with(name="Downloaded from http://example.com/file1.txt",
 190                                        owner_uuid=None, ensure_unique_name=True)
 191
 192         api.collections().update.assert_has_calls([
 193             mock.call(uuid=cm.manifest_locator(),
 194                       body={"collection":{"properties": {'http://example.com/file1.txt': {'Date': 'Tue, 17 May 2018 00:00:00 GMT'}}}})
 195         ])
 196
 197
 198     @mock.patch("requests.get")
 199     @mock.patch("requests.head")
 200     @mock.patch("arvados.collection.CollectionReader")
 201     def test_http_etag(self, collectionmock, headmock, getmock):
 202         api = mock.MagicMock()
 203
 204         api.collections().list().execute.return_value = {
 205             "items": [{
 206                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz3",
 207                 "portable_data_hash": "99999999999999999999999999999998+99",
 208                 "properties": {
 209                     'http://example.com/file1.txt': {
 210                         'Date': 'Tue, 15 May 2018 00:00:00 GMT',
 211                         'Expires': 'Tue, 16 May 2018 00:00:00 GMT',
 212                         'ETag': '123456'
 213                     }
 214                 }
 215             }]
 216         }
 217
 218         cm = mock.MagicMock()
 219         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz3"
 220         cm.portable_data_hash.return_value = "99999999999999999999999999999998+99"
 221         cm.keys.return_value = ["file1.txt"]
 222         collectionmock.return_value = cm
 223
 224         req = mock.MagicMock()
 225         req.status_code = 200
 226         req.headers = {
 227             'Date': 'Tue, 17 May 2018 00:00:00 GMT',
 228             'Expires': 'Tue, 19 May 2018 00:00:00 GMT',
 229             'ETag': '123456'
 230         }
 231         headmock.return_value = req
 232
 233         utcnow = mock.MagicMock()
 234         utcnow.return_value = datetime.datetime(2018, 5, 17)
 235
 236         r = arvados_cwl.http.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
 237         self.assertEqual(r, "keep:99999999999999999999999999999998+99/file1.txt")
 238
 239         getmock.assert_not_called()
 240         cm.open.assert_not_called()
 241
 242         api.collections().update.assert_has_calls([
 243             mock.call(uuid=cm.manifest_locator(),
 244                       body={"collection":{"properties": {'http://example.com/file1.txt': {
 245                           'Date': 'Tue, 17 May 2018 00:00:00 GMT',
 246                           'Expires': 'Tue, 19 May 2018 00:00:00 GMT',
 247                           'ETag': '123456'
 248                       }}}})
 249                       ])
 250
 251     @mock.patch("requests.get")
 252     @mock.patch("arvados.collection.Collection")
 253     def test_http_content_disp(self, collectionmock, getmock):
 254         api = mock.MagicMock()
 255
 256         api.collections().list().execute.return_value = {
 257             "items": []
 258         }
 259
 260         cm = mock.MagicMock()
 261         cm.manifest_locator.return_value = "zzzzz-4zz18-zzzzzzzzzzzzzz3"
 262         cm.portable_data_hash.return_value = "99999999999999999999999999999998+99"
 263         collectionmock.return_value = cm
 264
 265         req = mock.MagicMock()
 266         req.status_code = 200
 267         req.headers = {"Content-Disposition": "attachment; filename=file1.txt"}
 268         req.iter_content.return_value = ["abc"]
 269         getmock.return_value = req
 270
 271         utcnow = mock.MagicMock()
 272         utcnow.return_value = datetime.datetime(2018, 5, 15)
 273
 274         r = arvados_cwl.http.http_to_keep(api, None, "http://example.com/download?fn=/file1.txt", utcnow=utcnow)
 275         self.assertEqual(r, "keep:99999999999999999999999999999998+99/file1.txt")
 276
 277         getmock.assert_called_with("http://example.com/download?fn=/file1.txt", stream=True, allow_redirects=True)
 278
 279         cm.open.assert_called_with("file1.txt", "w")
 280         cm.save_new.assert_called_with(name="Downloaded from http://example.com/download?fn=/file1.txt",
 281                                        owner_uuid=None, ensure_unique_name=True)
 282
 283         api.collections().update.assert_has_calls([
 284             mock.call(uuid=cm.manifest_locator(),
 285                       body={"collection":{"properties": {"http://example.com/download?fn=/file1.txt": {'Date': 'Tue, 15 May 2018 00:00:00 GMT'}}}})
 286         ])