sdk/python/tests/test_arvfile.py

   1 #!/usr/bin/env python
   2
   3 import bz2
   4 import datetime
   5 import gzip
   6 import io
   7 import mock
   8 import os
   9 import unittest
  10 import time
  11
  12 import arvados
  13 from arvados._ranges import Range
  14 from arvados.keep import KeepLocator
  15 from arvados.collection import Collection, CollectionReader
  16 from arvados.arvfile import ArvadosFile, ArvadosFileReader
  17
  18 import arvados_testutil as tutil
  19 from test_stream import StreamFileReaderTestCase, StreamRetryTestMixin
  20
  21 class ArvadosFileWriterTestCase(unittest.TestCase):
  22     class MockKeep(object):
  23         def __init__(self, blocks):
  24             self.blocks = blocks
  25             self.requests = []
  26         def get(self, locator, num_retries=0):
  27             self.requests.append(locator)
  28             return self.blocks.get(locator)
  29         def get_from_cache(self, locator):
  30             self.requests.append(locator)
  31             return self.blocks.get(locator)
  32         def put(self, data, num_retries=None, copies=None):
  33             pdh = tutil.str_keep_locator(data)
  34             self.blocks[pdh] = str(data)
  35             return pdh
  36
  37     class MockApi(object):
  38         def __init__(self, b, r):
  39             self.body = b
  40             self.response = r
  41             self._schema = ArvadosFileWriterTestCase.MockApi.MockSchema()
  42             self._rootDesc = {}
  43         class MockSchema(object):
  44             def __init__(self):
  45                 self.schemas = {'Collection': {'properties': {'replication_desired': {'type':'integer'}}}}
  46         class MockCollections(object):
  47             def __init__(self, b, r):
  48                 self.body = b
  49                 self.response = r
  50             class Execute(object):
  51                 def __init__(self, r):
  52                     self.response = r
  53                 def execute(self, num_retries=None):
  54                     return self.response
  55             def create(self, ensure_unique_name=False, body=None):
  56                 if body != self.body:
  57                     raise Exception("Body %s does not match expectation %s" % (body, self.body))
  58                 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
  59             def update(self, uuid=None, body=None):
  60                 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
  61         def collections(self):
  62             return ArvadosFileWriterTestCase.MockApi.MockCollections(self.body, self.response)
  63
  64
  65     def test_truncate(self):
  66         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
  67         api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate",
  68                                                  "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
  69                                                  "replication_desired":None},
  70                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
  71                                                  "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
  72                                                  "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52"})
  73         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
  74                              api_client=api, keep_client=keep) as c:
  75             writer = c.open("count.txt", "r+")
  76             self.assertEqual(writer.size(), 10)
  77             self.assertEqual("0123456789", writer.read(12))
  78
  79             writer.truncate(8)
  80
  81             # Make sure reading off the end doesn't break
  82             self.assertEqual("", writer.read(12))
  83
  84             self.assertEqual(writer.size(), 8)
  85             writer.seek(0, os.SEEK_SET)
  86             self.assertEqual("01234567", writer.read(12))
  87
  88             self.assertIsNone(c.manifest_locator())
  89             self.assertTrue(c.modified())
  90             c.save_new("test_truncate")
  91             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
  92             self.assertFalse(c.modified())
  93
  94     def test_write_to_end(self):
  95         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
  96         api = ArvadosFileWriterTestCase.MockApi({"name":"test_append",
  97                                                  "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
  98                                                  "replication_desired":None},
  99                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 100                                                  "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
 101                                                  "portable_data_hash":"c5c3af76565c8efb6a806546bcf073f3+88"})
 102         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 103                              api_client=api, keep_client=keep) as c:
 104             writer = c.open("count.txt", "r+")
 105             self.assertEqual(writer.size(), 10)
 106
 107             writer.seek(5, os.SEEK_SET)
 108             self.assertEqual("56789", writer.read(8))
 109
 110             writer.seek(10, os.SEEK_SET)
 111             writer.write("foo")
 112             self.assertEqual(writer.size(), 13)
 113
 114             writer.seek(5, os.SEEK_SET)
 115             self.assertEqual("56789foo", writer.read(8))
 116
 117             self.assertIsNone(c.manifest_locator())
 118             self.assertTrue(c.modified())
 119             self.assertIsNone(keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
 120
 121             c.save_new("test_append")
 122             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 123             self.assertFalse(c.modified())
 124             self.assertEqual("foo", keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
 125
 126
 127     def test_append(self):
 128         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 129         c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
 130         writer = c.open("count.txt", "a+")
 131         self.assertEqual(writer.read(20), "0123456789")
 132         writer.seek(0, os.SEEK_SET)
 133
 134         writer.write("hello")
 135         self.assertEqual(writer.read(20), "0123456789hello")
 136         writer.seek(0, os.SEEK_SET)
 137
 138         writer.write("world")
 139         self.assertEqual(writer.read(20), "0123456789helloworld")
 140
 141         self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.portable_manifest_text())
 142
 143     def test_write_at_beginning(self):
 144         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 145         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 146                              keep_client=keep) as c:
 147             writer = c.open("count.txt", "r+")
 148             self.assertEqual("0123456789", writer.readfrom(0, 13))
 149             writer.seek(0, os.SEEK_SET)
 150             writer.write("foo")
 151             self.assertEqual(writer.size(), 10)
 152             self.assertEqual("foo3456789", writer.readfrom(0, 13))
 153             self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", c.portable_manifest_text())
 154
 155     def test_write_empty(self):
 156         keep = ArvadosFileWriterTestCase.MockKeep({})
 157         with Collection(keep_client=keep) as c:
 158             writer = c.open("count.txt", "w")
 159             self.assertEqual(writer.size(), 0)
 160             self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:count.txt\n", c.portable_manifest_text())
 161
 162     def test_save_manifest_text(self):
 163         keep = ArvadosFileWriterTestCase.MockKeep({})
 164         with Collection(keep_client=keep) as c:
 165             writer = c.open("count.txt", "w")
 166             writer.write("0123456789")
 167             self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
 168             self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 169
 170             self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.save_new(create_collection_record=False))
 171             self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 172
 173     def test_get_manifest_text_commits(self):
 174          keep = ArvadosFileWriterTestCase.MockKeep({})
 175          with Collection(keep_client=keep) as c:
 176              writer = c.open("count.txt", "w")
 177              writer.write("0123456789")
 178              self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
 179              self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 180              self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.manifest_text())
 181              self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 182
 183
 184     def test_write_in_middle(self):
 185         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 186         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 187                              keep_client=keep) as c:
 188             writer = c.open("count.txt", "r+")
 189             self.assertEqual("0123456789", writer.readfrom(0, 13))
 190             writer.seek(3, os.SEEK_SET)
 191             writer.write("foo")
 192             self.assertEqual(writer.size(), 10)
 193             self.assertEqual("012foo6789", writer.readfrom(0, 13))
 194             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", c.portable_manifest_text())
 195
 196     def test_write_at_end(self):
 197         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 198         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 199                              keep_client=keep) as c:
 200             writer = c.open("count.txt", "r+")
 201             self.assertEqual("0123456789", writer.readfrom(0, 13))
 202             writer.seek(7, os.SEEK_SET)
 203             writer.write("foo")
 204             self.assertEqual(writer.size(), 10)
 205             self.assertEqual("0123456foo", writer.readfrom(0, 13))
 206             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", c.portable_manifest_text())
 207
 208     def test_write_across_segment_boundary(self):
 209         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 210         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
 211                              keep_client=keep) as c:
 212             writer = c.open("count.txt", "r+")
 213             self.assertEqual("012345678901234", writer.readfrom(0, 15))
 214             writer.seek(7, os.SEEK_SET)
 215             writer.write("foobar")
 216             self.assertEqual(writer.size(), 20)
 217             self.assertEqual("0123456foobar34", writer.readfrom(0, 15))
 218             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", c.portable_manifest_text())
 219
 220     def test_write_across_several_segments(self):
 221         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 222         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
 223                              keep_client=keep) as c:
 224             writer = c.open("count.txt", "r+")
 225             self.assertEqual("012301230123", writer.readfrom(0, 15))
 226             writer.seek(2, os.SEEK_SET)
 227             writer.write("abcdefg")
 228             self.assertEqual(writer.size(), 12)
 229             self.assertEqual("01abcdefg123", writer.readfrom(0, 15))
 230             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", c.portable_manifest_text())
 231
 232     def test_write_large(self):
 233         keep = ArvadosFileWriterTestCase.MockKeep({})
 234         api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
 235                                                  "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n",
 236                                                  "replication_desired":None},
 237                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 238                                                  "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n",
 239                                                  "portable_data_hash":"9132ca8e3f671c76103a38f5bc24328c+108"})
 240         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 241                              api_client=api, keep_client=keep) as c:
 242             writer = c.open("count.txt", "r+")
 243             text = "0123456789" * 100
 244             for b in xrange(0, 100000):
 245                 writer.write(text)
 246             self.assertEqual(writer.size(), 100000000)
 247
 248             self.assertIsNone(c.manifest_locator())
 249             self.assertTrue(c.modified())
 250             c.save_new("test_write_large")
 251             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 252             self.assertFalse(c.modified())
 253
 254
 255     def test_large_write(self):
 256         keep = ArvadosFileWriterTestCase.MockKeep({})
 257         api = ArvadosFileWriterTestCase.MockApi({}, {})
 258         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 259                              api_client=api, keep_client=keep) as c:
 260             writer = c.open("count.txt", "r+")
 261             self.assertEqual(writer.size(), 0)
 262
 263             text = "0123456789"
 264             writer.write(text)
 265             text = "0123456789" * 9999999
 266             writer.write(text)
 267             self.assertEqual(writer.size(), 100000000)
 268
 269             self.assertEqual(c.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 48dd23ea1645fd47d789804d71b5bb8e+67108864 77c57dc6ac5a10bb2205caaa73187994+32891126 0:100000000:count.txt\n")
 270
 271     def test_rewrite_on_empty_file(self):
 272         keep = ArvadosFileWriterTestCase.MockKeep({})
 273         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 274                              keep_client=keep) as c:
 275             writer = c.open("count.txt", "r+")
 276             for b in xrange(0, 10):
 277                 writer.seek(0, os.SEEK_SET)
 278                 writer.write("0123456789")
 279
 280             self.assertEqual(writer.size(), 10)
 281             self.assertEqual("0123456789", writer.readfrom(0, 20))
 282             self.assertEqual(". 7a08b07e84641703e5f2c836aa59a170+100 90:10:count.txt\n", c.portable_manifest_text())
 283             writer.flush()
 284             self.assertEqual(writer.size(), 10)
 285             self.assertEqual("0123456789", writer.readfrom(0, 20))
 286             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.portable_manifest_text())
 287
 288     def test_rewrite_append_existing_file(self):
 289         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 290         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
 291                              keep_client=keep) as c:
 292             writer = c.open("count.txt", "r+")
 293             for b in xrange(0, 10):
 294                 writer.seek(10, os.SEEK_SET)
 295                 writer.write("abcdefghij")
 296
 297             self.assertEqual(writer.size(), 20)
 298             self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
 299             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:10:count.txt 100:10:count.txt\n", c.portable_manifest_text())
 300
 301             writer.arvadosfile.flush()
 302             self.assertEqual(writer.size(), 20)
 303             self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
 304             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.portable_manifest_text())
 305
 306     def test_rewrite_over_existing_file(self):
 307         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 308         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
 309                              keep_client=keep) as c:
 310             writer = c.open("count.txt", "r+")
 311             for b in xrange(0, 10):
 312                 writer.seek(5, os.SEEK_SET)
 313                 writer.write("abcdefghij")
 314
 315             self.assertEqual(writer.size(), 15)
 316             self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
 317             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:5:count.txt 100:10:count.txt\n", c.portable_manifest_text())
 318
 319             writer.arvadosfile.flush()
 320
 321             self.assertEqual(writer.size(), 15)
 322             self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
 323             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.portable_manifest_text())
 324
 325     def test_write_large_rewrite(self):
 326         keep = ArvadosFileWriterTestCase.MockKeep({})
 327         api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
 328                                                  "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n",
 329                                                  "replication_desired":None},
 330                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 331                                                  "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n",
 332                                                  "portable_data_hash":"217665c6b713e1b78dfba7ebd42344db+156"})
 333         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 334                              api_client=api, keep_client=keep) as c:
 335             writer = c.open("count.txt", "r+")
 336             text = ''.join(["0123456789" for a in xrange(0, 100)])
 337             for b in xrange(0, 100000):
 338                 writer.write(text)
 339             writer.seek(0, os.SEEK_SET)
 340             writer.write("foo")
 341             self.assertEqual(writer.size(), 100000000)
 342
 343             self.assertIsNone(c.manifest_locator())
 344             self.assertTrue(c.modified())
 345             c.save_new("test_write_large")
 346             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 347             self.assertFalse(c.modified())
 348
 349     def test_create(self):
 350         keep = ArvadosFileWriterTestCase.MockKeep({})
 351         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
 352                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 353                                                  "replication_desired":None},
 354                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 355                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 356                                                  "portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51"})
 357         with Collection(api_client=api, keep_client=keep) as c:
 358             writer = c.open("count.txt", "w+")
 359             self.assertEqual(writer.size(), 0)
 360             writer.write("01234567")
 361             self.assertEqual(writer.size(), 8)
 362
 363             self.assertIsNone(c.manifest_locator())
 364             self.assertTrue(c.modified())
 365             self.assertIsNone(keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 366             c.save_new("test_create")
 367             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 368             self.assertFalse(c.modified())
 369             self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 370
 371
 372     def test_create_subdir(self):
 373         keep = ArvadosFileWriterTestCase.MockKeep({})
 374         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
 375                                                  "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 376                                                  "replication_desired":None},
 377                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 378                                                  "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 379                                                  "portable_data_hash":"1b02aaa62528d28a5be41651cbb9d7c7+59"})
 380         with Collection(api_client=api, keep_client=keep) as c:
 381             self.assertIsNone(c.api_response())
 382             writer = c.open("foo/bar/count.txt", "w+")
 383             writer.write("01234567")
 384             self.assertFalse(c.committed())
 385             c.save_new("test_create")
 386             self.assertTrue(c.committed())
 387             self.assertEqual(c.api_response(), api.response)
 388
 389     def test_overwrite(self):
 390         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 391         api = ArvadosFileWriterTestCase.MockApi({"name":"test_overwrite",
 392                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 393                                                  "replication_desired":None},
 394                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 395                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 396                                                  "portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51"})
 397         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 398                              api_client=api, keep_client=keep) as c:
 399             writer = c.open("count.txt", "w+")
 400             self.assertEqual(writer.size(), 0)
 401             writer.write("01234567")
 402             self.assertEqual(writer.size(), 8)
 403
 404             self.assertIsNone(c.manifest_locator())
 405             self.assertTrue(c.modified())
 406             c.save_new("test_overwrite")
 407             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 408             self.assertFalse(c.modified())
 409
 410     def test_file_not_found(self):
 411         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
 412             with self.assertRaises(IOError):
 413                 writer = c.open("nocount.txt", "r")
 414
 415     def test_cannot_open_directory(self):
 416         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
 417             with self.assertRaises(IOError):
 418                 writer = c.open(".", "r")
 419
 420     def test_create_multiple(self):
 421         keep = ArvadosFileWriterTestCase.MockKeep({})
 422         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create_multiple",
 423                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n",
 424                                                  "replication_desired":None},
 425                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 426                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n",
 427                                                  "portable_data_hash":"71e7bb6c00d31fc2b4364199fd97be08+102"})
 428         with Collection(api_client=api, keep_client=keep) as c:
 429             w1 = c.open("count1.txt", "w")
 430             w2 = c.open("count2.txt", "w")
 431             w1.write("01234567")
 432             w2.write("abcdefgh")
 433             self.assertEqual(w1.size(), 8)
 434             self.assertEqual(w2.size(), 8)
 435
 436             self.assertIsNone(c.manifest_locator())
 437             self.assertTrue(c.modified())
 438             self.assertIsNone(keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 439             c.save_new("test_create_multiple")
 440             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 441             self.assertFalse(c.modified())
 442             self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 443
 444
 445 class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
 446     class MockParent(object):
 447         class MockBlockMgr(object):
 448             def __init__(self, blocks, nocache):
 449                 self.blocks = blocks
 450                 self.nocache = nocache
 451
 452             def block_prefetch(self, loc):
 453                 pass
 454
 455             def get_block_contents(self, loc, num_retries=0, cache_only=False):
 456                 if self.nocache and cache_only:
 457                     return None
 458                 return self.blocks[loc]
 459
 460         def __init__(self, blocks, nocache):
 461             self.blocks = blocks
 462             self.nocache = nocache
 463             self.lock = arvados.arvfile.NoopLock()
 464
 465         def root_collection(self):
 466             return self
 467
 468         def _my_block_manager(self):
 469             return ArvadosFileReaderTestCase.MockParent.MockBlockMgr(self.blocks, self.nocache)
 470
 471
 472     def make_count_reader(self, nocache=False):
 473         stream = []
 474         n = 0
 475         blocks = {}
 476         for d in ['01234', '34567', '67890']:
 477             loc = tutil.str_keep_locator(d)
 478             blocks[loc] = d
 479             stream.append(Range(loc, n, len(d)))
 480             n += len(d)
 481         af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), "count.txt", stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
 482         return ArvadosFileReader(af)
 483
 484     def test_read_block_crossing_behavior(self):
 485         # read() needs to return all the data requested if possible, even if it
 486         # crosses uncached blocks: https://arvados.org/issues/5856
 487         sfile = self.make_count_reader(nocache=True)
 488         self.assertEqual('12345678', sfile.read(8))
 489
 490     def test_successive_reads(self):
 491         # Override StreamFileReaderTestCase.test_successive_reads
 492         sfile = self.make_count_reader(nocache=True)
 493         self.assertEqual('1234', sfile.read(4))
 494         self.assertEqual('5678', sfile.read(4))
 495         self.assertEqual('9', sfile.read(4))
 496         self.assertEqual('', sfile.read(4))
 497
 498     def test_tell_after_block_read(self):
 499         # Override StreamFileReaderTestCase.test_tell_after_block_read
 500         sfile = self.make_count_reader(nocache=True)
 501         self.assertEqual('12345678', sfile.read(8))
 502         self.assertEqual(8, sfile.tell())
 503
 504     def test_prefetch(self):
 505         keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"})
 506         with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
 507             r = c.open("count.txt", "r")
 508             self.assertEqual("0123", r.read(4))
 509         self.assertIn("2e9ec317e197819358fbc43afca7d837+8", keep.requests)
 510         self.assertIn("e8dc4081b13434b45189a720b77b6818+8", keep.requests)
 511
 512     def test__eq__from_manifest(self):
 513         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 514             with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
 515                 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
 516                 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
 517
 518     def test__eq__from_writes(self):
 519         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 520             with Collection() as c2:
 521                 f = c2.open("count1.txt", "w")
 522                 f.write("0123456789")
 523
 524                 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
 525                 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
 526
 527     def test__ne__(self):
 528         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 529             with Collection() as c2:
 530                 f = c2.open("count1.txt", "w")
 531                 f.write("1234567890")
 532
 533                 self.assertTrue(c1["count1.txt"] != c2["count1.txt"])
 534                 self.assertFalse(c1["count1.txt"] == c2["count1.txt"])
 535
 536
 537 class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
 538     def reader_for(self, coll_name, **kwargs):
 539         stream = []
 540         segments = []
 541         n = 0
 542         for d in self.manifest_for(coll_name).split():
 543             try:
 544                 k = KeepLocator(d)
 545                 segments.append(Range(n, n, k.size))
 546                 stream.append(Range(d, n, k.size))
 547                 n += k.size
 548             except ValueError:
 549                 pass
 550
 551         blockmanager = arvados.arvfile._BlockManager(self.keep_client())
 552         blockmanager.prefetch_enabled = False
 553         col = Collection(keep_client=self.keep_client(), block_manager=blockmanager)
 554         af = ArvadosFile(col, "test",
 555                          stream=stream,
 556                          segments=segments)
 557         return ArvadosFileReader(af, **kwargs)
 558
 559     def read_for_test(self, reader, byte_count, **kwargs):
 560         return reader.read(byte_count, **kwargs)
 561
 562
 563 class ArvadosFileReadFromTestCase(ArvadosFileReadTestCase):
 564     def read_for_test(self, reader, byte_count, **kwargs):
 565         return reader.readfrom(0, byte_count, **kwargs)
 566
 567
 568 class ArvadosFileReadAllTestCase(ArvadosFileReadTestCase):
 569     def read_for_test(self, reader, byte_count, **kwargs):
 570         return ''.join(reader.readall(**kwargs))
 571
 572
 573 class ArvadosFileReadAllDecompressedTestCase(ArvadosFileReadTestCase):
 574     def read_for_test(self, reader, byte_count, **kwargs):
 575         return ''.join(reader.readall_decompressed(**kwargs))
 576
 577
 578 class ArvadosFileReadlinesTestCase(ArvadosFileReadTestCase):
 579     def read_for_test(self, reader, byte_count, **kwargs):
 580         return ''.join(reader.readlines(**kwargs))
 581
 582
 583 class ArvadosFileTestCase(unittest.TestCase):
 584     def datetime_to_hex(self, dt):
 585         return hex(int(time.mktime(dt.timetuple())))[2:]
 586
 587     def test_permission_expired(self):
 588         base_manifest = ". 781e5e245d69b566979b86e28d23f2c7+10+A715fd31f8111894f717eb1003c1b0216799dd9ec@{} 0:10:count.txt\n"
 589         now = datetime.datetime.now()
 590         a_week_ago = now - datetime.timedelta(days=7)
 591         a_month_ago = now - datetime.timedelta(days=30)
 592         a_week_from_now = now + datetime.timedelta(days=7)
 593         with Collection(base_manifest.format(self.datetime_to_hex(a_week_from_now))) as c:
 594             self.assertFalse(c.find('count.txt').permission_expired())
 595         with Collection(base_manifest.format(self.datetime_to_hex(a_week_ago))) as c:
 596             f = c.find('count.txt')
 597             self.assertTrue(f.permission_expired())
 598             self.assertTrue(f.permission_expired(a_week_from_now))
 599             self.assertFalse(f.permission_expired(a_month_ago))
 600
 601
 602 class BlockManagerTest(unittest.TestCase):
 603     def test_bufferblock_append(self):
 604         keep = ArvadosFileWriterTestCase.MockKeep({})
 605         with arvados.arvfile._BlockManager(keep) as blockmanager:
 606             bufferblock = blockmanager.alloc_bufferblock()
 607             bufferblock.append("foo")
 608
 609             self.assertEqual(bufferblock.size(), 3)
 610             self.assertEqual(bufferblock.buffer_view[0:3], "foo")
 611             self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 612
 613             bufferblock.append("bar")
 614
 615             self.assertEqual(bufferblock.size(), 6)
 616             self.assertEqual(bufferblock.buffer_view[0:6], "foobar")
 617             self.assertEqual(bufferblock.locator(), "3858f62230ac3c915f300c664312c63f+6")
 618
 619             bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
 620             with self.assertRaises(arvados.errors.AssertionError):
 621                 bufferblock.append("bar")
 622
 623     def test_bufferblock_dup(self):
 624         keep = ArvadosFileWriterTestCase.MockKeep({})
 625         with arvados.arvfile._BlockManager(keep) as blockmanager:
 626             bufferblock = blockmanager.alloc_bufferblock()
 627             bufferblock.append("foo")
 628
 629             self.assertEqual(bufferblock.size(), 3)
 630             self.assertEqual(bufferblock.buffer_view[0:3], "foo")
 631             self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 632             bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
 633
 634             bufferblock2 = blockmanager.dup_block(bufferblock, None)
 635             self.assertNotEqual(bufferblock.blockid, bufferblock2.blockid)
 636
 637             bufferblock2.append("bar")
 638
 639             self.assertEqual(bufferblock2.size(), 6)
 640             self.assertEqual(bufferblock2.buffer_view[0:6], "foobar")
 641             self.assertEqual(bufferblock2.locator(), "3858f62230ac3c915f300c664312c63f+6")
 642
 643             self.assertEqual(bufferblock.size(), 3)
 644             self.assertEqual(bufferblock.buffer_view[0:3], "foo")
 645             self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 646
 647     def test_bufferblock_get(self):
 648         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 649         with arvados.arvfile._BlockManager(keep) as blockmanager:
 650             bufferblock = blockmanager.alloc_bufferblock()
 651             bufferblock.append("foo")
 652
 653             self.assertEqual(blockmanager.get_block_contents("781e5e245d69b566979b86e28d23f2c7+10", 1), "0123456789")
 654             self.assertEqual(blockmanager.get_block_contents(bufferblock.blockid, 1), "foo")
 655
 656     def test_bufferblock_commit(self):
 657         mockkeep = mock.MagicMock()
 658         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
 659             bufferblock = blockmanager.alloc_bufferblock()
 660             bufferblock.owner = mock.MagicMock()
 661             def flush(sync=None):
 662                 blockmanager.commit_bufferblock(bufferblock, sync)
 663             bufferblock.owner.flush.side_effect = flush
 664             bufferblock.append("foo")
 665             blockmanager.commit_all()
 666             self.assertTrue(bufferblock.owner.flush.called)
 667             self.assertTrue(mockkeep.put.called)
 668             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
 669             self.assertIsNone(bufferblock.buffer_view)
 670
 671     def test_bufferblock_commit_pending(self):
 672         # Test for bug #7225
 673         mockkeep = mock.MagicMock()
 674         mockkeep.put.side_effect = lambda x: time.sleep(1)
 675         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
 676             bufferblock = blockmanager.alloc_bufferblock()
 677             bufferblock.append("foo")
 678
 679             blockmanager.commit_bufferblock(bufferblock, False)
 680             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.PENDING)
 681
 682             blockmanager.commit_bufferblock(bufferblock, True)
 683             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
 684
 685
 686     def test_bufferblock_commit_with_error(self):
 687         mockkeep = mock.MagicMock()
 688         mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail")
 689         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
 690             bufferblock = blockmanager.alloc_bufferblock()
 691             bufferblock.owner = mock.MagicMock()
 692             def flush(sync=None):
 693                 blockmanager.commit_bufferblock(bufferblock, sync)
 694             bufferblock.owner.flush.side_effect = flush
 695             bufferblock.append("foo")
 696             with self.assertRaises(arvados.errors.KeepWriteError) as err:
 697                 blockmanager.commit_all()
 698             self.assertTrue(bufferblock.owner.flush.called)
 699             self.assertEqual(str(err.exception), "Error writing some blocks: block acbd18db4cc2f85cedef654fccc4a4d8+3 raised KeepWriteError (fail)")
 700             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.ERROR)