sdk/python/tests/test_arvfile.py

   1 from __future__ import absolute_import
   2 from builtins import hex
   3 from builtins import str
   4 from builtins import range
   5 from builtins import object
   6 import bz2
   7 import datetime
   8 import gzip
   9 import io
  10 import mock
  11 import os
  12 import unittest
  13 import time
  14
  15 import arvados
  16 from arvados._ranges import Range
  17 from arvados.keep import KeepLocator
  18 from arvados.collection import Collection, CollectionReader
  19 from arvados.arvfile import ArvadosFile, ArvadosFileReader
  20
  21 from . import arvados_testutil as tutil
  22 from .test_stream import StreamFileReaderTestCase, StreamRetryTestMixin
  23
  24 class ArvadosFileWriterTestCase(unittest.TestCase):
  25     class MockKeep(object):
  26         def __init__(self, blocks):
  27             self.blocks = blocks
  28             self.requests = []
  29         def get(self, locator, num_retries=0):
  30             self.requests.append(locator)
  31             return self.blocks.get(locator)
  32         def get_from_cache(self, locator):
  33             self.requests.append(locator)
  34             return self.blocks.get(locator)
  35         def put(self, data, num_retries=None, copies=None):
  36             pdh = tutil.str_keep_locator(data)
  37             self.blocks[pdh] = bytes(data)
  38             return pdh
  39
  40     class MockApi(object):
  41         def __init__(self, b, r):
  42             self.body = b
  43             self.response = r
  44             self._schema = ArvadosFileWriterTestCase.MockApi.MockSchema()
  45             self._rootDesc = {}
  46         class MockSchema(object):
  47             def __init__(self):
  48                 self.schemas = {'Collection': {'properties': {'replication_desired': {'type':'integer'}}}}
  49         class MockCollections(object):
  50             def __init__(self, b, r):
  51                 self.body = b
  52                 self.response = r
  53             class Execute(object):
  54                 def __init__(self, r):
  55                     self.response = r
  56                 def execute(self, num_retries=None):
  57                     return self.response
  58             def create(self, ensure_unique_name=False, body=None):
  59                 if body != self.body:
  60                     raise Exception("Body %s does not match expectation %s" % (body, self.body))
  61                 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
  62             def update(self, uuid=None, body=None):
  63                 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
  64         def collections(self):
  65             return ArvadosFileWriterTestCase.MockApi.MockCollections(self.body, self.response)
  66
  67
  68     def test_truncate(self):
  69         keep = ArvadosFileWriterTestCase.MockKeep({
  70             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
  71         })
  72         api = ArvadosFileWriterTestCase.MockApi({
  73             "name": "test_truncate",
  74             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
  75             "replication_desired": None,
  76         }, {
  77             "uuid": "zzzzz-4zz18-mockcollection0",
  78             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
  79             "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52",
  80         })
  81         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
  82                              api_client=api, keep_client=keep) as c:
  83             writer = c.open("count.txt", "rb+")
  84             self.assertEqual(writer.size(), 10)
  85             self.assertEqual(b"0123456789", writer.read(12))
  86
  87             writer.truncate(8)
  88
  89             # Make sure reading off the end doesn't break
  90             self.assertEqual(b"", writer.read(12))
  91
  92             self.assertEqual(writer.size(), 8)
  93             writer.seek(0, os.SEEK_SET)
  94             self.assertEqual(b"01234567", writer.read(12))
  95
  96             self.assertIsNone(c.manifest_locator())
  97             self.assertTrue(c.modified())
  98             c.save_new("test_truncate")
  99             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 100             self.assertFalse(c.modified())
 101
 102     def test_write_to_end(self):
 103         keep = ArvadosFileWriterTestCase.MockKeep({
 104             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 105         })
 106         api = ArvadosFileWriterTestCase.MockApi({
 107             "name": "test_append",
 108             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
 109             "replication_desired": None,
 110         }, {
 111             "uuid": "zzzzz-4zz18-mockcollection0",
 112             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
 113             "portable_data_hash": "c5c3af76565c8efb6a806546bcf073f3+88",
 114         })
 115         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 116                              api_client=api, keep_client=keep) as c:
 117             writer = c.open("count.txt", "rb+")
 118             self.assertEqual(writer.size(), 10)
 119
 120             writer.seek(5, os.SEEK_SET)
 121             self.assertEqual(b"56789", writer.read(8))
 122
 123             writer.seek(10, os.SEEK_SET)
 124             writer.write("foo")
 125             self.assertEqual(writer.size(), 13)
 126
 127             writer.seek(5, os.SEEK_SET)
 128             self.assertEqual(b"56789foo", writer.read(8))
 129
 130             self.assertIsNone(c.manifest_locator())
 131             self.assertTrue(c.modified())
 132             self.assertIsNone(keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
 133
 134             c.save_new("test_append")
 135             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 136             self.assertFalse(c.modified())
 137             self.assertEqual(b"foo", keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
 138
 139
 140     def test_append(self):
 141         keep = ArvadosFileWriterTestCase.MockKeep({
 142             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 143         })
 144         c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
 145         writer = c.open("count.txt", "ab+")
 146         self.assertEqual(writer.read(20), b"0123456789")
 147         writer.seek(0, os.SEEK_SET)
 148
 149         writer.write("hello")
 150         self.assertEqual(writer.read(20), b"0123456789hello")
 151         writer.seek(0, os.SEEK_SET)
 152
 153         writer.write("world")
 154         self.assertEqual(writer.read(20), b"0123456789helloworld")
 155
 156         self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.portable_manifest_text())
 157
 158     def test_write_at_beginning(self):
 159         keep = ArvadosFileWriterTestCase.MockKeep({
 160             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 161         })
 162         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 163                              keep_client=keep) as c:
 164             writer = c.open("count.txt", "rb+")
 165             self.assertEqual(b"0123456789", writer.readfrom(0, 13))
 166             writer.seek(0, os.SEEK_SET)
 167             writer.write("foo")
 168             self.assertEqual(writer.size(), 10)
 169             self.assertEqual(b"foo3456789", writer.readfrom(0, 13))
 170             self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", c.portable_manifest_text())
 171
 172     def test_write_empty(self):
 173         keep = ArvadosFileWriterTestCase.MockKeep({})
 174         with Collection(keep_client=keep) as c:
 175             writer = c.open("count.txt", "wb")
 176             self.assertEqual(writer.size(), 0)
 177             self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:count.txt\n", c.portable_manifest_text())
 178
 179     def test_save_manifest_text(self):
 180         keep = ArvadosFileWriterTestCase.MockKeep({})
 181         with Collection(keep_client=keep) as c:
 182             writer = c.open("count.txt", "wb")
 183             writer.write(b"0123456789")
 184             self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
 185             self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 186
 187             self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.save_new(create_collection_record=False))
 188             self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 189
 190     def test_get_manifest_text_commits(self):
 191          keep = ArvadosFileWriterTestCase.MockKeep({})
 192          with Collection(keep_client=keep) as c:
 193              writer = c.open("count.txt", "wb")
 194              writer.write("0123456789")
 195              self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
 196              self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 197              self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.manifest_text())
 198              self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 199
 200
 201     def test_write_in_middle(self):
 202         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
 203         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 204                              keep_client=keep) as c:
 205             writer = c.open("count.txt", "rb+")
 206             self.assertEqual(b"0123456789", writer.readfrom(0, 13))
 207             writer.seek(3, os.SEEK_SET)
 208             writer.write("foo")
 209             self.assertEqual(writer.size(), 10)
 210             self.assertEqual(b"012foo6789", writer.readfrom(0, 13))
 211             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", c.portable_manifest_text())
 212
 213     def test_write_at_end(self):
 214         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
 215         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 216                              keep_client=keep) as c:
 217             writer = c.open("count.txt", "rb+")
 218             self.assertEqual(b"0123456789", writer.readfrom(0, 13))
 219             writer.seek(7, os.SEEK_SET)
 220             writer.write("foo")
 221             self.assertEqual(writer.size(), 10)
 222             self.assertEqual(b"0123456foo", writer.readfrom(0, 13))
 223             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", c.portable_manifest_text())
 224
 225     def test_write_across_segment_boundary(self):
 226         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
 227         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
 228                              keep_client=keep) as c:
 229             writer = c.open("count.txt", "rb+")
 230             self.assertEqual(b"012345678901234", writer.readfrom(0, 15))
 231             writer.seek(7, os.SEEK_SET)
 232             writer.write("foobar")
 233             self.assertEqual(writer.size(), 20)
 234             self.assertEqual(b"0123456foobar34", writer.readfrom(0, 15))
 235             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", c.portable_manifest_text())
 236
 237     def test_write_across_several_segments(self):
 238         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
 239         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
 240                              keep_client=keep) as c:
 241             writer = c.open("count.txt", "rb+")
 242             self.assertEqual(b"012301230123", writer.readfrom(0, 15))
 243             writer.seek(2, os.SEEK_SET)
 244             writer.write("abcdefg")
 245             self.assertEqual(writer.size(), 12)
 246             self.assertEqual(b"01abcdefg123", writer.readfrom(0, 15))
 247             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", c.portable_manifest_text())
 248
 249     def test_write_large(self):
 250         keep = ArvadosFileWriterTestCase.MockKeep({})
 251         api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
 252                                                  "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n",
 253                                                  "replication_desired":None},
 254                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 255                                                  "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n",
 256                                                  "portable_data_hash":"9132ca8e3f671c76103a38f5bc24328c+108"})
 257         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 258                              api_client=api, keep_client=keep) as c:
 259             writer = c.open("count.txt", "rb+")
 260             text = "0123456789" * 100
 261             for b in range(0, 100000):
 262                 writer.write(text)
 263             self.assertEqual(writer.size(), 100000000)
 264
 265             self.assertIsNone(c.manifest_locator())
 266             self.assertTrue(c.modified())
 267             c.save_new("test_write_large")
 268             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 269             self.assertFalse(c.modified())
 270
 271
 272     def test_large_write(self):
 273         keep = ArvadosFileWriterTestCase.MockKeep({})
 274         api = ArvadosFileWriterTestCase.MockApi({}, {})
 275         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 276                              api_client=api, keep_client=keep) as c:
 277             writer = c.open("count.txt", "rb+")
 278             self.assertEqual(writer.size(), 0)
 279
 280             text = "0123456789"
 281             writer.write(text)
 282             text = "0123456789" * 9999999
 283             writer.write(text)
 284             self.assertEqual(writer.size(), 100000000)
 285
 286             self.assertEqual(c.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 48dd23ea1645fd47d789804d71b5bb8e+67108864 77c57dc6ac5a10bb2205caaa73187994+32891126 0:100000000:count.txt\n")
 287
 288     def test_rewrite_on_empty_file(self):
 289         keep = ArvadosFileWriterTestCase.MockKeep({})
 290         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 291                              keep_client=keep) as c:
 292             writer = c.open("count.txt", "rb+")
 293             for b in range(0, 10):
 294                 writer.seek(0, os.SEEK_SET)
 295                 writer.write("0123456789")
 296
 297             self.assertEqual(writer.size(), 10)
 298             self.assertEqual(b"0123456789", writer.readfrom(0, 20))
 299             self.assertEqual(". 7a08b07e84641703e5f2c836aa59a170+100 90:10:count.txt\n", c.portable_manifest_text())
 300             writer.flush()
 301             self.assertEqual(writer.size(), 10)
 302             self.assertEqual(b"0123456789", writer.readfrom(0, 20))
 303             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.portable_manifest_text())
 304
 305     def test_rewrite_append_existing_file(self):
 306         keep = ArvadosFileWriterTestCase.MockKeep({
 307             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 308         })
 309         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
 310                              keep_client=keep) as c:
 311             writer = c.open("count.txt", "rb+")
 312             for b in range(0, 10):
 313                 writer.seek(10, os.SEEK_SET)
 314                 writer.write("abcdefghij")
 315
 316             self.assertEqual(writer.size(), 20)
 317             self.assertEqual(b"0123456789abcdefghij", writer.readfrom(0, 20))
 318             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:10:count.txt 100:10:count.txt\n", c.portable_manifest_text())
 319
 320             writer.arvadosfile.flush()
 321             self.assertEqual(writer.size(), 20)
 322             self.assertEqual(b"0123456789abcdefghij", writer.readfrom(0, 20))
 323             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.portable_manifest_text())
 324
 325     def test_rewrite_over_existing_file(self):
 326         keep = ArvadosFileWriterTestCase.MockKeep({
 327             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 328         })
 329         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
 330                              keep_client=keep) as c:
 331             writer = c.open("count.txt", "rb+")
 332             for b in range(0, 10):
 333                 writer.seek(5, os.SEEK_SET)
 334                 writer.write("abcdefghij")
 335
 336             self.assertEqual(writer.size(), 15)
 337             self.assertEqual(b"01234abcdefghij", writer.readfrom(0, 20))
 338             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:5:count.txt 100:10:count.txt\n", c.portable_manifest_text())
 339
 340             writer.arvadosfile.flush()
 341
 342             self.assertEqual(writer.size(), 15)
 343             self.assertEqual(b"01234abcdefghij", writer.readfrom(0, 20))
 344             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.portable_manifest_text())
 345
 346     def test_write_large_rewrite(self):
 347         keep = ArvadosFileWriterTestCase.MockKeep({})
 348         api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
 349                                                  "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n",
 350                                                  "replication_desired":None},
 351                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 352                                                  "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n",
 353                                                  "portable_data_hash":"217665c6b713e1b78dfba7ebd42344db+156"})
 354         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 355                              api_client=api, keep_client=keep) as c:
 356             writer = c.open("count.txt", "rb+")
 357             text = b''.join([b"0123456789" for a in range(0, 100)])
 358             for b in range(0, 100000):
 359                 writer.write(text)
 360             writer.seek(0, os.SEEK_SET)
 361             writer.write("foo")
 362             self.assertEqual(writer.size(), 100000000)
 363
 364             self.assertIsNone(c.manifest_locator())
 365             self.assertTrue(c.modified())
 366             c.save_new("test_write_large")
 367             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 368             self.assertFalse(c.modified())
 369
 370     def test_create(self):
 371         keep = ArvadosFileWriterTestCase.MockKeep({})
 372         api = ArvadosFileWriterTestCase.MockApi({
 373             "name":"test_create",
 374             "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 375             "replication_desired":None,
 376         }, {
 377             "uuid":"zzzzz-4zz18-mockcollection0",
 378             "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 379             "portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51",
 380         })
 381         with Collection(api_client=api, keep_client=keep) as c:
 382             writer = c.open("count.txt", "wb+")
 383             self.assertEqual(writer.size(), 0)
 384             writer.write("01234567")
 385             self.assertEqual(writer.size(), 8)
 386
 387             self.assertIsNone(c.manifest_locator())
 388             self.assertTrue(c.modified())
 389             self.assertIsNone(keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 390             c.save_new("test_create")
 391             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 392             self.assertFalse(c.modified())
 393             self.assertEqual(b"01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 394
 395
 396     def test_create_subdir(self):
 397         keep = ArvadosFileWriterTestCase.MockKeep({})
 398         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
 399                                                  "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 400                                                  "replication_desired":None},
 401                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 402                                                  "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 403                                                  "portable_data_hash":"1b02aaa62528d28a5be41651cbb9d7c7+59"})
 404         with Collection(api_client=api, keep_client=keep) as c:
 405             self.assertIsNone(c.api_response())
 406             writer = c.open("foo/bar/count.txt", "wb+")
 407             writer.write("01234567")
 408             self.assertFalse(c.committed())
 409             c.save_new("test_create")
 410             self.assertTrue(c.committed())
 411             self.assertEqual(c.api_response(), api.response)
 412
 413     def test_overwrite(self):
 414         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 415         api = ArvadosFileWriterTestCase.MockApi({"name":"test_overwrite",
 416                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 417                                                  "replication_desired":None},
 418                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 419                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 420                                                  "portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51"})
 421         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 422                              api_client=api, keep_client=keep) as c:
 423             writer = c.open("count.txt", "wb+")
 424             self.assertEqual(writer.size(), 0)
 425             writer.write("01234567")
 426             self.assertEqual(writer.size(), 8)
 427
 428             self.assertIsNone(c.manifest_locator())
 429             self.assertTrue(c.modified())
 430             c.save_new("test_overwrite")
 431             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 432             self.assertFalse(c.modified())
 433
 434     def test_file_not_found(self):
 435         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
 436             with self.assertRaises(IOError):
 437                 writer = c.open("nocount.txt", "rb")
 438
 439     def test_cannot_open_directory(self):
 440         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
 441             with self.assertRaises(IOError):
 442                 writer = c.open(".", "rb")
 443
 444     def test_create_multiple(self):
 445         keep = ArvadosFileWriterTestCase.MockKeep({})
 446         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create_multiple",
 447                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n",
 448                                                  "replication_desired":None},
 449                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 450                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n",
 451                                                  "portable_data_hash":"71e7bb6c00d31fc2b4364199fd97be08+102"})
 452         with Collection(api_client=api, keep_client=keep) as c:
 453             w1 = c.open("count1.txt", "wb")
 454             w2 = c.open("count2.txt", "wb")
 455             w1.write("01234567")
 456             w2.write("abcdefgh")
 457             self.assertEqual(w1.size(), 8)
 458             self.assertEqual(w2.size(), 8)
 459
 460             self.assertIsNone(c.manifest_locator())
 461             self.assertTrue(c.modified())
 462             self.assertIsNone(keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 463             c.save_new("test_create_multiple")
 464             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 465             self.assertFalse(c.modified())
 466             self.assertEqual(b"01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 467
 468
 469 class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
 470     class MockParent(object):
 471         class MockBlockMgr(object):
 472             def __init__(self, blocks, nocache):
 473                 self.blocks = blocks
 474                 self.nocache = nocache
 475
 476             def block_prefetch(self, loc):
 477                 pass
 478
 479             def get_block_contents(self, loc, num_retries=0, cache_only=False):
 480                 if self.nocache and cache_only:
 481                     return None
 482                 return self.blocks[loc]
 483
 484         def __init__(self, blocks, nocache):
 485             self.blocks = blocks
 486             self.nocache = nocache
 487             self.lock = arvados.arvfile.NoopLock()
 488
 489         def root_collection(self):
 490             return self
 491
 492         def _my_block_manager(self):
 493             return ArvadosFileReaderTestCase.MockParent.MockBlockMgr(self.blocks, self.nocache)
 494
 495
 496     def make_count_reader(self, nocache=False):
 497         stream = []
 498         n = 0
 499         blocks = {}
 500         for d in [b'01234', b'34567', b'67890']:
 501             loc = tutil.str_keep_locator(d)
 502             blocks[loc] = d
 503             stream.append(Range(loc, n, len(d)))
 504             n += len(d)
 505         af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), "count.txt", stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
 506         return ArvadosFileReader(af, mode="rb")
 507
 508     def test_read_block_crossing_behavior(self):
 509         # read() needs to return all the data requested if possible, even if it
 510         # crosses uncached blocks: https://arvados.org/issues/5856
 511         sfile = self.make_count_reader(nocache=True)
 512         self.assertEqual(b'12345678', sfile.read(8))
 513
 514     def test_successive_reads(self):
 515         # Override StreamFileReaderTestCase.test_successive_reads
 516         sfile = self.make_count_reader(nocache=True)
 517         self.assertEqual(b'1234', sfile.read(4))
 518         self.assertEqual(b'5678', sfile.read(4))
 519         self.assertEqual(b'9', sfile.read(4))
 520         self.assertEqual(b'', sfile.read(4))
 521
 522     def test_tell_after_block_read(self):
 523         # Override StreamFileReaderTestCase.test_tell_after_block_read
 524         sfile = self.make_count_reader(nocache=True)
 525         self.assertEqual(b'12345678', sfile.read(8))
 526         self.assertEqual(8, sfile.tell())
 527
 528     def test_prefetch(self):
 529         keep = ArvadosFileWriterTestCase.MockKeep({
 530             "2e9ec317e197819358fbc43afca7d837+8": b"01234567",
 531             "e8dc4081b13434b45189a720b77b6818+8": b"abcdefgh",
 532         })
 533         with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
 534             r = c.open("count.txt", "rb")
 535             self.assertEqual(b"0123", r.read(4))
 536         self.assertIn("2e9ec317e197819358fbc43afca7d837+8", keep.requests)
 537         self.assertIn("e8dc4081b13434b45189a720b77b6818+8", keep.requests)
 538
 539     def test__eq__from_manifest(self):
 540         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 541             with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
 542                 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
 543                 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
 544
 545     def test__eq__from_writes(self):
 546         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 547             with Collection() as c2:
 548                 f = c2.open("count1.txt", "wb")
 549                 f.write("0123456789")
 550
 551                 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
 552                 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
 553
 554     def test__ne__(self):
 555         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 556             with Collection() as c2:
 557                 f = c2.open("count1.txt", "wb")
 558                 f.write("1234567890")
 559
 560                 self.assertTrue(c1["count1.txt"] != c2["count1.txt"])
 561                 self.assertFalse(c1["count1.txt"] == c2["count1.txt"])
 562
 563
 564 class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
 565     def reader_for(self, coll_name, **kwargs):
 566         stream = []
 567         segments = []
 568         n = 0
 569         for d in self.manifest_for(coll_name).split():
 570             try:
 571                 k = KeepLocator(d)
 572                 segments.append(Range(n, n, k.size))
 573                 stream.append(Range(d, n, k.size))
 574                 n += k.size
 575             except ValueError:
 576                 pass
 577
 578         blockmanager = arvados.arvfile._BlockManager(self.keep_client())
 579         blockmanager.prefetch_enabled = False
 580         col = Collection(keep_client=self.keep_client(), block_manager=blockmanager)
 581         af = ArvadosFile(col, "test",
 582                          stream=stream,
 583                          segments=segments)
 584         kwargs.setdefault('mode', 'rb')
 585         return ArvadosFileReader(af, **kwargs)
 586
 587     def read_for_test(self, reader, byte_count, **kwargs):
 588         return reader.read(byte_count, **kwargs)
 589
 590
 591 class ArvadosFileReadFromTestCase(ArvadosFileReadTestCase):
 592     def read_for_test(self, reader, byte_count, **kwargs):
 593         return reader.readfrom(0, byte_count, **kwargs)
 594
 595
 596 class ArvadosFileReadAllTestCase(ArvadosFileReadTestCase):
 597     def read_for_test(self, reader, byte_count, **kwargs):
 598         return b''.join(reader.readall(**kwargs))
 599
 600
 601 class ArvadosFileReadAllDecompressedTestCase(ArvadosFileReadTestCase):
 602     def read_for_test(self, reader, byte_count, **kwargs):
 603         return b''.join(reader.readall_decompressed(**kwargs))
 604
 605
 606 class ArvadosFileReadlinesTestCase(ArvadosFileReadTestCase):
 607     def read_for_test(self, reader, byte_count, **kwargs):
 608         return ''.join(reader.readlines(**kwargs)).encode()
 609
 610
 611 class ArvadosFileTestCase(unittest.TestCase):
 612     def datetime_to_hex(self, dt):
 613         return hex(int(time.mktime(dt.timetuple())))[2:]
 614
 615     def test_permission_expired(self):
 616         base_manifest = ". 781e5e245d69b566979b86e28d23f2c7+10+A715fd31f8111894f717eb1003c1b0216799dd9ec@{} 0:10:count.txt\n"
 617         now = datetime.datetime.now()
 618         a_week_ago = now - datetime.timedelta(days=7)
 619         a_month_ago = now - datetime.timedelta(days=30)
 620         a_week_from_now = now + datetime.timedelta(days=7)
 621         with Collection(base_manifest.format(self.datetime_to_hex(a_week_from_now))) as c:
 622             self.assertFalse(c.find('count.txt').permission_expired())
 623         with Collection(base_manifest.format(self.datetime_to_hex(a_week_ago))) as c:
 624             f = c.find('count.txt')
 625             self.assertTrue(f.permission_expired())
 626             self.assertTrue(f.permission_expired(a_week_from_now))
 627             self.assertFalse(f.permission_expired(a_month_ago))
 628
 629
 630 class BlockManagerTest(unittest.TestCase):
 631     def test_bufferblock_append(self):
 632         keep = ArvadosFileWriterTestCase.MockKeep({})
 633         with arvados.arvfile._BlockManager(keep) as blockmanager:
 634             bufferblock = blockmanager.alloc_bufferblock()
 635             bufferblock.append("foo")
 636
 637             self.assertEqual(bufferblock.size(), 3)
 638             self.assertEqual(bufferblock.buffer_view[0:3], b"foo")
 639             self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 640
 641             bufferblock.append("bar")
 642
 643             self.assertEqual(bufferblock.size(), 6)
 644             self.assertEqual(bufferblock.buffer_view[0:6], b"foobar")
 645             self.assertEqual(bufferblock.locator(), "3858f62230ac3c915f300c664312c63f+6")
 646
 647             bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
 648             with self.assertRaises(arvados.errors.AssertionError):
 649                 bufferblock.append("bar")
 650
 651     def test_bufferblock_dup(self):
 652         keep = ArvadosFileWriterTestCase.MockKeep({})
 653         with arvados.arvfile._BlockManager(keep) as blockmanager:
 654             bufferblock = blockmanager.alloc_bufferblock()
 655             bufferblock.append("foo")
 656
 657             self.assertEqual(bufferblock.size(), 3)
 658             self.assertEqual(bufferblock.buffer_view[0:3], b"foo")
 659             self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 660             bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
 661
 662             bufferblock2 = blockmanager.dup_block(bufferblock, None)
 663             self.assertNotEqual(bufferblock.blockid, bufferblock2.blockid)
 664
 665             bufferblock2.append("bar")
 666
 667             self.assertEqual(bufferblock2.size(), 6)
 668             self.assertEqual(bufferblock2.buffer_view[0:6], b"foobar")
 669             self.assertEqual(bufferblock2.locator(), "3858f62230ac3c915f300c664312c63f+6")
 670
 671             self.assertEqual(bufferblock.size(), 3)
 672             self.assertEqual(bufferblock.buffer_view[0:3], b"foo")
 673             self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 674
 675     def test_bufferblock_get(self):
 676         keep = ArvadosFileWriterTestCase.MockKeep({
 677             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 678         })
 679         with arvados.arvfile._BlockManager(keep) as blockmanager:
 680             bufferblock = blockmanager.alloc_bufferblock()
 681             bufferblock.append("foo")
 682
 683             self.assertEqual(blockmanager.get_block_contents("781e5e245d69b566979b86e28d23f2c7+10", 1), b"0123456789")
 684             self.assertEqual(blockmanager.get_block_contents(bufferblock.blockid, 1), b"foo")
 685
 686     def test_bufferblock_commit(self):
 687         mockkeep = mock.MagicMock()
 688         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
 689             bufferblock = blockmanager.alloc_bufferblock()
 690             bufferblock.owner = mock.MagicMock()
 691             def flush(sync=None):
 692                 blockmanager.commit_bufferblock(bufferblock, sync)
 693             bufferblock.owner.flush.side_effect = flush
 694             bufferblock.append("foo")
 695             blockmanager.commit_all()
 696             self.assertTrue(bufferblock.owner.flush.called)
 697             self.assertTrue(mockkeep.put.called)
 698             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
 699             self.assertIsNone(bufferblock.buffer_view)
 700
 701     def test_bufferblock_commit_pending(self):
 702         # Test for bug #7225
 703         mockkeep = mock.MagicMock()
 704         mockkeep.put.side_effect = lambda x: time.sleep(1)
 705         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
 706             bufferblock = blockmanager.alloc_bufferblock()
 707             bufferblock.append("foo")
 708
 709             blockmanager.commit_bufferblock(bufferblock, False)
 710             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.PENDING)
 711
 712             blockmanager.commit_bufferblock(bufferblock, True)
 713             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
 714
 715
 716     def test_bufferblock_commit_with_error(self):
 717         mockkeep = mock.MagicMock()
 718         mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail")
 719         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
 720             bufferblock = blockmanager.alloc_bufferblock()
 721             bufferblock.owner = mock.MagicMock()
 722             def flush(sync=None):
 723                 blockmanager.commit_bufferblock(bufferblock, sync)
 724             bufferblock.owner.flush.side_effect = flush
 725             bufferblock.append("foo")
 726             with self.assertRaises(arvados.errors.KeepWriteError) as err:
 727                 blockmanager.commit_all()
 728             self.assertTrue(bufferblock.owner.flush.called)
 729             self.assertEqual(str(err.exception), "Error writing some blocks: block acbd18db4cc2f85cedef654fccc4a4d8+3 raised KeepWriteError (fail)")
 730             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.ERROR)