sdk/python/tests/test_arvfile.py

   1 #!/usr/bin/env python
   2
   3 import bz2
   4 import gzip
   5 import io
   6 import mock
   7 import os
   8 import unittest
   9 import hashlib
  10
  11 import arvados
  12 from arvados._ranges import Range
  13 from arvados.keep import KeepLocator
  14 from arvados.collection import Collection, CollectionReader
  15 from arvados.arvfile import ArvadosFile, ArvadosFileReader, SYNC_READONLY, SYNC_EXPLICIT
  16
  17 import arvados_testutil as tutil
  18 from test_stream import StreamFileReaderTestCase, StreamRetryTestMixin
  19
  20 class ArvadosFileWriterTestCase(unittest.TestCase):
  21     class MockKeep(object):
  22         def __init__(self, blocks):
  23             self.blocks = blocks
  24             self.requests = []
  25         def get(self, locator, num_retries=0):
  26             self.requests.append(locator)
  27             return self.blocks.get(locator)
  28         def get_from_cache(self, locator):
  29             self.requests.append(locator)
  30             return self.blocks.get(locator)
  31         def put(self, data, num_retries=None):
  32             pdh = "%s+%i" % (hashlib.md5(data).hexdigest(), len(data))
  33             self.blocks[pdh] = str(data)
  34             return pdh
  35
  36     class MockApi(object):
  37         def __init__(self, b, r):
  38             self.body = b
  39             self.response = r
  40         class MockCollections(object):
  41             def __init__(self, b, r):
  42                 self.body = b
  43                 self.response = r
  44             class Execute(object):
  45                 def __init__(self, r):
  46                     self.response = r
  47                 def execute(self, num_retries=None):
  48                     return self.response
  49             def create(self, ensure_unique_name=False, body=None):
  50                 if body != self.body:
  51                     raise Exception("Body %s does not match expectation %s" % (body, self.body))
  52                 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
  53             def update(self, uuid=None, body=None):
  54                 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
  55         def collections(self):
  56             return ArvadosFileWriterTestCase.MockApi.MockCollections(self.body, self.response)
  57
  58
  59     def test_truncate(self):
  60         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
  61         api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate",
  62                                                  "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n"},
  63                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
  64                                                  "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n"})
  65         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
  66                              api_client=api, keep_client=keep) as c:
  67             writer = c.open("count.txt", "r+")
  68             self.assertEqual(writer.size(), 10)
  69             self.assertEqual("0123456789", writer.read(12))
  70
  71             writer.truncate(8)
  72
  73             # Make sure reading off the end doesn't break
  74             self.assertEqual("", writer.read(12))
  75
  76             self.assertEqual(writer.size(), 8)
  77             writer.seek(0, os.SEEK_SET)
  78             self.assertEqual("01234567", writer.read(12))
  79
  80             self.assertEqual(None, c._manifest_locator)
  81             self.assertEqual(True, c.modified())
  82             c.save_new("test_truncate")
  83             self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
  84             self.assertEqual(False, c.modified())
  85
  86     def test_write_to_end(self):
  87         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
  88         api = ArvadosFileWriterTestCase.MockApi({"name":"test_append",
  89                                                  "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n"},
  90                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
  91                                                  "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n"})
  92         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
  93                              api_client=api, keep_client=keep) as c:
  94             writer = c.open("count.txt", "r+")
  95             self.assertEqual(writer.size(), 10)
  96
  97             writer.seek(5, os.SEEK_SET)
  98             self.assertEqual("56789", writer.read(8))
  99
 100             writer.seek(10, os.SEEK_SET)
 101             writer.write("foo")
 102             self.assertEqual(writer.size(), 13)
 103
 104             writer.seek(5, os.SEEK_SET)
 105             self.assertEqual("56789foo", writer.read(8))
 106
 107             self.assertEqual(None, c._manifest_locator)
 108             self.assertEqual(True, c.modified())
 109             self.assertEqual(None, keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
 110
 111             c.save_new("test_append")
 112             self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
 113             self.assertEqual(False, c.modified())
 114             self.assertEqual("foo", keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
 115
 116
 117     def test_append(self):
 118         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 119         c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
 120         writer = c.open("count.txt", "a+")
 121         self.assertEqual(writer.read(20), "0123456789")
 122         writer.seek(0, os.SEEK_SET)
 123
 124         writer.write("hello")
 125         self.assertEqual(writer.read(20), "0123456789hello")
 126         writer.seek(0, os.SEEK_SET)
 127
 128         writer.write("world")
 129         self.assertEqual(writer.read(20), "0123456789helloworld")
 130
 131         self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.manifest_text())
 132
 133     def test_write_at_beginning(self):
 134         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 135         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 136                              keep_client=keep) as c:
 137             writer = c.open("count.txt", "r+")
 138             self.assertEqual("0123456789", writer.readfrom(0, 13))
 139             writer.seek(0, os.SEEK_SET)
 140             writer.write("foo")
 141             self.assertEqual(writer.size(), 10)
 142             self.assertEqual("foo3456789", writer.readfrom(0, 13))
 143             self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", c.manifest_text())
 144
 145     def test_write_in_middle(self):
 146         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 147         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 148                              keep_client=keep) as c:
 149             writer = c.open("count.txt", "r+")
 150             self.assertEqual("0123456789", writer.readfrom(0, 13))
 151             writer.seek(3, os.SEEK_SET)
 152             writer.write("foo")
 153             self.assertEqual(writer.size(), 10)
 154             self.assertEqual("012foo6789", writer.readfrom(0, 13))
 155             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", c.manifest_text())
 156
 157     def test_write_at_end(self):
 158         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 159         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 160                              keep_client=keep) as c:
 161             writer = c.open("count.txt", "r+")
 162             self.assertEqual("0123456789", writer.readfrom(0, 13))
 163             writer.seek(7, os.SEEK_SET)
 164             writer.write("foo")
 165             self.assertEqual(writer.size(), 10)
 166             self.assertEqual("0123456foo", writer.readfrom(0, 13))
 167             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", c.manifest_text())
 168
 169     def test_write_across_segment_boundary(self):
 170         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 171         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
 172                              keep_client=keep) as c:
 173             writer = c.open("count.txt", "r+")
 174             self.assertEqual("012345678901234", writer.readfrom(0, 15))
 175             writer.seek(7, os.SEEK_SET)
 176             writer.write("foobar")
 177             self.assertEqual(writer.size(), 20)
 178             self.assertEqual("0123456foobar34", writer.readfrom(0, 15))
 179             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", c.manifest_text())
 180
 181     def test_write_across_several_segments(self):
 182         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 183         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
 184                              keep_client=keep) as c:
 185             writer = c.open("count.txt", "r+")
 186             self.assertEqual("012301230123", writer.readfrom(0, 15))
 187             writer.seek(2, os.SEEK_SET)
 188             writer.write("abcdefg")
 189             self.assertEqual(writer.size(), 12)
 190             self.assertEqual("01abcdefg123", writer.readfrom(0, 15))
 191             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", c.manifest_text())
 192
 193     def test_write_large(self):
 194         keep = ArvadosFileWriterTestCase.MockKeep({})
 195         api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
 196                                                  "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n"},
 197                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 198                                                  "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n"})
 199         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 200                              api_client=api, keep_client=keep) as c:
 201             writer = c.open("count.txt", "r+")
 202             text = ''.join(["0123456789" for a in xrange(0, 100)])
 203             for b in xrange(0, 100000):
 204                 writer.write(text)
 205             self.assertEqual(writer.size(), 100000000)
 206
 207             self.assertEqual(None, c._manifest_locator)
 208             self.assertEqual(True, c.modified())
 209             c.save_new("test_write_large")
 210             self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
 211             self.assertEqual(False, c.modified())
 212
 213     def test_rewrite_on_empty_file(self):
 214         keep = ArvadosFileWriterTestCase.MockKeep({})
 215         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 216                              keep_client=keep) as c:
 217             writer = c.open("count.txt", "r+")
 218             for b in xrange(0, 10):
 219                 writer.seek(0, os.SEEK_SET)
 220                 writer.write("0123456789")
 221             writer.arvadosfile._repack_writes()
 222             self.assertEqual(writer.size(), 10)
 223             self.assertEqual("0123456789", writer.readfrom(0, 20))
 224             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.manifest_text())
 225
 226     def test_rewrite_append_existing_file(self):
 227         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 228         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
 229                              keep_client=keep) as c:
 230             writer = c.open("count.txt", "r+")
 231             for b in xrange(0, 10):
 232                 writer.seek(10, os.SEEK_SET)
 233                 writer.write("abcdefghij")
 234             writer.arvadosfile._repack_writes()
 235             self.assertEqual(writer.size(), 20)
 236             self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
 237             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.manifest_text())
 238
 239     def test_rewrite_over_existing_file(self):
 240         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 241         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
 242                              keep_client=keep) as c:
 243             writer = c.open("count.txt", "r+")
 244             for b in xrange(0, 10):
 245                 writer.seek(5, os.SEEK_SET)
 246                 writer.write("abcdefghij")
 247             writer.arvadosfile._repack_writes()
 248             self.assertEqual(writer.size(), 15)
 249             self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
 250             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.manifest_text())
 251
 252     def test_write_large_rewrite(self):
 253         keep = ArvadosFileWriterTestCase.MockKeep({})
 254         api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
 255                                                  "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n"},
 256                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 257                                                  "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n"})
 258         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 259                              api_client=api, keep_client=keep) as c:
 260             writer = c.open("count.txt", "r+")
 261             text = ''.join(["0123456789" for a in xrange(0, 100)])
 262             for b in xrange(0, 100000):
 263                 writer.write(text)
 264             writer.seek(0, os.SEEK_SET)
 265             writer.write("foo")
 266             self.assertEqual(writer.size(), 100000000)
 267
 268             self.assertEqual(None, c._manifest_locator)
 269             self.assertEqual(True, c.modified())
 270             c.save_new("test_write_large")
 271             self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
 272             self.assertEqual(False, c.modified())
 273
 274     def test_create(self):
 275         keep = ArvadosFileWriterTestCase.MockKeep({})
 276         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
 277                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
 278                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 279                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
 280         with Collection(api_client=api, keep_client=keep) as c:
 281             writer = c.open("count.txt", "w+")
 282             self.assertEqual(writer.size(), 0)
 283             writer.write("01234567")
 284             self.assertEqual(writer.size(), 8)
 285
 286             self.assertEqual(None, c._manifest_locator)
 287             self.assertEqual(True, c.modified())
 288             self.assertEqual(None, keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 289             c.save_new("test_create")
 290             self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
 291             self.assertEqual(False, c.modified())
 292             self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 293
 294
 295     def test_create_subdir(self):
 296         keep = ArvadosFileWriterTestCase.MockKeep({})
 297         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
 298                                                  "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
 299                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 300                                                  "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
 301         with Collection(api_client=api, keep_client=keep) as c:
 302             self.assertIsNone(c.api_response())
 303             writer = c.open("foo/bar/count.txt", "w+")
 304             writer.write("01234567")
 305             c.save_new("test_create")
 306             self.assertEqual(c.api_response(), api.response)
 307
 308     def test_overwrite(self):
 309         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 310         api = ArvadosFileWriterTestCase.MockApi({"name":"test_overwrite",
 311                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
 312                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 313                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
 314         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 315                              api_client=api, keep_client=keep) as c:
 316             writer = c.open("count.txt", "w+")
 317             self.assertEqual(writer.size(), 0)
 318             writer.write("01234567")
 319             self.assertEqual(writer.size(), 8)
 320
 321             self.assertEqual(None, c._manifest_locator)
 322             self.assertEqual(True, c.modified())
 323             c.save_new("test_overwrite")
 324             self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
 325             self.assertEqual(False, c.modified())
 326
 327     def test_file_not_found(self):
 328         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
 329             with self.assertRaises(IOError):
 330                 writer = c.open("nocount.txt", "r")
 331
 332     def test_cannot_open_directory(self):
 333         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
 334             with self.assertRaises(IOError):
 335                 writer = c.open(".", "r")
 336
 337     def test_create_multiple(self):
 338         keep = ArvadosFileWriterTestCase.MockKeep({})
 339         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create_multiple",
 340                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n"},
 341                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 342                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n"})
 343         with Collection(api_client=api, keep_client=keep) as c:
 344             w1 = c.open("count1.txt", "w")
 345             w2 = c.open("count2.txt", "w")
 346             w1.write("01234567")
 347             w2.write("abcdefgh")
 348             self.assertEqual(w1.size(), 8)
 349             self.assertEqual(w2.size(), 8)
 350
 351             self.assertEqual(None, c._manifest_locator)
 352             self.assertEqual(True, c.modified())
 353             self.assertEqual(None, keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 354             c.save_new("test_create_multiple")
 355             self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
 356             self.assertEqual(False, c.modified())
 357             self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 358
 359
 360 class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
 361     class MockParent(object):
 362         class MockBlockMgr(object):
 363             def __init__(self, blocks, nocache):
 364                 self.blocks = blocks
 365                 self.nocache = nocache
 366
 367             def block_prefetch(self, loc):
 368                 pass
 369
 370             def get_block_contents(self, loc, num_retries=0, cache_only=False):
 371                 if self.nocache and cache_only:
 372                     return None
 373                 return self.blocks[loc]
 374
 375         def __init__(self, blocks, nocache):
 376             self.blocks = blocks
 377             self.nocache = nocache
 378             self.lock = arvados.arvfile.NoopLock()
 379
 380         def root_collection(self):
 381             return self
 382
 383         def _my_block_manager(self):
 384             return ArvadosFileReaderTestCase.MockParent.MockBlockMgr(self.blocks, self.nocache)
 385
 386         def sync_mode(self):
 387             return SYNC_READONLY
 388
 389
 390     def make_count_reader(self, nocache=False):
 391         stream = []
 392         n = 0
 393         blocks = {}
 394         for d in ['01234', '34567', '67890']:
 395             loc = '{}+{}'.format(hashlib.md5(d).hexdigest(), len(d))
 396             blocks[loc] = d
 397             stream.append(Range(loc, n, len(d)))
 398             n += len(d)
 399         af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
 400         return ArvadosFileReader(af, "count.txt")
 401
 402     def test_read_returns_first_block(self):
 403         # read() calls will be aligned on block boundaries - see #3663.
 404         sfile = self.make_count_reader(nocache=True)
 405         self.assertEqual('123', sfile.read(10))
 406
 407     def test_successive_reads(self):
 408         sfile = self.make_count_reader(nocache=True)
 409         for expect in ['123', '456', '789', '']:
 410             self.assertEqual(expect, sfile.read(10))
 411
 412     def test_tell_after_block_read(self):
 413         sfile = self.make_count_reader(nocache=True)
 414         sfile.read(5)
 415         self.assertEqual(3, sfile.tell())
 416
 417     def test_prefetch(self):
 418         keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"})
 419         with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
 420             r = c.open("count.txt", "r")
 421             self.assertEqual("0123", r.read(4))
 422         self.assertIn("2e9ec317e197819358fbc43afca7d837+8", keep.requests)
 423         self.assertIn("e8dc4081b13434b45189a720b77b6818+8", keep.requests)
 424
 425     def test__eq__from_manifest(self):
 426         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 427             with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
 428                 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
 429                 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
 430
 431     def test__eq__from_writes(self):
 432         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 433             with Collection() as c2:
 434                 with c2.open("count1.txt", "w") as f:
 435                     f.write("0123456789")
 436
 437                 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
 438                 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
 439
 440     def test__ne__(self):
 441         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 442             with Collection() as c2:
 443                 with c2.open("count1.txt", "w") as f:
 444                     f.write("1234567890")
 445
 446                 self.assertTrue(c1["count1.txt"] != c2["count1.txt"])
 447                 self.assertFalse(c1["count1.txt"] == c2["count1.txt"])
 448
 449
 450 class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
 451     def reader_for(self, coll_name, **kwargs):
 452         stream = []
 453         segments = []
 454         n = 0
 455         for d in self.manifest_for(coll_name).split():
 456             try:
 457                 k = KeepLocator(d)
 458                 segments.append(Range(n, n, k.size))
 459                 stream.append(Range(d, n, k.size))
 460                 n += k.size
 461             except ValueError:
 462                 pass
 463         col = Collection(keep_client=self.keep_client())
 464         col._my_block_manager().prefetch_enabled = False
 465         af = ArvadosFile(col,
 466                          stream=stream,
 467                          segments=segments)
 468         return ArvadosFileReader(af, "test", **kwargs)
 469
 470     def read_for_test(self, reader, byte_count, **kwargs):
 471         return reader.read(byte_count, **kwargs)
 472
 473
 474 class ArvadosFileReadFromTestCase(ArvadosFileReadTestCase):
 475     def read_for_test(self, reader, byte_count, **kwargs):
 476         return reader.readfrom(0, byte_count, **kwargs)
 477
 478
 479 class ArvadosFileReadAllTestCase(ArvadosFileReadTestCase):
 480     def read_for_test(self, reader, byte_count, **kwargs):
 481         return ''.join(reader.readall(**kwargs))
 482
 483
 484 class ArvadosFileReadAllDecompressedTestCase(ArvadosFileReadTestCase):
 485     def read_for_test(self, reader, byte_count, **kwargs):
 486         return ''.join(reader.readall_decompressed(**kwargs))
 487
 488
 489 class ArvadosFileReadlinesTestCase(ArvadosFileReadTestCase):
 490     def read_for_test(self, reader, byte_count, **kwargs):
 491         return ''.join(reader.readlines(**kwargs))
 492
 493 class BlockManagerTest(unittest.TestCase):
 494     def test_bufferblock_append(self):
 495         keep = ArvadosFileWriterTestCase.MockKeep({})
 496         blockmanager = arvados.arvfile._BlockManager(keep)
 497         bufferblock = blockmanager.alloc_bufferblock()
 498         bufferblock.append("foo")
 499
 500         self.assertEqual(bufferblock.size(), 3)
 501         self.assertEqual(bufferblock.buffer_view[0:3], "foo")
 502         self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 503
 504         bufferblock.append("bar")
 505
 506         self.assertEqual(bufferblock.size(), 6)
 507         self.assertEqual(bufferblock.buffer_view[0:6], "foobar")
 508         self.assertEqual(bufferblock.locator(), "3858f62230ac3c915f300c664312c63f+6")
 509
 510         bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
 511         with self.assertRaises(arvados.errors.AssertionError):
 512             bufferblock.append("bar")
 513
 514     def test_bufferblock_dup(self):
 515         keep = ArvadosFileWriterTestCase.MockKeep({})
 516         blockmanager = arvados.arvfile._BlockManager(keep)
 517         bufferblock = blockmanager.alloc_bufferblock()
 518         bufferblock.append("foo")
 519
 520         self.assertEqual(bufferblock.size(), 3)
 521         self.assertEqual(bufferblock.buffer_view[0:3], "foo")
 522         self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 523         bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
 524
 525         bufferblock2 = blockmanager.dup_block(bufferblock, None)
 526         self.assertNotEqual(bufferblock.blockid, bufferblock2.blockid)
 527
 528         bufferblock2.append("bar")
 529
 530         self.assertEqual(bufferblock2.size(), 6)
 531         self.assertEqual(bufferblock2.buffer_view[0:6], "foobar")
 532         self.assertEqual(bufferblock2.locator(), "3858f62230ac3c915f300c664312c63f+6")
 533
 534         self.assertEqual(bufferblock.size(), 3)
 535         self.assertEqual(bufferblock.buffer_view[0:3], "foo")
 536         self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 537
 538     def test_bufferblock_get(self):
 539         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 540         blockmanager = arvados.arvfile._BlockManager(keep)
 541         bufferblock = blockmanager.alloc_bufferblock()
 542         bufferblock.append("foo")
 543
 544         self.assertEqual(blockmanager.get_block_contents("781e5e245d69b566979b86e28d23f2c7+10", 1), "0123456789")
 545         self.assertEqual(blockmanager.get_block_contents(bufferblock.blockid, 1), "foo")
 546
 547     def test_bufferblock_commit(self):
 548         mockkeep = mock.MagicMock()
 549         blockmanager = arvados.arvfile._BlockManager(mockkeep)
 550         bufferblock = blockmanager.alloc_bufferblock()
 551         bufferblock.append("foo")
 552         blockmanager.commit_all()
 553         self.assertTrue(mockkeep.put.called)
 554         self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
 555         self.assertIsNone(bufferblock.buffer_view)
 556
 557
 558     def test_bufferblock_commit_with_error(self):
 559         mockkeep = mock.MagicMock()
 560         mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail")
 561         blockmanager = arvados.arvfile._BlockManager(mockkeep)
 562         bufferblock = blockmanager.alloc_bufferblock()
 563         bufferblock.append("foo")
 564         with self.assertRaises(arvados.errors.KeepWriteError) as err:
 565             blockmanager.commit_all()
 566         self.assertEquals(str(err.exception), "Error writing some blocks: acbd18db4cc2f85cedef654fccc4a4d8+3 raised KeepWriteError (fail)")
 567         self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.PENDING)