sdk/python/tests/test_arvfile.py

   1 #!/usr/bin/env python
   2
   3 import bz2
   4 import gzip
   5 import io
   6 import mock
   7 import os
   8 import unittest
   9 import hashlib
  10
  11 import arvados
  12 from arvados import ArvadosFile, ArvadosFileReader, Range, import_manifest, export_manifest, KeepLocator
  13 from arvados.arvfile import SYNC_READONLY, SYNC_EXPLICIT
  14
  15 import arvados_testutil as tutil
  16 from test_stream import StreamFileReaderTestCase, StreamRetryTestMixin
  17
  18 class ArvadosFileWriterTestCase(unittest.TestCase):
  19     class MockKeep(object):
  20         def __init__(self, blocks):
  21             self.blocks = blocks
  22             self.requests = []
  23         def get(self, locator, num_retries=0):
  24             self.requests.append(locator)
  25             return self.blocks.get(locator)
  26         def get_from_cache(self, locator):
  27             self.requests.append(locator)
  28             return self.blocks.get(locator)
  29         def put(self, data, num_retries=None):
  30             pdh = "%s+%i" % (hashlib.md5(data).hexdigest(), len(data))
  31             self.blocks[pdh] = str(data)
  32             return pdh
  33
  34     class MockApi(object):
  35         def __init__(self, b, r):
  36             self.b = b
  37             self.r = r
  38         class MockCollections(object):
  39             def __init__(self, b, r):
  40                 self.b = b
  41                 self.r = r
  42             class Execute(object):
  43                 def __init__(self, r):
  44                     self.r = r
  45                 def execute(self, num_retries=None):
  46                     return self.r
  47             def create(self, ensure_unique_name=False, body=None):
  48                 if body != self.b:
  49                     raise Exception("Body %s does not match expectation %s" % (body, self.b))
  50                 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.r)
  51             def update(self, uuid=None, body=None):
  52                 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.r)
  53         def collections(self):
  54             return ArvadosFileWriterTestCase.MockApi.MockCollections(self.b, self.r)
  55
  56
  57     def test_truncate(self):
  58         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
  59         api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate",
  60                                                  "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n"},
  61                                                 {"uuid":"zzzzz-4zz18-mockcollection0"})
  62         with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
  63                              api_client=api, keep_client=keep) as c:
  64             writer = c.open("count.txt", "r+")
  65             self.assertEqual(writer.size(), 10)
  66             writer.seek(5)
  67             self.assertEqual("56789", writer.read(8))
  68             writer.truncate(8)
  69             writer.seek(5, os.SEEK_SET)
  70             self.assertEqual("567", writer.read(8))
  71
  72             self.assertEqual(None, c._manifest_locator)
  73             self.assertEqual(True, c.modified())
  74             c.save_new("test_truncate")
  75             self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
  76             self.assertEqual(False, c.modified())
  77
  78     def test_append0(self):
  79         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
  80         api = ArvadosFileWriterTestCase.MockApi({"name":"test_append",
  81                                                  "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n"},
  82                                                 {"uuid":"zzzzz-4zz18-mockcollection0"})
  83         with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
  84                              api_client=api, keep_client=keep) as c:
  85             writer = c.open("count.txt", "r+")
  86             self.assertEqual(writer.size(), 10)
  87
  88             writer.seek(5, os.SEEK_SET)
  89             self.assertEqual("56789", writer.read(8))
  90
  91             writer.seek(10, os.SEEK_SET)
  92             writer.write("foo")
  93             self.assertEqual(writer.size(), 13)
  94
  95             writer.seek(5, os.SEEK_SET)
  96             self.assertEqual("56789foo", writer.read(8))
  97
  98             self.assertEqual(None, c._manifest_locator)
  99             self.assertEqual(True, c.modified())
 100             self.assertEqual(None, keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
 101
 102             c.save_new("test_append")
 103             self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
 104             self.assertEqual(False, c.modified())
 105             self.assertEqual("foo", keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
 106
 107
 108     def test_append1(self):
 109         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 110         c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
 111         writer = c.open("count.txt", "a+")
 112         self.assertEqual(writer.read(20), "0123456789")
 113         writer.seek(0, os.SEEK_SET)
 114
 115         writer.write("hello")
 116         self.assertEqual(writer.read(20), "0123456789hello")
 117         writer.seek(0, os.SEEK_SET)
 118
 119         writer.write("world")
 120         self.assertEqual(writer.read(20), "0123456789helloworld")
 121
 122         self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", export_manifest(c))
 123
 124     def test_write0(self):
 125         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 126         with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 127                              keep_client=keep) as c:
 128             writer = c.open("count.txt", "r+")
 129             self.assertEqual("0123456789", writer.readfrom(0, 13))
 130             writer.seek(0, os.SEEK_SET)
 131             writer.write("foo")
 132             self.assertEqual(writer.size(), 10)
 133             self.assertEqual("foo3456789", writer.readfrom(0, 13))
 134             self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", export_manifest(c))
 135
 136     def test_write1(self):
 137         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 138         with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 139                              keep_client=keep) as c:
 140             writer = c.open("count.txt", "r+")
 141             self.assertEqual("0123456789", writer.readfrom(0, 13))
 142             writer.seek(3, os.SEEK_SET)
 143             writer.write("foo")
 144             self.assertEqual(writer.size(), 10)
 145             self.assertEqual("012foo6789", writer.readfrom(0, 13))
 146             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", export_manifest(c))
 147
 148     def test_write2(self):
 149         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 150         with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 151                              keep_client=keep) as c:
 152             writer = c.open("count.txt", "r+")
 153             self.assertEqual("0123456789", writer.readfrom(0, 13))
 154             writer.seek(7, os.SEEK_SET)
 155             writer.write("foo")
 156             self.assertEqual(writer.size(), 10)
 157             self.assertEqual("0123456foo", writer.readfrom(0, 13))
 158             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", export_manifest(c))
 159
 160     def test_write3(self):
 161         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 162         with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
 163                              keep_client=keep) as c:
 164             writer = c.open("count.txt", "r+")
 165             self.assertEqual("012345678901234", writer.readfrom(0, 15))
 166             writer.seek(7, os.SEEK_SET)
 167             writer.write("foobar")
 168             self.assertEqual(writer.size(), 20)
 169             self.assertEqual("0123456foobar34", writer.readfrom(0, 15))
 170             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", export_manifest(c))
 171
 172     def test_write4(self):
 173         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 174         with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
 175                              keep_client=keep) as c:
 176             writer = c.open("count.txt", "r+")
 177             self.assertEqual("012301230123", writer.readfrom(0, 15))
 178             writer.seek(2, os.SEEK_SET)
 179             writer.write("abcdefg")
 180             self.assertEqual(writer.size(), 12)
 181             self.assertEqual("01abcdefg123", writer.readfrom(0, 15))
 182             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", export_manifest(c))
 183
 184     def test_write_large(self):
 185         keep = ArvadosFileWriterTestCase.MockKeep({})
 186         api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
 187                                                  "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n"},
 188                                                 {"uuid":"zzzzz-4zz18-mockcollection0"})
 189         with arvados.WritableCollection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 190                              api_client=api, keep_client=keep) as c:
 191             writer = c.open("count.txt", "r+")
 192             text = ''.join(["0123456789" for a in xrange(0, 100)])
 193             for b in xrange(0, 100000):
 194                 writer.write(text)
 195             self.assertEqual(writer.size(), 100000000)
 196
 197             self.assertEqual(None, c._manifest_locator)
 198             self.assertEqual(True, c.modified())
 199             c.save_new("test_write_large")
 200             self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
 201             self.assertEqual(False, c.modified())
 202
 203     def test_write_rewrite0(self):
 204         keep = ArvadosFileWriterTestCase.MockKeep({})
 205         with arvados.WritableCollection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 206                              keep_client=keep) as c:
 207             writer = c.open("count.txt", "r+")
 208             for b in xrange(0, 10):
 209                 writer.seek(0, os.SEEK_SET)
 210                 writer.write("0123456789")
 211             writer.arvadosfile._repack_writes()
 212             self.assertEqual(writer.size(), 10)
 213             self.assertEqual("0123456789", writer.readfrom(0, 20))
 214             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", export_manifest(c))
 215
 216     def test_write_rewrite1(self):
 217         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 218         with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
 219                              keep_client=keep) as c:
 220             writer = c.open("count.txt", "r+")
 221             for b in xrange(0, 10):
 222                 writer.seek(10, os.SEEK_SET)
 223                 writer.write("abcdefghij")
 224             writer.arvadosfile._repack_writes()
 225             self.assertEqual(writer.size(), 20)
 226             self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
 227             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", export_manifest(c))
 228
 229     def test_write_rewrite2(self):
 230         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 231         with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
 232                              keep_client=keep) as c:
 233             writer = c.open("count.txt", "r+")
 234             for b in xrange(0, 10):
 235                 writer.seek(5, os.SEEK_SET)
 236                 writer.write("abcdefghij")
 237             writer.arvadosfile._repack_writes()
 238             self.assertEqual(writer.size(), 15)
 239             self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
 240             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", export_manifest(c))
 241
 242     def test_write_large_rewrite0(self):
 243         keep = ArvadosFileWriterTestCase.MockKeep({})
 244         api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
 245                                                  "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n"},
 246                                                 {"uuid":"zzzzz-4zz18-mockcollection0"})
 247         with arvados.WritableCollection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 248                              api_client=api, keep_client=keep) as c:
 249             writer = c.open("count.txt", "r+")
 250             text = ''.join(["0123456789" for a in xrange(0, 100)])
 251             for b in xrange(0, 100000):
 252                 writer.write(text)
 253             writer.seek(0, os.SEEK_SET)
 254             writer.write("foo")
 255             self.assertEqual(writer.size(), 100000000)
 256
 257             self.assertEqual(None, c._manifest_locator)
 258             self.assertEqual(True, c.modified())
 259             c.save_new("test_write_large")
 260             self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
 261             self.assertEqual(False, c.modified())
 262
 263     def test_create(self):
 264         keep = ArvadosFileWriterTestCase.MockKeep({})
 265         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
 266                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
 267                                                 {"uuid":"zzzzz-4zz18-mockcollection0"})
 268         with arvados.WritableCollection(api_client=api, keep_client=keep) as c:
 269             writer = c.open("count.txt", "w+")
 270             self.assertEqual(writer.size(), 0)
 271             writer.write("01234567")
 272             self.assertEqual(writer.size(), 8)
 273
 274             self.assertEqual(None, c._manifest_locator)
 275             self.assertEqual(True, c.modified())
 276             self.assertEqual(None, keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 277             c.save_new("test_create")
 278             self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
 279             self.assertEqual(False, c.modified())
 280             self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 281
 282
 283     def test_create_subdir(self):
 284         keep = ArvadosFileWriterTestCase.MockKeep({})
 285         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
 286                                                  "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
 287                                                 {"uuid":"zzzzz-4zz18-mockcollection0"})
 288         with arvados.WritableCollection(api_client=api, keep_client=keep) as c:
 289             writer = c.open("foo/bar/count.txt", "w+")
 290             writer.write("01234567")
 291             c.save_new("test_create")
 292
 293     def test_overwrite(self):
 294         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 295         api = ArvadosFileWriterTestCase.MockApi({"name":"test_overwrite",
 296                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
 297                                                 {"uuid":"zzzzz-4zz18-mockcollection0"})
 298         with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 299                              api_client=api, keep_client=keep) as c:
 300             writer = c.open("count.txt", "w+")
 301             self.assertEqual(writer.size(), 0)
 302             writer.write("01234567")
 303             self.assertEqual(writer.size(), 8)
 304
 305             self.assertEqual(None, c._manifest_locator)
 306             self.assertEqual(True, c.modified())
 307             c.save_new("test_overwrite")
 308             self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
 309             self.assertEqual(False, c.modified())
 310
 311     def test_file_not_found(self):
 312         with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
 313             with self.assertRaises(IOError):
 314                 writer = c.open("nocount.txt", "r")
 315
 316     def test_cannot_open_directory(self):
 317         with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
 318             with self.assertRaises(IOError):
 319                 writer = c.open(".", "r")
 320
 321     def test_create_multiple(self):
 322         keep = ArvadosFileWriterTestCase.MockKeep({})
 323         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create_multiple",
 324                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n"},
 325                                                 {"uuid":"zzzzz-4zz18-mockcollection0"})
 326         with arvados.WritableCollection(api_client=api, keep_client=keep) as c:
 327             w1 = c.open("count1.txt", "w")
 328             w2 = c.open("count2.txt", "w")
 329             w1.write("01234567")
 330             w2.write("abcdefgh")
 331             self.assertEqual(w1.size(), 8)
 332             self.assertEqual(w2.size(), 8)
 333
 334             self.assertEqual(None, c._manifest_locator)
 335             self.assertEqual(True, c.modified())
 336             self.assertEqual(None, keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 337             c.save_new("test_create_multiple")
 338             self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
 339             self.assertEqual(False, c.modified())
 340             self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 341
 342
 343 class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
 344     class MockParent(object):
 345         class MockBlockMgr(object):
 346             def __init__(self, blocks, nocache):
 347                 self.blocks = blocks
 348                 self.nocache = nocache
 349
 350             def block_prefetch(self, loc):
 351                 pass
 352
 353             def get_block_contents(self, loc, num_retries=0, cache_only=False):
 354                 if self.nocache and cache_only:
 355                     return None
 356                 return self.blocks[loc]
 357
 358         def __init__(self, blocks, nocache):
 359             self.blocks = blocks
 360             self.nocache = nocache
 361             self.lock = arvados.arvfile.NoopLock()
 362
 363         def root_collection(self):
 364             return self
 365
 366         def _my_block_manager(self):
 367             return ArvadosFileReaderTestCase.MockParent.MockBlockMgr(self.blocks, self.nocache)
 368
 369         def sync_mode(self):
 370             return SYNC_READONLY
 371
 372
 373     def make_count_reader(self, nocache=False):
 374         stream = []
 375         n = 0
 376         blocks = {}
 377         for d in ['01234', '34567', '67890']:
 378             loc = '{}+{}'.format(hashlib.md5(d).hexdigest(), len(d))
 379             blocks[loc] = d
 380             stream.append(Range(loc, n, len(d)))
 381             n += len(d)
 382         af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
 383         return ArvadosFileReader(af, "count.txt")
 384
 385     def test_read_returns_first_block(self):
 386         # read() calls will be aligned on block boundaries - see #3663.
 387         sfile = self.make_count_reader(nocache=True)
 388         self.assertEqual('123', sfile.read(10))
 389
 390     def test_successive_reads(self):
 391         sfile = self.make_count_reader(nocache=True)
 392         for expect in ['123', '456', '789', '']:
 393             self.assertEqual(expect, sfile.read(10))
 394
 395     def test_tell_after_block_read(self):
 396         sfile = self.make_count_reader(nocache=True)
 397         sfile.read(5)
 398         self.assertEqual(3, sfile.tell())
 399
 400     def test_prefetch(self):
 401         keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"})
 402         with arvados.WritableCollection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
 403             r = c.open("count.txt", "r")
 404             self.assertEqual("0123", r.read(4))
 405         self.assertIn("2e9ec317e197819358fbc43afca7d837+8", keep.requests)
 406         self.assertIn("e8dc4081b13434b45189a720b77b6818+8", keep.requests)
 407
 408     def test__eq__(self):
 409         with arvados.arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 410             with arvados.arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
 411                 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
 412                 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
 413
 414
 415 class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
 416     def reader_for(self, coll_name, **kwargs):
 417         stream = []
 418         segments = []
 419         n = 0
 420         for d in self.manifest_for(coll_name).split():
 421             try:
 422                 k = KeepLocator(d)
 423                 segments.append(Range(n, n, k.size))
 424                 stream.append(Range(d, n, k.size))
 425                 n += k.size
 426             except ValueError:
 427                 pass
 428         col = arvados.ReadOnlyCollection(keep_client=self.keep_client())
 429         col._my_block_manager().prefetch_enabled = False
 430         af = ArvadosFile(col,
 431                          stream=stream,
 432                          segments=segments)
 433         return ArvadosFileReader(af, "test", **kwargs)
 434
 435     def read_for_test(self, reader, byte_count, **kwargs):
 436         return reader.read(byte_count, **kwargs)
 437
 438
 439 class ArvadosFileReadFromTestCase(ArvadosFileReadTestCase):
 440     def read_for_test(self, reader, byte_count, **kwargs):
 441         return reader.readfrom(0, byte_count, **kwargs)
 442
 443
 444 class ArvadosFileReadAllTestCase(ArvadosFileReadTestCase):
 445     def read_for_test(self, reader, byte_count, **kwargs):
 446         return ''.join(reader.readall(**kwargs))
 447
 448
 449 class ArvadosFileReadAllDecompressedTestCase(ArvadosFileReadTestCase):
 450     def read_for_test(self, reader, byte_count, **kwargs):
 451         return ''.join(reader.readall_decompressed(**kwargs))
 452
 453
 454 class ArvadosFileReadlinesTestCase(ArvadosFileReadTestCase):
 455     def read_for_test(self, reader, byte_count, **kwargs):
 456         return ''.join(reader.readlines(**kwargs))
 457
 458 class BlockManagerTest(unittest.TestCase):
 459     def test_bufferblock_append(self):
 460         keep = ArvadosFileWriterTestCase.MockKeep({})
 461         blockmanager = arvados.arvfile.BlockManager(keep)
 462         bufferblock = blockmanager.alloc_bufferblock()
 463         bufferblock.append("foo")
 464
 465         self.assertEqual(bufferblock.size(), 3)
 466         self.assertEqual(bufferblock.buffer_view[0:3], "foo")
 467         self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 468
 469         bufferblock.append("bar")
 470
 471         self.assertEqual(bufferblock.size(), 6)
 472         self.assertEqual(bufferblock.buffer_view[0:6], "foobar")
 473         self.assertEqual(bufferblock.locator(), "3858f62230ac3c915f300c664312c63f+6")
 474
 475         bufferblock.state = arvados.arvfile.BufferBlock.PENDING
 476         with self.assertRaises(arvados.errors.AssertionError):
 477             bufferblock.append("bar")
 478
 479     def test_bufferblock_dup(self):
 480         keep = ArvadosFileWriterTestCase.MockKeep({})
 481         blockmanager = arvados.arvfile.BlockManager(keep)
 482         bufferblock = blockmanager.alloc_bufferblock()
 483         bufferblock.append("foo")
 484
 485         self.assertEqual(bufferblock.size(), 3)
 486         self.assertEqual(bufferblock.buffer_view[0:3], "foo")
 487         self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 488         bufferblock.state = arvados.arvfile.BufferBlock.PENDING
 489
 490         bufferblock2 = blockmanager.dup_block(bufferblock.blockid, None)
 491         self.assertNotEqual(bufferblock.blockid, bufferblock2.blockid)
 492
 493         bufferblock2.append("bar")
 494
 495         self.assertEqual(bufferblock2.size(), 6)
 496         self.assertEqual(bufferblock2.buffer_view[0:6], "foobar")
 497         self.assertEqual(bufferblock2.locator(), "3858f62230ac3c915f300c664312c63f+6")
 498
 499         self.assertEqual(bufferblock.size(), 3)
 500         self.assertEqual(bufferblock.buffer_view[0:3], "foo")
 501         self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 502
 503     def test_bufferblock_get(self):
 504         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 505         blockmanager = arvados.arvfile.BlockManager(keep)
 506         bufferblock = blockmanager.alloc_bufferblock()
 507         bufferblock.append("foo")
 508
 509         self.assertEqual(blockmanager.get_block_contents("781e5e245d69b566979b86e28d23f2c7+10", 1), "0123456789")
 510         self.assertEqual(blockmanager.get_block_contents(bufferblock.blockid, 1), "foo")
 511
 512     def test_bufferblock_commit(self):
 513         mockkeep = mock.MagicMock()
 514         blockmanager = arvados.arvfile.BlockManager(mockkeep)
 515         bufferblock = blockmanager.alloc_bufferblock()
 516         bufferblock.append("foo")
 517         blockmanager.commit_all()
 518         self.assertTrue(mockkeep.put.called)
 519         self.assertEqual(bufferblock.state, arvados.arvfile.BufferBlock.COMMITTED)
 520         self.assertIsNone(bufferblock.buffer_view)
 521
 522
 523     def test_bufferblock_commit_with_error(self):
 524         mockkeep = mock.MagicMock()
 525         mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail")
 526         blockmanager = arvados.arvfile.BlockManager(mockkeep)
 527         bufferblock = blockmanager.alloc_bufferblock()
 528         bufferblock.append("foo")
 529         with self.assertRaises(arvados.errors.KeepWriteError) as err:
 530             blockmanager.commit_all()
 531         self.assertEquals(str(err.exception), "Error writing some blocks: acbd18db4cc2f85cedef654fccc4a4d8+3 raised KeepWriteError (fail)")
 532         self.assertEqual(bufferblock.state, arvados.arvfile.BufferBlock.PENDING)