sdk/python/tests/test_arvfile.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from __future__ import absolute_import
   6 from builtins import hex
   7 from builtins import str
   8 from builtins import range
   9 from builtins import object
  10 import datetime
  11 import mock
  12 import os
  13 import unittest
  14 import time
  15
  16 import arvados
  17 from arvados._ranges import Range
  18 from arvados.keep import KeepLocator
  19 from arvados.collection import Collection
  20 from arvados.arvfile import ArvadosFile, ArvadosFileReader
  21
  22 from . import arvados_testutil as tutil
  23 from .test_stream import StreamFileReaderTestCase, StreamRetryTestMixin
  24
  25 class ArvadosFileWriterTestCase(unittest.TestCase):
  26     class MockKeep(object):
  27         def __init__(self, blocks):
  28             self.blocks = blocks
  29             self.requests = []
  30             self.num_prefetch_threads = 1
  31         def get(self, locator, num_retries=0, prefetch=False):
  32             self.requests.append(locator)
  33             return self.blocks.get(locator)
  34         def get_from_cache(self, locator):
  35             self.requests.append(locator)
  36             return self.blocks.get(locator)
  37         def put(self, data, num_retries=None, copies=None, classes=[]):
  38             pdh = tutil.str_keep_locator(data)
  39             self.blocks[pdh] = bytes(data)
  40             return pdh
  41         def block_prefetch(self, loc):
  42             self.requests.append(loc)
  43
  44     class MockApi(object):
  45         def __init__(self, b, r):
  46             self.body = b
  47             self.response = r
  48             self._schema = ArvadosFileWriterTestCase.MockApi.MockSchema()
  49             self._rootDesc = {}
  50         class MockSchema(object):
  51             def __init__(self):
  52                 self.schemas = {'Collection': {'properties': {'replication_desired': {'type':'integer'}}}}
  53         class MockCollections(object):
  54             def __init__(self, b, r):
  55                 self.body = b
  56                 self.response = r
  57             class Execute(object):
  58                 def __init__(self, r):
  59                     self.response = r
  60                 def execute(self, num_retries=None):
  61                     return self.response
  62             def create(self, ensure_unique_name=False, body=None):
  63                 if body != self.body:
  64                     raise Exception("Body %s does not match expectation %s" % (body, self.body))
  65                 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
  66             def update(self, uuid=None, body=None):
  67                 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
  68         def collections(self):
  69             return ArvadosFileWriterTestCase.MockApi.MockCollections(self.body, self.response)
  70
  71
  72     def test_truncate(self):
  73         keep = ArvadosFileWriterTestCase.MockKeep({
  74             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
  75         })
  76         api = ArvadosFileWriterTestCase.MockApi({
  77             "name": "test_truncate",
  78             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
  79             "replication_desired": None,
  80         }, {
  81             "uuid": "zzzzz-4zz18-mockcollection0",
  82             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
  83             "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52",
  84         })
  85         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
  86                         api_client=api, keep_client=keep) as c:
  87             writer = c.open("count.txt", "rb+")
  88             self.assertEqual(writer.size(), 10)
  89             self.assertEqual(b"0123456789", writer.read(12))
  90
  91             writer.truncate(8)
  92
  93             # Make sure reading off the end doesn't break
  94             self.assertEqual(b"", writer.read(12))
  95
  96             self.assertEqual(writer.size(), 8)
  97             writer.seek(0, os.SEEK_SET)
  98             self.assertEqual(b"01234567", writer.read(12))
  99
 100             self.assertIsNone(c.manifest_locator())
 101             self.assertTrue(c.modified())
 102             c.save_new("test_truncate")
 103             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 104             self.assertFalse(c.modified())
 105
 106
 107     def test_truncate2(self):
 108         keep = ArvadosFileWriterTestCase.MockKeep({
 109             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 110         })
 111         api = ArvadosFileWriterTestCase.MockApi({
 112             "name": "test_truncate2",
 113             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
 114             "replication_desired": None,
 115         }, {
 116             "uuid": "zzzzz-4zz18-mockcollection0",
 117             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
 118             "portable_data_hash": "272da898abdf86ddc71994835e3155f8+95",
 119         })
 120         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 121                         api_client=api, keep_client=keep) as c:
 122             writer = c.open("count.txt", "rb+")
 123             self.assertEqual(writer.size(), 10)
 124             self.assertEqual(b"0123456789", writer.read(12))
 125
 126             # extend file size
 127             writer.truncate(12)
 128
 129             self.assertEqual(writer.size(), 12)
 130             writer.seek(0, os.SEEK_SET)
 131             self.assertEqual(b"0123456789\x00\x00", writer.read(12))
 132
 133             self.assertIsNone(c.manifest_locator())
 134             self.assertTrue(c.modified())
 135             c.save_new("test_truncate2")
 136             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 137             self.assertFalse(c.modified())
 138
 139     def test_truncate3(self):
 140         keep = ArvadosFileWriterTestCase.MockKeep({
 141             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 142             "a925576942e94b2ef57a066101b48876+10": b"abcdefghij",
 143         })
 144         api = ArvadosFileWriterTestCase.MockApi({
 145             "name": "test_truncate",
 146             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
 147             "replication_desired": None,
 148         }, {
 149             "uuid": "zzzzz-4zz18-mockcollection0",
 150             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
 151             "portable_data_hash": "7fcd0eaac3aad4c31a6a0e756475da92+52",
 152         })
 153         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n',
 154                         api_client=api, keep_client=keep) as c:
 155             writer = c.open("count.txt", "rb+")
 156             self.assertEqual(writer.size(), 20)
 157             self.assertEqual(b"0123456789ab", writer.read(12))
 158             self.assertEqual(12, writer.tell())
 159
 160             writer.truncate(8)
 161
 162             # Make sure reading off the end doesn't break
 163             self.assertEqual(12, writer.tell())
 164             self.assertEqual(b"", writer.read(12))
 165
 166             self.assertEqual(writer.size(), 8)
 167             self.assertEqual(2, writer.seek(-10, os.SEEK_CUR))
 168             self.assertEqual(b"234567", writer.read(12))
 169
 170             self.assertIsNone(c.manifest_locator())
 171             self.assertTrue(c.modified())
 172             c.save_new("test_truncate")
 173             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 174             self.assertFalse(c.modified())
 175
 176     def test_write_to_end(self):
 177         keep = ArvadosFileWriterTestCase.MockKeep({
 178             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 179         })
 180         api = ArvadosFileWriterTestCase.MockApi({
 181             "name": "test_append",
 182             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
 183             "replication_desired": None,
 184         }, {
 185             "uuid": "zzzzz-4zz18-mockcollection0",
 186             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
 187             "portable_data_hash": "c5c3af76565c8efb6a806546bcf073f3+88",
 188         })
 189         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 190                              api_client=api, keep_client=keep) as c:
 191             writer = c.open("count.txt", "rb+")
 192             self.assertEqual(writer.size(), 10)
 193
 194             self.assertEqual(5, writer.seek(5, os.SEEK_SET))
 195             self.assertEqual(b"56789", writer.read(8))
 196
 197             writer.seek(10, os.SEEK_SET)
 198             writer.write("foo")
 199             self.assertEqual(writer.size(), 13)
 200
 201             writer.seek(5, os.SEEK_SET)
 202             self.assertEqual(b"56789foo", writer.read(8))
 203
 204             self.assertIsNone(c.manifest_locator())
 205             self.assertTrue(c.modified())
 206             self.assertIsNone(keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
 207
 208             c.save_new("test_append")
 209             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 210             self.assertFalse(c.modified())
 211             self.assertEqual(b"foo", keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
 212
 213
 214     def test_append(self):
 215         keep = ArvadosFileWriterTestCase.MockKeep({
 216             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 217         })
 218         for (mode, convert) in (
 219                 ('a+', lambda data: data.decode(encoding='utf-8')),
 220                 ('at+', lambda data: data.decode(encoding='utf-8')),
 221                 ('ab+', lambda data: data)):
 222             c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
 223             writer = c.open("count.txt", mode)
 224             self.assertEqual(writer.read(20), convert(b"0123456789"))
 225
 226             writer.seek(0, os.SEEK_SET)
 227             writer.write(convert(b"hello"))
 228             self.assertEqual(writer.read(), convert(b""))
 229             if 'b' in mode:
 230                 writer.seek(-5, os.SEEK_CUR)
 231                 self.assertEqual(writer.read(3), convert(b"hel"))
 232                 self.assertEqual(writer.read(), convert(b"lo"))
 233             else:
 234                 with self.assertRaises(IOError):
 235                     writer.seek(-5, os.SEEK_CUR)
 236                 with self.assertRaises(IOError):
 237                     writer.seek(-3, os.SEEK_END)
 238             writer.seek(0, os.SEEK_SET)
 239             writer.read(7)
 240             self.assertEqual(7, writer.tell())
 241             self.assertEqual(7, writer.seek(7, os.SEEK_SET))
 242
 243             writer.seek(0, os.SEEK_SET)
 244             self.assertEqual(writer.read(), convert(b"0123456789hello"))
 245
 246             writer.seek(0)
 247             writer.write(convert(b"world"))
 248             self.assertEqual(writer.read(), convert(b""))
 249             writer.seek(0)
 250             self.assertEqual(writer.read(), convert(b"0123456789helloworld"))
 251
 252             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.portable_manifest_text())
 253
 254     def test_write_at_beginning(self):
 255         keep = ArvadosFileWriterTestCase.MockKeep({
 256             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 257         })
 258         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 259                              keep_client=keep) as c:
 260             writer = c.open("count.txt", "rb+")
 261             self.assertEqual(b"0123456789", writer.readfrom(0, 13))
 262             writer.seek(0, os.SEEK_SET)
 263             writer.write("foo")
 264             self.assertEqual(writer.size(), 10)
 265             self.assertEqual(b"foo3456789", writer.readfrom(0, 13))
 266             self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", c.portable_manifest_text())
 267
 268     def test_write_empty(self):
 269         keep = ArvadosFileWriterTestCase.MockKeep({})
 270         with Collection(keep_client=keep) as c:
 271             writer = c.open("count.txt", "wb")
 272             self.assertEqual(writer.size(), 0)
 273             self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:count.txt\n", c.portable_manifest_text())
 274
 275     def test_save_manifest_text(self):
 276         keep = ArvadosFileWriterTestCase.MockKeep({})
 277         with Collection(keep_client=keep) as c:
 278             writer = c.open("count.txt", "wb")
 279             writer.write(b"0123456789")
 280             self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
 281             self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 282
 283             self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.save_new(create_collection_record=False))
 284             self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 285
 286     def test_get_manifest_text_commits(self):
 287          keep = ArvadosFileWriterTestCase.MockKeep({})
 288          with Collection(keep_client=keep) as c:
 289              writer = c.open("count.txt", "wb")
 290              writer.write("0123456789")
 291              self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
 292              self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 293              self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.manifest_text())
 294              self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 295
 296
 297     def test_write_in_middle(self):
 298         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
 299         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 300                              keep_client=keep) as c:
 301             writer = c.open("count.txt", "rb+")
 302             self.assertEqual(b"0123456789", writer.readfrom(0, 13))
 303             writer.seek(3, os.SEEK_SET)
 304             writer.write("foo")
 305             self.assertEqual(writer.size(), 10)
 306             self.assertEqual(b"012foo6789", writer.readfrom(0, 13))
 307             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", c.portable_manifest_text())
 308
 309     def test_write_at_end(self):
 310         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
 311         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 312                              keep_client=keep) as c:
 313             writer = c.open("count.txt", "rb+")
 314             self.assertEqual(b"0123456789", writer.readfrom(0, 13))
 315             writer.seek(7, os.SEEK_SET)
 316             writer.write("foo")
 317             self.assertEqual(writer.size(), 10)
 318             self.assertEqual(b"0123456foo", writer.readfrom(0, 13))
 319             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", c.portable_manifest_text())
 320
 321     def test_write_across_segment_boundary(self):
 322         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
 323         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
 324                              keep_client=keep) as c:
 325             writer = c.open("count.txt", "rb+")
 326             self.assertEqual(b"012345678901234", writer.readfrom(0, 15))
 327             writer.seek(7, os.SEEK_SET)
 328             writer.write("foobar")
 329             self.assertEqual(writer.size(), 20)
 330             self.assertEqual(b"0123456foobar34", writer.readfrom(0, 15))
 331             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", c.portable_manifest_text())
 332
 333     def test_write_across_several_segments(self):
 334         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
 335         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
 336                              keep_client=keep) as c:
 337             writer = c.open("count.txt", "rb+")
 338             self.assertEqual(b"012301230123", writer.readfrom(0, 15))
 339             writer.seek(2, os.SEEK_SET)
 340             writer.write("abcdefg")
 341             self.assertEqual(writer.size(), 12)
 342             self.assertEqual(b"01abcdefg123", writer.readfrom(0, 15))
 343             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", c.portable_manifest_text())
 344
 345     def test_write_large(self):
 346         keep = ArvadosFileWriterTestCase.MockKeep({})
 347         api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
 348                                                  "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n",
 349                                                  "replication_desired":None},
 350                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 351                                                  "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n",
 352                                                  "portable_data_hash":"9132ca8e3f671c76103a38f5bc24328c+108"})
 353         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 354                              api_client=api, keep_client=keep) as c:
 355             writer = c.open("count.txt", "rb+")
 356             text = "0123456789" * 100
 357             for b in range(0, 100000):
 358                 writer.write(text)
 359             self.assertEqual(writer.size(), 100000000)
 360
 361             self.assertIsNone(c.manifest_locator())
 362             self.assertTrue(c.modified())
 363             c.save_new("test_write_large")
 364             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 365             self.assertFalse(c.modified())
 366
 367     def test_large_write(self):
 368         keep = ArvadosFileWriterTestCase.MockKeep({})
 369         api = ArvadosFileWriterTestCase.MockApi({}, {})
 370         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 371                              api_client=api, keep_client=keep) as c:
 372             writer = c.open("count.txt", "rb+")
 373             self.assertEqual(writer.size(), 0)
 374
 375             text = "0123456789"
 376             writer.write(text)
 377             text = "0123456789" * 9999999
 378             writer.write(text)
 379             self.assertEqual(writer.size(), 100000000)
 380
 381             self.assertEqual(c.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 48dd23ea1645fd47d789804d71b5bb8e+67108864 77c57dc6ac5a10bb2205caaa73187994+32891126 0:100000000:count.txt\n")
 382
 383     def test_sparse_write(self):
 384         keep = ArvadosFileWriterTestCase.MockKeep({})
 385         api = ArvadosFileWriterTestCase.MockApi({}, {})
 386         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 387                              api_client=api, keep_client=keep) as c:
 388             writer = c.open("count.txt", "rb+")
 389             self.assertEqual(writer.size(), 0)
 390
 391             text = b"0123456789"
 392             writer.seek(2)
 393             writer.write(text)
 394             self.assertEqual(writer.size(), 12)
 395             writer.seek(0, os.SEEK_SET)
 396             self.assertEqual(writer.read(), b"\x00\x00"+text)
 397
 398             self.assertEqual(c.manifest_text(), ". 7f614da9329cd3aebf59b91aadc30bf0+67108864 781e5e245d69b566979b86e28d23f2c7+10 0:2:count.txt 67108864:10:count.txt\n")
 399
 400     def test_sparse_write2(self):
 401         keep = ArvadosFileWriterTestCase.MockKeep({})
 402         api = ArvadosFileWriterTestCase.MockApi({}, {})
 403         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 404                              api_client=api, keep_client=keep) as c:
 405             writer = c.open("count.txt", "rb+")
 406             self.assertEqual(writer.size(), 0)
 407
 408             text = "0123456789"
 409             writer.seek((arvados.config.KEEP_BLOCK_SIZE*2) + 2)
 410             writer.write(text)
 411             self.assertEqual(writer.size(), (arvados.config.KEEP_BLOCK_SIZE*2) + 12)
 412             writer.seek(0, os.SEEK_SET)
 413
 414             self.assertEqual(c.manifest_text(), ". 7f614da9329cd3aebf59b91aadc30bf0+67108864 781e5e245d69b566979b86e28d23f2c7+10 0:67108864:count.txt 0:67108864:count.txt 0:2:count.txt 67108864:10:count.txt\n")
 415
 416     def test_sparse_write3(self):
 417         keep = ArvadosFileWriterTestCase.MockKeep({})
 418         api = ArvadosFileWriterTestCase.MockApi({}, {})
 419         for r in [[0, 1, 2, 3, 4], [4, 3, 2, 1, 0], [3, 2, 0, 4, 1]]:
 420             with Collection(api_client=api, keep_client=keep) as c:
 421                 writer = c.open("count.txt", "rb+")
 422                 self.assertEqual(writer.size(), 0)
 423
 424                 for i in r:
 425                     w = ("%s" % i) * 10
 426                     writer.seek(i*10)
 427                     writer.write(w.encode())
 428                 writer.seek(0)
 429                 self.assertEqual(writer.read(), b"00000000001111111111222222222233333333334444444444")
 430
 431     def test_sparse_write4(self):
 432         keep = ArvadosFileWriterTestCase.MockKeep({})
 433         api = ArvadosFileWriterTestCase.MockApi({}, {})
 434         for r in [[0, 1, 2, 4], [4, 2, 1, 0], [2, 0, 4, 1]]:
 435             with Collection(api_client=api, keep_client=keep) as c:
 436                 writer = c.open("count.txt", "rb+")
 437                 self.assertEqual(writer.size(), 0)
 438
 439                 for i in r:
 440                     w = ("%s" % i) * 10
 441                     writer.seek(i*10)
 442                     writer.write(w.encode())
 443                 writer.seek(0)
 444                 self.assertEqual(writer.read(), b"000000000011111111112222222222\x00\x00\x00\x00\x00\x00\x00\x00\x00\x004444444444")
 445
 446     def test_rewrite_on_empty_file(self):
 447         keep = ArvadosFileWriterTestCase.MockKeep({})
 448         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 449                              keep_client=keep) as c:
 450             writer = c.open("count.txt", "rb+")
 451             for b in range(0, 10):
 452                 writer.seek(0, os.SEEK_SET)
 453                 writer.write("0123456789")
 454
 455             self.assertEqual(writer.size(), 10)
 456             self.assertEqual(b"0123456789", writer.readfrom(0, 20))
 457             self.assertEqual(". 7a08b07e84641703e5f2c836aa59a170+100 90:10:count.txt\n", c.portable_manifest_text())
 458             writer.flush()
 459             self.assertEqual(writer.size(), 10)
 460             self.assertEqual(b"0123456789", writer.readfrom(0, 20))
 461             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.portable_manifest_text())
 462
 463     def test_rewrite_append_existing_file(self):
 464         keep = ArvadosFileWriterTestCase.MockKeep({
 465             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 466         })
 467         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
 468                              keep_client=keep) as c:
 469             writer = c.open("count.txt", "rb+")
 470             for b in range(0, 10):
 471                 writer.seek(10, os.SEEK_SET)
 472                 writer.write("abcdefghij")
 473
 474             self.assertEqual(writer.size(), 20)
 475             self.assertEqual(b"0123456789abcdefghij", writer.readfrom(0, 20))
 476             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:10:count.txt 100:10:count.txt\n", c.portable_manifest_text())
 477
 478             writer.arvadosfile.flush()
 479             self.assertEqual(writer.size(), 20)
 480             self.assertEqual(b"0123456789abcdefghij", writer.readfrom(0, 20))
 481             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.portable_manifest_text())
 482
 483     def test_rewrite_over_existing_file(self):
 484         keep = ArvadosFileWriterTestCase.MockKeep({
 485             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 486         })
 487         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
 488                              keep_client=keep) as c:
 489             writer = c.open("count.txt", "rb+")
 490             for b in range(0, 10):
 491                 writer.seek(5, os.SEEK_SET)
 492                 writer.write("abcdefghij")
 493
 494             self.assertEqual(writer.size(), 15)
 495             self.assertEqual(b"01234abcdefghij", writer.readfrom(0, 20))
 496             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:5:count.txt 100:10:count.txt\n", c.portable_manifest_text())
 497
 498             writer.arvadosfile.flush()
 499
 500             self.assertEqual(writer.size(), 15)
 501             self.assertEqual(b"01234abcdefghij", writer.readfrom(0, 20))
 502             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.portable_manifest_text())
 503
 504     def test_write_large_rewrite(self):
 505         keep = ArvadosFileWriterTestCase.MockKeep({})
 506         api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
 507                                                  "manifest_text": ". 3dc0d4bc21f48060bedcb2c91af4f906+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 0:3:count.txt 32892006:67107997:count.txt 3:32892000:count.txt\n",
 508                                                  "replication_desired":None},
 509                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 510                                                  "manifest_text": ". 3dc0d4bc21f48060bedcb2c91af4f906+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 0:3:count.txt 32892006:67107997:count.txt 3:32892000:count.txt\n",
 511                                                  "portable_data_hash":"217665c6b713e1b78dfba7ebd42344db+156"})
 512         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 513                              api_client=api, keep_client=keep) as c:
 514             writer = c.open("count.txt", "rb+")
 515             text = b''.join([b"0123456789" for a in range(0, 100)])
 516             for b in range(0, 100000):
 517                 writer.write(text)
 518             writer.seek(0, os.SEEK_SET)
 519             writer.write("foo")
 520             self.assertEqual(writer.size(), 100000000)
 521
 522             self.assertIsNone(c.manifest_locator())
 523             self.assertTrue(c.modified())
 524             c.save_new("test_write_large")
 525             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 526             self.assertFalse(c.modified())
 527
 528     def test_create(self):
 529         keep = ArvadosFileWriterTestCase.MockKeep({})
 530         api = ArvadosFileWriterTestCase.MockApi({
 531             "name":"test_create",
 532             "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 533             "replication_desired":None,
 534         }, {
 535             "uuid":"zzzzz-4zz18-mockcollection0",
 536             "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 537             "portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51",
 538         })
 539         with Collection(api_client=api, keep_client=keep) as c:
 540             writer = c.open("count.txt", "wb+")
 541             self.assertEqual(writer.size(), 0)
 542             writer.write("01234567")
 543             self.assertEqual(writer.size(), 8)
 544
 545             self.assertIsNone(c.manifest_locator())
 546             self.assertTrue(c.modified())
 547             self.assertIsNone(keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 548             c.save_new("test_create")
 549             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 550             self.assertFalse(c.modified())
 551             self.assertEqual(b"01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 552
 553
 554     def test_create_subdir(self):
 555         keep = ArvadosFileWriterTestCase.MockKeep({})
 556         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
 557                                                  "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 558                                                  "replication_desired":None},
 559                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 560                                                  "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 561                                                  "portable_data_hash":"1b02aaa62528d28a5be41651cbb9d7c7+59"})
 562         with Collection(api_client=api, keep_client=keep) as c:
 563             self.assertIsNone(c.api_response())
 564             writer = c.open("foo/bar/count.txt", "wb+")
 565             writer.write("01234567")
 566             self.assertFalse(c.committed())
 567             c.save_new("test_create")
 568             self.assertTrue(c.committed())
 569             self.assertEqual(c.api_response(), api.response)
 570
 571     def test_overwrite(self):
 572         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 573         api = ArvadosFileWriterTestCase.MockApi({"name":"test_overwrite",
 574                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 575                                                  "replication_desired":None},
 576                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 577                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 578                                                  "portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51"})
 579         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 580                              api_client=api, keep_client=keep) as c:
 581             writer = c.open("count.txt", "wb+")
 582             self.assertEqual(writer.size(), 0)
 583             writer.write("01234567")
 584             self.assertEqual(writer.size(), 8)
 585
 586             self.assertIsNone(c.manifest_locator())
 587             self.assertTrue(c.modified())
 588             c.save_new("test_overwrite")
 589             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 590             self.assertFalse(c.modified())
 591
 592     def test_file_not_found(self):
 593         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
 594             with self.assertRaises(IOError):
 595                 writer = c.open("nocount.txt", "rb")
 596
 597     def test_cannot_open_directory(self):
 598         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
 599             with self.assertRaises(IOError):
 600                 writer = c.open(".", "rb")
 601
 602     def test_create_multiple(self):
 603         keep = ArvadosFileWriterTestCase.MockKeep({})
 604         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create_multiple",
 605                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n",
 606                                                  "replication_desired":None},
 607                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 608                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n",
 609                                                  "portable_data_hash":"71e7bb6c00d31fc2b4364199fd97be08+102"})
 610         with Collection(api_client=api, keep_client=keep) as c:
 611             w1 = c.open("count1.txt", "wb")
 612             w2 = c.open("count2.txt", "wb")
 613             w1.write("01234567")
 614             w2.write("abcdefgh")
 615             self.assertEqual(w1.size(), 8)
 616             self.assertEqual(w2.size(), 8)
 617
 618             self.assertIsNone(c.manifest_locator())
 619             self.assertTrue(c.modified())
 620             self.assertIsNone(keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 621             c.save_new("test_create_multiple")
 622             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 623             self.assertFalse(c.modified())
 624             self.assertEqual(b"01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 625
 626
 627 class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
 628     class MockParent(object):
 629         class MockBlockMgr(object):
 630             def __init__(self, blocks, nocache):
 631                 self.blocks = blocks
 632                 self.nocache = nocache
 633                 self._keep = ArvadosFileWriterTestCase.MockKeep({})
 634
 635             def block_prefetch(self, loc):
 636                 pass
 637
 638             def get_block_contents(self, loc, num_retries=0, cache_only=False):
 639                 if self.nocache and cache_only:
 640                     return None
 641                 return self.blocks[loc]
 642
 643         def __init__(self, blocks, nocache):
 644             self.blocks = blocks
 645             self.nocache = nocache
 646             self.lock = arvados.arvfile.NoopLock()
 647
 648         def root_collection(self):
 649             return self
 650
 651         def _my_block_manager(self):
 652             return ArvadosFileReaderTestCase.MockParent.MockBlockMgr(self.blocks, self.nocache)
 653
 654
 655     def make_count_reader(self, nocache=False):
 656         stream = []
 657         n = 0
 658         blocks = {}
 659         for d in [b'01234', b'34567', b'67890']:
 660             loc = tutil.str_keep_locator(d)
 661             blocks[loc] = d
 662             stream.append(Range(loc, n, len(d)))
 663             n += len(d)
 664         af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), "count.txt", stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
 665         return ArvadosFileReader(af, mode="rb")
 666
 667     def test_read_block_crossing_behavior(self):
 668         # read() needs to return all the data requested if possible, even if it
 669         # crosses uncached blocks: https://arvados.org/issues/5856
 670         sfile = self.make_count_reader(nocache=True)
 671         self.assertEqual(b'12345678', sfile.read(8))
 672
 673     def test_successive_reads(self):
 674         # Override StreamFileReaderTestCase.test_successive_reads
 675         sfile = self.make_count_reader(nocache=True)
 676         self.assertEqual(b'1234', sfile.read(4))
 677         self.assertEqual(b'5678', sfile.read(4))
 678         self.assertEqual(b'9', sfile.read(4))
 679         self.assertEqual(b'', sfile.read(4))
 680
 681     def test_tell_after_block_read(self):
 682         # Override StreamFileReaderTestCase.test_tell_after_block_read
 683         sfile = self.make_count_reader(nocache=True)
 684         self.assertEqual(b'12345678', sfile.read(8))
 685         self.assertEqual(8, sfile.tell())
 686
 687     def test_prefetch(self):
 688         keep = ArvadosFileWriterTestCase.MockKeep({
 689             "2e9ec317e197819358fbc43afca7d837+8": b"01234567",
 690             "e8dc4081b13434b45189a720b77b6818+8": b"abcdefgh",
 691         })
 692         with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
 693             r = c.open("count.txt", "rb")
 694             self.assertEqual(b"0123", r.read(4))
 695         self.assertIn("2e9ec317e197819358fbc43afca7d837+8", keep.requests)
 696         self.assertIn("e8dc4081b13434b45189a720b77b6818+8", keep.requests)
 697
 698     def test__eq__from_manifest(self):
 699         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 700             with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
 701                 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
 702                 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
 703
 704     def test__eq__from_writes(self):
 705         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 706             with Collection() as c2:
 707                 f = c2.open("count1.txt", "wb")
 708                 f.write("0123456789")
 709
 710                 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
 711                 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
 712
 713     def test__ne__(self):
 714         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 715             with Collection() as c2:
 716                 f = c2.open("count1.txt", "wb")
 717                 f.write("1234567890")
 718
 719                 self.assertTrue(c1["count1.txt"] != c2["count1.txt"])
 720                 self.assertFalse(c1["count1.txt"] == c2["count1.txt"])
 721
 722
 723 class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
 724     def reader_for(self, coll_name, **kwargs):
 725         stream = []
 726         segments = []
 727         n = 0
 728         for d in self.manifest_for(coll_name).split():
 729             try:
 730                 k = KeepLocator(d)
 731                 segments.append(Range(n, n, k.size))
 732                 stream.append(Range(d, n, k.size))
 733                 n += k.size
 734             except ValueError:
 735                 pass
 736
 737         blockmanager = arvados.arvfile._BlockManager(self.keep_client())
 738         blockmanager.prefetch_enabled = False
 739         col = Collection(keep_client=self.keep_client(), block_manager=blockmanager)
 740         af = ArvadosFile(col, "test",
 741                          stream=stream,
 742                          segments=segments)
 743         kwargs.setdefault('mode', 'rb')
 744         return ArvadosFileReader(af, **kwargs)
 745
 746     def read_for_test(self, reader, byte_count, **kwargs):
 747         return reader.read(byte_count, **kwargs)
 748
 749
 750 class ArvadosFileReadFromTestCase(ArvadosFileReadTestCase):
 751     def read_for_test(self, reader, byte_count, **kwargs):
 752         return reader.readfrom(0, byte_count, **kwargs)
 753
 754
 755 class ArvadosFileReadAllTestCase(ArvadosFileReadTestCase):
 756     def read_for_test(self, reader, byte_count, **kwargs):
 757         return b''.join(reader.readall(**kwargs))
 758
 759
 760 class ArvadosFileReadAllDecompressedTestCase(ArvadosFileReadTestCase):
 761     def read_for_test(self, reader, byte_count, **kwargs):
 762         return b''.join(reader.readall_decompressed(**kwargs))
 763
 764
 765 class ArvadosFileReadlinesTestCase(ArvadosFileReadTestCase):
 766     def read_for_test(self, reader, byte_count, **kwargs):
 767         return ''.join(reader.readlines(**kwargs)).encode()
 768
 769
 770 class ArvadosFileTestCase(unittest.TestCase):
 771     def datetime_to_hex(self, dt):
 772         return hex(int(time.mktime(dt.timetuple())))[2:]
 773
 774     def test_permission_expired(self):
 775         base_manifest = ". 781e5e245d69b566979b86e28d23f2c7+10+A715fd31f8111894f717eb1003c1b0216799dd9ec@{} 0:10:count.txt\n"
 776         now = datetime.datetime.now()
 777         a_week_ago = now - datetime.timedelta(days=7)
 778         a_month_ago = now - datetime.timedelta(days=30)
 779         a_week_from_now = now + datetime.timedelta(days=7)
 780         with Collection(base_manifest.format(self.datetime_to_hex(a_week_from_now))) as c:
 781             self.assertFalse(c.find('count.txt').permission_expired())
 782         with Collection(base_manifest.format(self.datetime_to_hex(a_week_ago))) as c:
 783             f = c.find('count.txt')
 784             self.assertTrue(f.permission_expired())
 785             self.assertTrue(f.permission_expired(a_week_from_now))
 786             self.assertFalse(f.permission_expired(a_month_ago))
 787
 788
 789 class BlockManagerTest(unittest.TestCase):
 790     def test_bufferblock_append(self):
 791         keep = ArvadosFileWriterTestCase.MockKeep({})
 792         with arvados.arvfile._BlockManager(keep) as blockmanager:
 793             bufferblock = blockmanager.alloc_bufferblock()
 794             bufferblock.append("foo")
 795
 796             self.assertEqual(bufferblock.size(), 3)
 797             self.assertEqual(bufferblock.buffer_view[0:3], b"foo")
 798             self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 799
 800             bufferblock.append("bar")
 801
 802             self.assertEqual(bufferblock.size(), 6)
 803             self.assertEqual(bufferblock.buffer_view[0:6], b"foobar")
 804             self.assertEqual(bufferblock.locator(), "3858f62230ac3c915f300c664312c63f+6")
 805
 806             bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
 807             with self.assertRaises(arvados.errors.AssertionError):
 808                 bufferblock.append("bar")
 809
 810     def test_bufferblock_dup(self):
 811         keep = ArvadosFileWriterTestCase.MockKeep({})
 812         with arvados.arvfile._BlockManager(keep) as blockmanager:
 813             bufferblock = blockmanager.alloc_bufferblock()
 814             bufferblock.append("foo")
 815
 816             self.assertEqual(bufferblock.size(), 3)
 817             self.assertEqual(bufferblock.buffer_view[0:3], b"foo")
 818             self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 819             bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
 820
 821             bufferblock2 = blockmanager.dup_block(bufferblock, None)
 822             self.assertNotEqual(bufferblock.blockid, bufferblock2.blockid)
 823
 824             bufferblock2.append("bar")
 825
 826             self.assertEqual(bufferblock2.size(), 6)
 827             self.assertEqual(bufferblock2.buffer_view[0:6], b"foobar")
 828             self.assertEqual(bufferblock2.locator(), "3858f62230ac3c915f300c664312c63f+6")
 829
 830             self.assertEqual(bufferblock.size(), 3)
 831             self.assertEqual(bufferblock.buffer_view[0:3], b"foo")
 832             self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 833
 834     def test_bufferblock_get(self):
 835         keep = ArvadosFileWriterTestCase.MockKeep({
 836             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 837         })
 838         with arvados.arvfile._BlockManager(keep) as blockmanager:
 839             bufferblock = blockmanager.alloc_bufferblock()
 840             bufferblock.append("foo")
 841
 842             self.assertEqual(blockmanager.get_block_contents("781e5e245d69b566979b86e28d23f2c7+10", 1), b"0123456789")
 843             self.assertEqual(blockmanager.get_block_contents(bufferblock.blockid, 1), b"foo")
 844
 845     def test_bufferblock_commit(self):
 846         mockkeep = mock.MagicMock()
 847         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
 848             bufferblock = blockmanager.alloc_bufferblock()
 849             bufferblock.owner = mock.MagicMock(spec=arvados.arvfile.ArvadosFile)
 850             def flush(sync=None):
 851                 blockmanager.commit_bufferblock(bufferblock, sync)
 852             bufferblock.owner.flush.side_effect = flush
 853             bufferblock.append("foo")
 854             blockmanager.commit_all()
 855             self.assertTrue(bufferblock.owner.flush.called)
 856             self.assertTrue(mockkeep.put.called)
 857             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
 858             self.assertIsNone(bufferblock.buffer_view)
 859
 860     def test_bufferblock_commit_pending(self):
 861         # Test for bug #7225
 862         mockkeep = mock.MagicMock()
 863         mockkeep.put.side_effect = lambda *args, **kwargs: time.sleep(1)
 864         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
 865             bufferblock = blockmanager.alloc_bufferblock()
 866             bufferblock.append("foo")
 867
 868             blockmanager.commit_bufferblock(bufferblock, False)
 869             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.PENDING)
 870
 871             blockmanager.commit_bufferblock(bufferblock, True)
 872             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
 873
 874     def test_bufferblock_commit_with_error(self):
 875         mockkeep = mock.MagicMock()
 876         mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail")
 877         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
 878             bufferblock = blockmanager.alloc_bufferblock()
 879             bufferblock.owner = mock.MagicMock(spec=arvados.arvfile.ArvadosFile)
 880             def flush(sync=None):
 881                 blockmanager.commit_bufferblock(bufferblock, sync)
 882             bufferblock.owner.flush.side_effect = flush
 883             bufferblock.append("foo")
 884             with self.assertRaises(arvados.errors.KeepWriteError) as err:
 885                 blockmanager.commit_all()
 886             self.assertTrue(bufferblock.owner.flush.called)
 887             self.assertEqual(str(err.exception), "Error writing some blocks: block acbd18db4cc2f85cedef654fccc4a4d8+3 raised KeepWriteError (fail)")
 888             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.ERROR)