sdk/python/tests/test_arvfile.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from __future__ import absolute_import
   6 from builtins import hex
   7 from builtins import str
   8 from builtins import range
   9 from builtins import object
  10 import datetime
  11 import os
  12 import unittest
  13 import time
  14
  15 from unittest import mock
  16
  17 import arvados
  18 from arvados._ranges import Range
  19 from arvados.keep import KeepLocator
  20 from arvados.collection import Collection
  21 from arvados.arvfile import ArvadosFile, ArvadosFileReader
  22
  23 from . import arvados_testutil as tutil
  24 from .test_stream import StreamFileReaderTestCase, StreamRetryTestMixin
  25
  26 class ArvadosFileWriterTestCase(unittest.TestCase):
  27     class MockKeep(object):
  28         def __init__(self, blocks):
  29             self.blocks = blocks
  30             self.requests = []
  31             self.num_prefetch_threads = 1
  32         def get(self, locator, num_retries=0, prefetch=False):
  33             self.requests.append(locator)
  34             return self.blocks.get(locator)
  35         def get_from_cache(self, locator):
  36             self.requests.append(locator)
  37             return self.blocks.get(locator)
  38         def put(self, data, num_retries=None, copies=None, classes=[]):
  39             pdh = tutil.str_keep_locator(data)
  40             self.blocks[pdh] = bytes(data)
  41             return pdh
  42         def block_prefetch(self, loc):
  43             self.requests.append(loc)
  44
  45     class MockApi(object):
  46         def __init__(self, b, r):
  47             self.body = b
  48             self.response = r
  49             self._schema = ArvadosFileWriterTestCase.MockApi.MockSchema()
  50             self._rootDesc = {}
  51         class MockSchema(object):
  52             def __init__(self):
  53                 self.schemas = {'Collection': {'properties': {'replication_desired': {'type':'integer'}}}}
  54         class MockCollections(object):
  55             def __init__(self, b, r):
  56                 self.body = b
  57                 self.response = r
  58             class Execute(object):
  59                 def __init__(self, r):
  60                     self.response = r
  61                 def execute(self, num_retries=None):
  62                     return self.response
  63             def create(self, ensure_unique_name=False, body=None):
  64                 if body != self.body:
  65                     raise Exception("Body %s does not match expectation %s" % (body, self.body))
  66                 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
  67             def update(self, uuid=None, body=None):
  68                 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
  69         def collections(self):
  70             return ArvadosFileWriterTestCase.MockApi.MockCollections(self.body, self.response)
  71
  72
  73     def test_truncate(self):
  74         keep = ArvadosFileWriterTestCase.MockKeep({
  75             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
  76         })
  77         api = ArvadosFileWriterTestCase.MockApi({
  78             "name": "test_truncate",
  79             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
  80             "replication_desired": None,
  81         }, {
  82             "uuid": "zzzzz-4zz18-mockcollection0",
  83             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
  84             "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52",
  85         })
  86         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
  87                         api_client=api, keep_client=keep) as c:
  88             writer = c.open("count.txt", "rb+")
  89             self.assertEqual(writer.size(), 10)
  90             self.assertEqual(b"0123456789", writer.read(12))
  91
  92             writer.truncate(8)
  93
  94             # Make sure reading off the end doesn't break
  95             self.assertEqual(b"", writer.read(12))
  96
  97             self.assertEqual(writer.size(), 8)
  98             writer.seek(0, os.SEEK_SET)
  99             self.assertEqual(b"01234567", writer.read(12))
 100
 101             self.assertIsNone(c.manifest_locator())
 102             self.assertTrue(c.modified())
 103             c.save_new("test_truncate")
 104             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 105             self.assertFalse(c.modified())
 106
 107
 108     def test_truncate2(self):
 109         keep = ArvadosFileWriterTestCase.MockKeep({
 110             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 111         })
 112         api = ArvadosFileWriterTestCase.MockApi({
 113             "name": "test_truncate2",
 114             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
 115             "replication_desired": None,
 116         }, {
 117             "uuid": "zzzzz-4zz18-mockcollection0",
 118             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
 119             "portable_data_hash": "272da898abdf86ddc71994835e3155f8+95",
 120         })
 121         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 122                         api_client=api, keep_client=keep) as c:
 123             writer = c.open("count.txt", "rb+")
 124             self.assertEqual(writer.size(), 10)
 125             self.assertEqual(b"0123456789", writer.read(12))
 126
 127             # extend file size
 128             writer.truncate(12)
 129
 130             self.assertEqual(writer.size(), 12)
 131             writer.seek(0, os.SEEK_SET)
 132             self.assertEqual(b"0123456789\x00\x00", writer.read(12))
 133
 134             self.assertIsNone(c.manifest_locator())
 135             self.assertTrue(c.modified())
 136             c.save_new("test_truncate2")
 137             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 138             self.assertFalse(c.modified())
 139
 140     def test_truncate3(self):
 141         keep = ArvadosFileWriterTestCase.MockKeep({
 142             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 143             "a925576942e94b2ef57a066101b48876+10": b"abcdefghij",
 144         })
 145         api = ArvadosFileWriterTestCase.MockApi({
 146             "name": "test_truncate",
 147             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
 148             "replication_desired": None,
 149         }, {
 150             "uuid": "zzzzz-4zz18-mockcollection0",
 151             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
 152             "portable_data_hash": "7fcd0eaac3aad4c31a6a0e756475da92+52",
 153         })
 154         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n',
 155                         api_client=api, keep_client=keep) as c:
 156             writer = c.open("count.txt", "rb+")
 157             self.assertEqual(writer.size(), 20)
 158             self.assertEqual(b"0123456789ab", writer.read(12))
 159             self.assertEqual(12, writer.tell())
 160
 161             writer.truncate(8)
 162
 163             # Make sure reading off the end doesn't break
 164             self.assertEqual(12, writer.tell())
 165             self.assertEqual(b"", writer.read(12))
 166
 167             self.assertEqual(writer.size(), 8)
 168             self.assertEqual(2, writer.seek(-10, os.SEEK_CUR))
 169             self.assertEqual(b"234567", writer.read(12))
 170
 171             self.assertIsNone(c.manifest_locator())
 172             self.assertTrue(c.modified())
 173             c.save_new("test_truncate")
 174             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 175             self.assertFalse(c.modified())
 176
 177     def test_write_to_end(self):
 178         keep = ArvadosFileWriterTestCase.MockKeep({
 179             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 180         })
 181         api = ArvadosFileWriterTestCase.MockApi({
 182             "name": "test_append",
 183             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
 184             "replication_desired": None,
 185         }, {
 186             "uuid": "zzzzz-4zz18-mockcollection0",
 187             "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
 188             "portable_data_hash": "c5c3af76565c8efb6a806546bcf073f3+88",
 189         })
 190         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 191                              api_client=api, keep_client=keep) as c:
 192             writer = c.open("count.txt", "rb+")
 193             self.assertEqual(writer.size(), 10)
 194
 195             self.assertEqual(5, writer.seek(5, os.SEEK_SET))
 196             self.assertEqual(b"56789", writer.read(8))
 197
 198             writer.seek(10, os.SEEK_SET)
 199             writer.write("foo")
 200             self.assertEqual(writer.size(), 13)
 201
 202             writer.seek(5, os.SEEK_SET)
 203             self.assertEqual(b"56789foo", writer.read(8))
 204
 205             self.assertIsNone(c.manifest_locator())
 206             self.assertTrue(c.modified())
 207             self.assertIsNone(keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
 208
 209             c.save_new("test_append")
 210             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 211             self.assertFalse(c.modified())
 212             self.assertEqual(b"foo", keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
 213
 214
 215     def test_append(self):
 216         keep = ArvadosFileWriterTestCase.MockKeep({
 217             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 218         })
 219         for (mode, convert) in (
 220                 ('a+', lambda data: data.decode(encoding='utf-8')),
 221                 ('at+', lambda data: data.decode(encoding='utf-8')),
 222                 ('ab+', lambda data: data)):
 223             c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
 224             writer = c.open("count.txt", mode)
 225             self.assertEqual(writer.read(20), convert(b"0123456789"))
 226
 227             writer.seek(0, os.SEEK_SET)
 228             writer.write(convert(b"hello"))
 229             self.assertEqual(writer.read(), convert(b""))
 230             if 'b' in mode:
 231                 writer.seek(-5, os.SEEK_CUR)
 232                 self.assertEqual(writer.read(3), convert(b"hel"))
 233                 self.assertEqual(writer.read(), convert(b"lo"))
 234             else:
 235                 with self.assertRaises(IOError):
 236                     writer.seek(-5, os.SEEK_CUR)
 237                 with self.assertRaises(IOError):
 238                     writer.seek(-3, os.SEEK_END)
 239             writer.seek(0, os.SEEK_SET)
 240             writer.read(7)
 241             self.assertEqual(7, writer.tell())
 242             self.assertEqual(7, writer.seek(7, os.SEEK_SET))
 243
 244             writer.seek(0, os.SEEK_SET)
 245             self.assertEqual(writer.read(), convert(b"0123456789hello"))
 246
 247             writer.seek(0)
 248             writer.write(convert(b"world"))
 249             self.assertEqual(writer.read(), convert(b""))
 250             writer.seek(0)
 251             self.assertEqual(writer.read(), convert(b"0123456789helloworld"))
 252
 253             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.portable_manifest_text())
 254
 255     def test_write_at_beginning(self):
 256         keep = ArvadosFileWriterTestCase.MockKeep({
 257             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 258         })
 259         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 260                              keep_client=keep) as c:
 261             writer = c.open("count.txt", "rb+")
 262             self.assertEqual(b"0123456789", writer.readfrom(0, 13))
 263             writer.seek(0, os.SEEK_SET)
 264             writer.write("foo")
 265             self.assertEqual(writer.size(), 10)
 266             self.assertEqual(b"foo3456789", writer.readfrom(0, 13))
 267             self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", c.portable_manifest_text())
 268
 269     def test_write_empty(self):
 270         keep = ArvadosFileWriterTestCase.MockKeep({})
 271         with Collection(keep_client=keep) as c:
 272             writer = c.open("count.txt", "wb")
 273             self.assertEqual(writer.size(), 0)
 274             self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:count.txt\n", c.portable_manifest_text())
 275
 276     def test_save_manifest_text(self):
 277         keep = ArvadosFileWriterTestCase.MockKeep({})
 278         with Collection(keep_client=keep) as c:
 279             writer = c.open("count.txt", "wb")
 280             writer.write(b"0123456789")
 281             self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
 282             self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 283
 284             self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.save_new(create_collection_record=False))
 285             self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 286
 287     def test_get_manifest_text_commits(self):
 288          keep = ArvadosFileWriterTestCase.MockKeep({})
 289          with Collection(keep_client=keep) as c:
 290              writer = c.open("count.txt", "wb")
 291              writer.write("0123456789")
 292              self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
 293              self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 294              self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.manifest_text())
 295              self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 296
 297
 298     def test_write_in_middle(self):
 299         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
 300         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 301                              keep_client=keep) as c:
 302             writer = c.open("count.txt", "rb+")
 303             self.assertEqual(b"0123456789", writer.readfrom(0, 13))
 304             writer.seek(3, os.SEEK_SET)
 305             writer.write("foo")
 306             self.assertEqual(writer.size(), 10)
 307             self.assertEqual(b"012foo6789", writer.readfrom(0, 13))
 308             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", c.portable_manifest_text())
 309
 310     def test_write_at_end(self):
 311         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
 312         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 313                              keep_client=keep) as c:
 314             writer = c.open("count.txt", "rb+")
 315             self.assertEqual(b"0123456789", writer.readfrom(0, 13))
 316             writer.seek(7, os.SEEK_SET)
 317             writer.write("foo")
 318             self.assertEqual(writer.size(), 10)
 319             self.assertEqual(b"0123456foo", writer.readfrom(0, 13))
 320             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", c.portable_manifest_text())
 321
 322     def test_write_across_segment_boundary(self):
 323         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
 324         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
 325                              keep_client=keep) as c:
 326             writer = c.open("count.txt", "rb+")
 327             self.assertEqual(b"012345678901234", writer.readfrom(0, 15))
 328             writer.seek(7, os.SEEK_SET)
 329             writer.write("foobar")
 330             self.assertEqual(writer.size(), 20)
 331             self.assertEqual(b"0123456foobar34", writer.readfrom(0, 15))
 332             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", c.portable_manifest_text())
 333
 334     def test_write_across_several_segments(self):
 335         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
 336         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
 337                              keep_client=keep) as c:
 338             writer = c.open("count.txt", "rb+")
 339             self.assertEqual(b"012301230123", writer.readfrom(0, 15))
 340             writer.seek(2, os.SEEK_SET)
 341             writer.write("abcdefg")
 342             self.assertEqual(writer.size(), 12)
 343             self.assertEqual(b"01abcdefg123", writer.readfrom(0, 15))
 344             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", c.portable_manifest_text())
 345
 346     def test_write_large(self):
 347         keep = ArvadosFileWriterTestCase.MockKeep({})
 348         api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
 349                                                  "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n",
 350                                                  "replication_desired":None},
 351                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 352                                                  "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n",
 353                                                  "portable_data_hash":"9132ca8e3f671c76103a38f5bc24328c+108"})
 354         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 355                              api_client=api, keep_client=keep) as c:
 356             writer = c.open("count.txt", "rb+")
 357             text = "0123456789" * 100
 358             for b in range(0, 100000):
 359                 writer.write(text)
 360             self.assertEqual(writer.size(), 100000000)
 361
 362             self.assertIsNone(c.manifest_locator())
 363             self.assertTrue(c.modified())
 364             c.save_new("test_write_large")
 365             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 366             self.assertFalse(c.modified())
 367
 368     def test_large_write(self):
 369         keep = ArvadosFileWriterTestCase.MockKeep({})
 370         api = ArvadosFileWriterTestCase.MockApi({}, {})
 371         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 372                              api_client=api, keep_client=keep) as c:
 373             writer = c.open("count.txt", "rb+")
 374             self.assertEqual(writer.size(), 0)
 375
 376             text = "0123456789"
 377             writer.write(text)
 378             text = "0123456789" * 9999999
 379             writer.write(text)
 380             self.assertEqual(writer.size(), 100000000)
 381
 382             self.assertEqual(c.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 48dd23ea1645fd47d789804d71b5bb8e+67108864 77c57dc6ac5a10bb2205caaa73187994+32891126 0:100000000:count.txt\n")
 383
 384     def test_sparse_write(self):
 385         keep = ArvadosFileWriterTestCase.MockKeep({})
 386         api = ArvadosFileWriterTestCase.MockApi({}, {})
 387         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 388                              api_client=api, keep_client=keep) as c:
 389             writer = c.open("count.txt", "rb+")
 390             self.assertEqual(writer.size(), 0)
 391
 392             text = b"0123456789"
 393             writer.seek(2)
 394             writer.write(text)
 395             self.assertEqual(writer.size(), 12)
 396             writer.seek(0, os.SEEK_SET)
 397             self.assertEqual(writer.read(), b"\x00\x00"+text)
 398
 399             self.assertEqual(c.manifest_text(), ". 7f614da9329cd3aebf59b91aadc30bf0+67108864 781e5e245d69b566979b86e28d23f2c7+10 0:2:count.txt 67108864:10:count.txt\n")
 400
 401     def test_sparse_write2(self):
 402         keep = ArvadosFileWriterTestCase.MockKeep({})
 403         api = ArvadosFileWriterTestCase.MockApi({}, {})
 404         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 405                              api_client=api, keep_client=keep) as c:
 406             writer = c.open("count.txt", "rb+")
 407             self.assertEqual(writer.size(), 0)
 408
 409             text = "0123456789"
 410             writer.seek((arvados.config.KEEP_BLOCK_SIZE*2) + 2)
 411             writer.write(text)
 412             self.assertEqual(writer.size(), (arvados.config.KEEP_BLOCK_SIZE*2) + 12)
 413             writer.seek(0, os.SEEK_SET)
 414
 415             self.assertEqual(c.manifest_text(), ". 7f614da9329cd3aebf59b91aadc30bf0+67108864 781e5e245d69b566979b86e28d23f2c7+10 0:67108864:count.txt 0:67108864:count.txt 0:2:count.txt 67108864:10:count.txt\n")
 416
 417     def test_sparse_write3(self):
 418         keep = ArvadosFileWriterTestCase.MockKeep({})
 419         api = ArvadosFileWriterTestCase.MockApi({}, {})
 420         for r in [[0, 1, 2, 3, 4], [4, 3, 2, 1, 0], [3, 2, 0, 4, 1]]:
 421             with Collection(api_client=api, keep_client=keep) as c:
 422                 writer = c.open("count.txt", "rb+")
 423                 self.assertEqual(writer.size(), 0)
 424
 425                 for i in r:
 426                     w = ("%s" % i) * 10
 427                     writer.seek(i*10)
 428                     writer.write(w.encode())
 429                 writer.seek(0)
 430                 self.assertEqual(writer.read(), b"00000000001111111111222222222233333333334444444444")
 431
 432     def test_sparse_write4(self):
 433         keep = ArvadosFileWriterTestCase.MockKeep({})
 434         api = ArvadosFileWriterTestCase.MockApi({}, {})
 435         for r in [[0, 1, 2, 4], [4, 2, 1, 0], [2, 0, 4, 1]]:
 436             with Collection(api_client=api, keep_client=keep) as c:
 437                 writer = c.open("count.txt", "rb+")
 438                 self.assertEqual(writer.size(), 0)
 439
 440                 for i in r:
 441                     w = ("%s" % i) * 10
 442                     writer.seek(i*10)
 443                     writer.write(w.encode())
 444                 writer.seek(0)
 445                 self.assertEqual(writer.read(), b"000000000011111111112222222222\x00\x00\x00\x00\x00\x00\x00\x00\x00\x004444444444")
 446
 447     def test_rewrite_on_empty_file(self):
 448         keep = ArvadosFileWriterTestCase.MockKeep({})
 449         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 450                              keep_client=keep) as c:
 451             writer = c.open("count.txt", "rb+")
 452             for b in range(0, 10):
 453                 writer.seek(0, os.SEEK_SET)
 454                 writer.write("0123456789")
 455
 456             self.assertEqual(writer.size(), 10)
 457             self.assertEqual(b"0123456789", writer.readfrom(0, 20))
 458             self.assertEqual(". 7a08b07e84641703e5f2c836aa59a170+100 90:10:count.txt\n", c.portable_manifest_text())
 459             writer.flush()
 460             self.assertEqual(writer.size(), 10)
 461             self.assertEqual(b"0123456789", writer.readfrom(0, 20))
 462             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.portable_manifest_text())
 463
 464     def test_rewrite_append_existing_file(self):
 465         keep = ArvadosFileWriterTestCase.MockKeep({
 466             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 467         })
 468         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
 469                              keep_client=keep) as c:
 470             writer = c.open("count.txt", "rb+")
 471             for b in range(0, 10):
 472                 writer.seek(10, os.SEEK_SET)
 473                 writer.write("abcdefghij")
 474
 475             self.assertEqual(writer.size(), 20)
 476             self.assertEqual(b"0123456789abcdefghij", writer.readfrom(0, 20))
 477             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:10:count.txt 100:10:count.txt\n", c.portable_manifest_text())
 478
 479             writer.arvadosfile.flush()
 480             self.assertEqual(writer.size(), 20)
 481             self.assertEqual(b"0123456789abcdefghij", writer.readfrom(0, 20))
 482             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.portable_manifest_text())
 483
 484     def test_rewrite_over_existing_file(self):
 485         keep = ArvadosFileWriterTestCase.MockKeep({
 486             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 487         })
 488         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
 489                              keep_client=keep) as c:
 490             writer = c.open("count.txt", "rb+")
 491             for b in range(0, 10):
 492                 writer.seek(5, os.SEEK_SET)
 493                 writer.write("abcdefghij")
 494
 495             self.assertEqual(writer.size(), 15)
 496             self.assertEqual(b"01234abcdefghij", writer.readfrom(0, 20))
 497             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:5:count.txt 100:10:count.txt\n", c.portable_manifest_text())
 498
 499             writer.arvadosfile.flush()
 500
 501             self.assertEqual(writer.size(), 15)
 502             self.assertEqual(b"01234abcdefghij", writer.readfrom(0, 20))
 503             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.portable_manifest_text())
 504
 505     def test_write_large_rewrite(self):
 506         keep = ArvadosFileWriterTestCase.MockKeep({})
 507         api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
 508                                                  "manifest_text": ". 3dc0d4bc21f48060bedcb2c91af4f906+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 0:3:count.txt 32892006:67107997:count.txt 3:32892000:count.txt\n",
 509                                                  "replication_desired":None},
 510                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 511                                                  "manifest_text": ". 3dc0d4bc21f48060bedcb2c91af4f906+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 0:3:count.txt 32892006:67107997:count.txt 3:32892000:count.txt\n",
 512                                                  "portable_data_hash":"217665c6b713e1b78dfba7ebd42344db+156"})
 513         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 514                              api_client=api, keep_client=keep) as c:
 515             writer = c.open("count.txt", "rb+")
 516             text = b''.join([b"0123456789" for a in range(0, 100)])
 517             for b in range(0, 100000):
 518                 writer.write(text)
 519             writer.seek(0, os.SEEK_SET)
 520             writer.write("foo")
 521             self.assertEqual(writer.size(), 100000000)
 522
 523             self.assertIsNone(c.manifest_locator())
 524             self.assertTrue(c.modified())
 525             c.save_new("test_write_large")
 526             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 527             self.assertFalse(c.modified())
 528
 529     def test_create(self):
 530         keep = ArvadosFileWriterTestCase.MockKeep({})
 531         api = ArvadosFileWriterTestCase.MockApi({
 532             "name":"test_create",
 533             "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 534             "replication_desired":None,
 535         }, {
 536             "uuid":"zzzzz-4zz18-mockcollection0",
 537             "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 538             "portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51",
 539         })
 540         with Collection(api_client=api, keep_client=keep) as c:
 541             writer = c.open("count.txt", "wb+")
 542             self.assertEqual(writer.size(), 0)
 543             writer.write("01234567")
 544             self.assertEqual(writer.size(), 8)
 545
 546             self.assertIsNone(c.manifest_locator())
 547             self.assertTrue(c.modified())
 548             self.assertIsNone(keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 549             c.save_new("test_create")
 550             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 551             self.assertFalse(c.modified())
 552             self.assertEqual(b"01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 553
 554
 555     def test_create_subdir(self):
 556         keep = ArvadosFileWriterTestCase.MockKeep({})
 557         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
 558                                                  "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 559                                                  "replication_desired":None},
 560                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 561                                                  "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 562                                                  "portable_data_hash":"1b02aaa62528d28a5be41651cbb9d7c7+59"})
 563         with Collection(api_client=api, keep_client=keep) as c:
 564             self.assertIsNone(c.api_response())
 565             writer = c.open("foo/bar/count.txt", "wb+")
 566             writer.write("01234567")
 567             self.assertFalse(c.committed())
 568             c.save_new("test_create")
 569             self.assertTrue(c.committed())
 570             self.assertEqual(c.api_response(), api.response)
 571
 572     def test_overwrite(self):
 573         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 574         api = ArvadosFileWriterTestCase.MockApi({"name":"test_overwrite",
 575                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 576                                                  "replication_desired":None},
 577                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 578                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 579                                                  "portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51"})
 580         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 581                              api_client=api, keep_client=keep) as c:
 582             writer = c.open("count.txt", "wb+")
 583             self.assertEqual(writer.size(), 0)
 584             writer.write("01234567")
 585             self.assertEqual(writer.size(), 8)
 586
 587             self.assertIsNone(c.manifest_locator())
 588             self.assertTrue(c.modified())
 589             c.save_new("test_overwrite")
 590             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 591             self.assertFalse(c.modified())
 592
 593     def test_file_not_found(self):
 594         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
 595             with self.assertRaises(IOError):
 596                 writer = c.open("nocount.txt", "rb")
 597
 598     def test_cannot_open_directory(self):
 599         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
 600             with self.assertRaises(IOError):
 601                 writer = c.open(".", "rb")
 602
 603     def test_create_multiple(self):
 604         keep = ArvadosFileWriterTestCase.MockKeep({})
 605         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create_multiple",
 606                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n",
 607                                                  "replication_desired":None},
 608                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 609                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n",
 610                                                  "portable_data_hash":"71e7bb6c00d31fc2b4364199fd97be08+102"})
 611         with Collection(api_client=api, keep_client=keep) as c:
 612             w1 = c.open("count1.txt", "wb")
 613             w2 = c.open("count2.txt", "wb")
 614             w1.write("01234567")
 615             w2.write("abcdefgh")
 616             self.assertEqual(w1.size(), 8)
 617             self.assertEqual(w2.size(), 8)
 618
 619             self.assertIsNone(c.manifest_locator())
 620             self.assertTrue(c.modified())
 621             self.assertIsNone(keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 622             c.save_new("test_create_multiple")
 623             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 624             self.assertFalse(c.modified())
 625             self.assertEqual(b"01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 626
 627
 628 class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
 629     class MockParent(object):
 630         class MockBlockMgr(object):
 631             def __init__(self, blocks, nocache):
 632                 self.blocks = blocks
 633                 self.nocache = nocache
 634                 self._keep = ArvadosFileWriterTestCase.MockKeep({})
 635                 self.prefetch_lookahead = 0
 636
 637             def block_prefetch(self, loc):
 638                 pass
 639
 640             def get_block_contents(self, loc, num_retries=0, cache_only=False):
 641                 if self.nocache and cache_only:
 642                     return None
 643                 return self.blocks[loc]
 644
 645         def __init__(self, blocks, nocache):
 646             self.blocks = blocks
 647             self.nocache = nocache
 648             self.lock = arvados.arvfile.NoopLock()
 649
 650         def root_collection(self):
 651             return self
 652
 653         def _my_block_manager(self):
 654             return ArvadosFileReaderTestCase.MockParent.MockBlockMgr(self.blocks, self.nocache)
 655
 656
 657     def make_count_reader(self, nocache=False):
 658         stream = []
 659         n = 0
 660         blocks = {}
 661         for d in [b'01234', b'34567', b'67890']:
 662             loc = tutil.str_keep_locator(d)
 663             blocks[loc] = d
 664             stream.append(Range(loc, n, len(d)))
 665             n += len(d)
 666         af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), "count.txt", stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
 667         return ArvadosFileReader(af, mode="rb")
 668
 669     def test_read_block_crossing_behavior(self):
 670         # read() needs to return all the data requested if possible, even if it
 671         # crosses uncached blocks: https://arvados.org/issues/5856
 672         sfile = self.make_count_reader(nocache=True)
 673         self.assertEqual(b'12345678', sfile.read(8))
 674
 675     def test_successive_reads(self):
 676         # Override StreamFileReaderTestCase.test_successive_reads
 677         sfile = self.make_count_reader(nocache=True)
 678         self.assertEqual(b'1234', sfile.read(4))
 679         self.assertEqual(b'5678', sfile.read(4))
 680         self.assertEqual(b'9', sfile.read(4))
 681         self.assertEqual(b'', sfile.read(4))
 682
 683     def test_tell_after_block_read(self):
 684         # Override StreamFileReaderTestCase.test_tell_after_block_read
 685         sfile = self.make_count_reader(nocache=True)
 686         self.assertEqual(b'12345678', sfile.read(8))
 687         self.assertEqual(8, sfile.tell())
 688
 689     def test_prefetch(self):
 690         keep = ArvadosFileWriterTestCase.MockKeep({
 691             "2e9ec317e197819358fbc43afca7d837+8": b"01234567",
 692             "e8dc4081b13434b45189a720b77b6818+8": b"abcdefgh",
 693         })
 694         with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
 695             r = c.open("count.txt", "rb")
 696             self.assertEqual(b"0123", r.read(4))
 697         self.assertEqual(["2e9ec317e197819358fbc43afca7d837+8",
 698                           "e8dc4081b13434b45189a720b77b6818+8"], keep.requests)
 699
 700     def test_prefetch_disabled(self):
 701         keep = ArvadosFileWriterTestCase.MockKeep({
 702             "2e9ec317e197819358fbc43afca7d837+8": b"01234567",
 703             "e8dc4081b13434b45189a720b77b6818+8": b"abcdefgh",
 704         })
 705         keep.num_prefetch_threads = 0
 706         with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
 707             r = c.open("count.txt", "rb")
 708             self.assertEqual(b"0123", r.read(4))
 709
 710         self.assertEqual(["2e9ec317e197819358fbc43afca7d837+8"], keep.requests)
 711
 712     def test_prefetch_first_read_only(self):
 713         # test behavior that prefetch only happens every 128 reads
 714         # check that it doesn't make a prefetch request on the second read
 715         keep = ArvadosFileWriterTestCase.MockKeep({
 716             "2e9ec317e197819358fbc43afca7d837+8": b"01234567",
 717             "e8dc4081b13434b45189a720b77b6818+8": b"abcdefgh",
 718         })
 719         with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
 720             r = c.open("count.txt", "rb")
 721             self.assertEqual(b"0123", r.read(4))
 722             self.assertEqual(b"45", r.read(2))
 723         self.assertEqual(["2e9ec317e197819358fbc43afca7d837+8",
 724                           "e8dc4081b13434b45189a720b77b6818+8",
 725                           "2e9ec317e197819358fbc43afca7d837+8"], keep.requests)
 726         self.assertEqual(3, len(keep.requests))
 727
 728     def test_prefetch_again(self):
 729         # test behavior that prefetch only happens every 128 reads
 730         # check that it does make another prefetch request after 128 reads
 731         keep = ArvadosFileWriterTestCase.MockKeep({
 732             "2e9ec317e197819358fbc43afca7d837+8": b"01234567",
 733             "e8dc4081b13434b45189a720b77b6818+8": b"abcdefgh",
 734         })
 735         with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
 736             r = c.open("count.txt", "rb")
 737             for i in range(0, 129):
 738                 r.seek(0)
 739                 self.assertEqual(b"0123", r.read(4))
 740         self.assertEqual(["2e9ec317e197819358fbc43afca7d837+8",
 741                           "e8dc4081b13434b45189a720b77b6818+8",
 742                           "2e9ec317e197819358fbc43afca7d837+8",
 743                           "2e9ec317e197819358fbc43afca7d837+8"], keep.requests[0:4])
 744         self.assertEqual(["2e9ec317e197819358fbc43afca7d837+8",
 745                           "2e9ec317e197819358fbc43afca7d837+8",
 746                           "2e9ec317e197819358fbc43afca7d837+8",
 747                           "e8dc4081b13434b45189a720b77b6818+8"], keep.requests[127:131])
 748         # gets the 1st block 129 times from keep (cache),
 749         # and the 2nd block twice to get 131 requests
 750         self.assertEqual(131, len(keep.requests))
 751
 752     def test__eq__from_manifest(self):
 753         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 754             with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
 755                 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
 756                 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
 757
 758     def test__eq__from_writes(self):
 759         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 760             with Collection() as c2:
 761                 f = c2.open("count1.txt", "wb")
 762                 f.write("0123456789")
 763
 764                 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
 765                 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
 766
 767     def test__ne__(self):
 768         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 769             with Collection() as c2:
 770                 f = c2.open("count1.txt", "wb")
 771                 f.write("1234567890")
 772
 773                 self.assertTrue(c1["count1.txt"] != c2["count1.txt"])
 774                 self.assertFalse(c1["count1.txt"] == c2["count1.txt"])
 775
 776
 777 class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
 778     def reader_for(self, coll_name, **kwargs):
 779         stream = []
 780         segments = []
 781         n = 0
 782         for d in self.manifest_for(coll_name).split():
 783             try:
 784                 k = KeepLocator(d)
 785                 segments.append(Range(n, n, k.size))
 786                 stream.append(Range(d, n, k.size))
 787                 n += k.size
 788             except ValueError:
 789                 pass
 790
 791         blockmanager = arvados.arvfile._BlockManager(self.keep_client())
 792         blockmanager.prefetch_enabled = False
 793         col = Collection(keep_client=self.keep_client(), block_manager=blockmanager)
 794         af = ArvadosFile(col, "test",
 795                          stream=stream,
 796                          segments=segments)
 797         kwargs.setdefault('mode', 'rb')
 798         return ArvadosFileReader(af, **kwargs)
 799
 800     def read_for_test(self, reader, byte_count, **kwargs):
 801         return reader.read(byte_count, **kwargs)
 802
 803
 804 class ArvadosFileReadFromTestCase(ArvadosFileReadTestCase):
 805     def read_for_test(self, reader, byte_count, **kwargs):
 806         return reader.readfrom(0, byte_count, **kwargs)
 807
 808
 809 class ArvadosFileReadAllTestCase(ArvadosFileReadTestCase):
 810     def read_for_test(self, reader, byte_count, **kwargs):
 811         return b''.join(reader.readall(**kwargs))
 812
 813
 814 class ArvadosFileReadAllDecompressedTestCase(ArvadosFileReadTestCase):
 815     def read_for_test(self, reader, byte_count, **kwargs):
 816         return b''.join(reader.readall_decompressed(**kwargs))
 817
 818
 819 class ArvadosFileReadlinesTestCase(ArvadosFileReadTestCase):
 820     def read_for_test(self, reader, byte_count, **kwargs):
 821         return ''.join(reader.readlines(**kwargs)).encode()
 822
 823
 824 class ArvadosFileTestCase(unittest.TestCase):
 825     def datetime_to_hex(self, dt):
 826         return hex(int(time.mktime(dt.timetuple())))[2:]
 827
 828     def test_permission_expired(self):
 829         base_manifest = ". 781e5e245d69b566979b86e28d23f2c7+10+A715fd31f8111894f717eb1003c1b0216799dd9ec@{} 0:10:count.txt\n"
 830         now = datetime.datetime.now()
 831         a_week_ago = now - datetime.timedelta(days=7)
 832         a_month_ago = now - datetime.timedelta(days=30)
 833         a_week_from_now = now + datetime.timedelta(days=7)
 834         with Collection(base_manifest.format(self.datetime_to_hex(a_week_from_now))) as c:
 835             self.assertFalse(c.find('count.txt').permission_expired())
 836         with Collection(base_manifest.format(self.datetime_to_hex(a_week_ago))) as c:
 837             f = c.find('count.txt')
 838             self.assertTrue(f.permission_expired())
 839             self.assertTrue(f.permission_expired(a_week_from_now))
 840             self.assertFalse(f.permission_expired(a_month_ago))
 841
 842
 843 class BlockManagerTest(unittest.TestCase):
 844     def test_bufferblock_append(self):
 845         keep = ArvadosFileWriterTestCase.MockKeep({})
 846         with arvados.arvfile._BlockManager(keep) as blockmanager:
 847             bufferblock = blockmanager.alloc_bufferblock()
 848             bufferblock.append("foo")
 849
 850             self.assertEqual(bufferblock.size(), 3)
 851             self.assertEqual(bufferblock.buffer_view[0:3], b"foo")
 852             self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 853
 854             bufferblock.append("bar")
 855
 856             self.assertEqual(bufferblock.size(), 6)
 857             self.assertEqual(bufferblock.buffer_view[0:6], b"foobar")
 858             self.assertEqual(bufferblock.locator(), "3858f62230ac3c915f300c664312c63f+6")
 859
 860             bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
 861             with self.assertRaises(arvados.errors.AssertionError):
 862                 bufferblock.append("bar")
 863
 864     def test_bufferblock_dup(self):
 865         keep = ArvadosFileWriterTestCase.MockKeep({})
 866         with arvados.arvfile._BlockManager(keep) as blockmanager:
 867             bufferblock = blockmanager.alloc_bufferblock()
 868             bufferblock.append("foo")
 869
 870             self.assertEqual(bufferblock.size(), 3)
 871             self.assertEqual(bufferblock.buffer_view[0:3], b"foo")
 872             self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 873             bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
 874
 875             bufferblock2 = blockmanager.dup_block(bufferblock, None)
 876             self.assertNotEqual(bufferblock.blockid, bufferblock2.blockid)
 877
 878             bufferblock2.append("bar")
 879
 880             self.assertEqual(bufferblock2.size(), 6)
 881             self.assertEqual(bufferblock2.buffer_view[0:6], b"foobar")
 882             self.assertEqual(bufferblock2.locator(), "3858f62230ac3c915f300c664312c63f+6")
 883
 884             self.assertEqual(bufferblock.size(), 3)
 885             self.assertEqual(bufferblock.buffer_view[0:3], b"foo")
 886             self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 887
 888     def test_bufferblock_get(self):
 889         keep = ArvadosFileWriterTestCase.MockKeep({
 890             "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 891         })
 892         with arvados.arvfile._BlockManager(keep) as blockmanager:
 893             bufferblock = blockmanager.alloc_bufferblock()
 894             bufferblock.append("foo")
 895
 896             self.assertEqual(blockmanager.get_block_contents("781e5e245d69b566979b86e28d23f2c7+10", 1), b"0123456789")
 897             self.assertEqual(blockmanager.get_block_contents(bufferblock.blockid, 1), b"foo")
 898
 899     def test_bufferblock_commit(self):
 900         mockkeep = mock.MagicMock()
 901         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
 902             bufferblock = blockmanager.alloc_bufferblock()
 903             bufferblock.owner = mock.MagicMock(spec=arvados.arvfile.ArvadosFile)
 904             def flush(sync=None):
 905                 blockmanager.commit_bufferblock(bufferblock, sync)
 906             bufferblock.owner.flush.side_effect = flush
 907             bufferblock.append("foo")
 908             blockmanager.commit_all()
 909             self.assertTrue(bufferblock.owner.flush.called)
 910             self.assertTrue(mockkeep.put.called)
 911             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
 912             self.assertIsNone(bufferblock.buffer_view)
 913
 914     def test_bufferblock_commit_pending(self):
 915         # Test for bug #7225
 916         mockkeep = mock.MagicMock()
 917         mockkeep.put.side_effect = lambda *args, **kwargs: time.sleep(1)
 918         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
 919             bufferblock = blockmanager.alloc_bufferblock()
 920             bufferblock.append("foo")
 921
 922             blockmanager.commit_bufferblock(bufferblock, False)
 923             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.PENDING)
 924
 925             blockmanager.commit_bufferblock(bufferblock, True)
 926             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
 927
 928     def test_bufferblock_commit_with_error(self):
 929         mockkeep = mock.MagicMock()
 930         mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail")
 931         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
 932             bufferblock = blockmanager.alloc_bufferblock()
 933             bufferblock.owner = mock.MagicMock(spec=arvados.arvfile.ArvadosFile)
 934             def flush(sync=None):
 935                 blockmanager.commit_bufferblock(bufferblock, sync)
 936             bufferblock.owner.flush.side_effect = flush
 937             bufferblock.append("foo")
 938             with self.assertRaises(arvados.errors.KeepWriteError) as err:
 939                 blockmanager.commit_all()
 940             self.assertTrue(bufferblock.owner.flush.called)
 941             self.assertEqual(str(err.exception), "Error writing some blocks: block acbd18db4cc2f85cedef654fccc4a4d8+3 raised KeepWriteError (fail)")
 942             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.ERROR)