sdk/python/tests/test_arvfile.py

   1 #!/usr/bin/env python
   2
   3 import bz2
   4 import gzip
   5 import io
   6 import mock
   7 import os
   8 import unittest
   9 import time
  10
  11 import arvados
  12 from arvados._ranges import Range
  13 from arvados.keep import KeepLocator
  14 from arvados.collection import Collection, CollectionReader
  15 from arvados.arvfile import ArvadosFile, ArvadosFileReader
  16
  17 import arvados_testutil as tutil
  18 from test_stream import StreamFileReaderTestCase, StreamRetryTestMixin
  19
  20 class ArvadosFileWriterTestCase(unittest.TestCase):
  21     class MockKeep(object):
  22         def __init__(self, blocks):
  23             self.blocks = blocks
  24             self.requests = []
  25         def get(self, locator, num_retries=0):
  26             self.requests.append(locator)
  27             return self.blocks.get(locator)
  28         def get_from_cache(self, locator):
  29             self.requests.append(locator)
  30             return self.blocks.get(locator)
  31         def put(self, data, num_retries=None):
  32             pdh = tutil.str_keep_locator(data)
  33             self.blocks[pdh] = str(data)
  34             return pdh
  35
  36     class MockApi(object):
  37         def __init__(self, b, r):
  38             self.body = b
  39             self.response = r
  40             self._schema = ArvadosFileWriterTestCase.MockApi.MockSchema()
  41         class MockSchema(object):
  42             def __init__(self):
  43                 self.schemas = {'Collection': {'properties': {'replication_desired': 2}}}
  44         class MockCollections(object):
  45             def __init__(self, b, r):
  46                 self.body = b
  47                 self.response = r
  48             class Execute(object):
  49                 def __init__(self, r):
  50                     self.response = r
  51                 def execute(self, num_retries=None):
  52                     return self.response
  53             def create(self, ensure_unique_name=False, body=None):
  54                 if body != self.body:
  55                     raise Exception("Body %s does not match expectation %s" % (body, self.body))
  56                 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
  57             def update(self, uuid=None, body=None):
  58                 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
  59         def collections(self):
  60             return ArvadosFileWriterTestCase.MockApi.MockCollections(self.body, self.response)
  61
  62
  63     def test_truncate(self):
  64         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
  65         api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate",
  66                                                  "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
  67                                                  "replication_desired":None},
  68                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
  69                                                  "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n"})
  70         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
  71                              api_client=api, keep_client=keep) as c:
  72             writer = c.open("count.txt", "r+")
  73             self.assertEqual(writer.size(), 10)
  74             self.assertEqual("0123456789", writer.read(12))
  75
  76             writer.truncate(8)
  77
  78             # Make sure reading off the end doesn't break
  79             self.assertEqual("", writer.read(12))
  80
  81             self.assertEqual(writer.size(), 8)
  82             writer.seek(0, os.SEEK_SET)
  83             self.assertEqual("01234567", writer.read(12))
  84
  85             self.assertIsNone(c.manifest_locator())
  86             self.assertTrue(c.modified())
  87             c.save_new("test_truncate")
  88             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
  89             self.assertFalse(c.modified())
  90
  91     def test_write_to_end(self):
  92         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
  93         api = ArvadosFileWriterTestCase.MockApi({"name":"test_append",
  94                                                  "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
  95                                                  "replication_desired":None},
  96                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
  97                                                  "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n"})
  98         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
  99                              api_client=api, keep_client=keep) as c:
 100             writer = c.open("count.txt", "r+")
 101             self.assertEqual(writer.size(), 10)
 102
 103             writer.seek(5, os.SEEK_SET)
 104             self.assertEqual("56789", writer.read(8))
 105
 106             writer.seek(10, os.SEEK_SET)
 107             writer.write("foo")
 108             self.assertEqual(writer.size(), 13)
 109
 110             writer.seek(5, os.SEEK_SET)
 111             self.assertEqual("56789foo", writer.read(8))
 112
 113             self.assertIsNone(c.manifest_locator())
 114             self.assertTrue(c.modified())
 115             self.assertIsNone(keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
 116
 117             c.save_new("test_append")
 118             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 119             self.assertFalse(c.modified())
 120             self.assertEqual("foo", keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
 121
 122
 123     def test_append(self):
 124         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 125         c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
 126         writer = c.open("count.txt", "a+")
 127         self.assertEqual(writer.read(20), "0123456789")
 128         writer.seek(0, os.SEEK_SET)
 129
 130         writer.write("hello")
 131         self.assertEqual(writer.read(20), "0123456789hello")
 132         writer.seek(0, os.SEEK_SET)
 133
 134         writer.write("world")
 135         self.assertEqual(writer.read(20), "0123456789helloworld")
 136
 137         self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.portable_manifest_text())
 138
 139     def test_write_at_beginning(self):
 140         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 141         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 142                              keep_client=keep) as c:
 143             writer = c.open("count.txt", "r+")
 144             self.assertEqual("0123456789", writer.readfrom(0, 13))
 145             writer.seek(0, os.SEEK_SET)
 146             writer.write("foo")
 147             self.assertEqual(writer.size(), 10)
 148             self.assertEqual("foo3456789", writer.readfrom(0, 13))
 149             self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", c.portable_manifest_text())
 150
 151     def test_write_empty(self):
 152         keep = ArvadosFileWriterTestCase.MockKeep({})
 153         with Collection(keep_client=keep) as c:
 154             writer = c.open("count.txt", "w")
 155             self.assertEqual(writer.size(), 0)
 156             self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:count.txt\n", c.portable_manifest_text())
 157
 158     def test_save_manifest_text(self):
 159         keep = ArvadosFileWriterTestCase.MockKeep({})
 160         with Collection(keep_client=keep) as c:
 161             writer = c.open("count.txt", "w")
 162             writer.write("0123456789")
 163             self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
 164             self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 165
 166             self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.save_new(create_collection_record=False))
 167             self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 168
 169     def test_get_manifest_text_commits(self):
 170          keep = ArvadosFileWriterTestCase.MockKeep({})
 171          with Collection(keep_client=keep) as c:
 172              writer = c.open("count.txt", "w")
 173              writer.write("0123456789")
 174              self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
 175              self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 176              self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.manifest_text())
 177              self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 178
 179
 180     def test_write_in_middle(self):
 181         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 182         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 183                              keep_client=keep) as c:
 184             writer = c.open("count.txt", "r+")
 185             self.assertEqual("0123456789", writer.readfrom(0, 13))
 186             writer.seek(3, os.SEEK_SET)
 187             writer.write("foo")
 188             self.assertEqual(writer.size(), 10)
 189             self.assertEqual("012foo6789", writer.readfrom(0, 13))
 190             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", c.portable_manifest_text())
 191
 192     def test_write_at_end(self):
 193         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 194         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 195                              keep_client=keep) as c:
 196             writer = c.open("count.txt", "r+")
 197             self.assertEqual("0123456789", writer.readfrom(0, 13))
 198             writer.seek(7, os.SEEK_SET)
 199             writer.write("foo")
 200             self.assertEqual(writer.size(), 10)
 201             self.assertEqual("0123456foo", writer.readfrom(0, 13))
 202             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", c.portable_manifest_text())
 203
 204     def test_write_across_segment_boundary(self):
 205         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 206         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
 207                              keep_client=keep) as c:
 208             writer = c.open("count.txt", "r+")
 209             self.assertEqual("012345678901234", writer.readfrom(0, 15))
 210             writer.seek(7, os.SEEK_SET)
 211             writer.write("foobar")
 212             self.assertEqual(writer.size(), 20)
 213             self.assertEqual("0123456foobar34", writer.readfrom(0, 15))
 214             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", c.portable_manifest_text())
 215
 216     def test_write_across_several_segments(self):
 217         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 218         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
 219                              keep_client=keep) as c:
 220             writer = c.open("count.txt", "r+")
 221             self.assertEqual("012301230123", writer.readfrom(0, 15))
 222             writer.seek(2, os.SEEK_SET)
 223             writer.write("abcdefg")
 224             self.assertEqual(writer.size(), 12)
 225             self.assertEqual("01abcdefg123", writer.readfrom(0, 15))
 226             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", c.portable_manifest_text())
 227
 228     def test_write_large(self):
 229         keep = ArvadosFileWriterTestCase.MockKeep({})
 230         api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
 231                                                  "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n",
 232                                                  "replication_desired":None},
 233                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 234                                                  "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n"})
 235         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 236                              api_client=api, keep_client=keep) as c:
 237             writer = c.open("count.txt", "r+")
 238             text = "0123456789" * 100
 239             for b in xrange(0, 100000):
 240                 writer.write(text)
 241             self.assertEqual(writer.size(), 100000000)
 242
 243             self.assertIsNone(c.manifest_locator())
 244             self.assertTrue(c.modified())
 245             c.save_new("test_write_large")
 246             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 247             self.assertFalse(c.modified())
 248
 249
 250     def test_large_write(self):
 251         keep = ArvadosFileWriterTestCase.MockKeep({})
 252         api = ArvadosFileWriterTestCase.MockApi({}, {})
 253         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 254                              api_client=api, keep_client=keep) as c:
 255             writer = c.open("count.txt", "r+")
 256             self.assertEqual(writer.size(), 0)
 257
 258             text = "0123456789"
 259             writer.write(text)
 260             text = "0123456789" * 9999999
 261             writer.write(text)
 262             self.assertEqual(writer.size(), 100000000)
 263
 264             self.assertEqual(c.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 48dd23ea1645fd47d789804d71b5bb8e+67108864 77c57dc6ac5a10bb2205caaa73187994+32891126 0:100000000:count.txt\n")
 265
 266     def test_rewrite_on_empty_file(self):
 267         keep = ArvadosFileWriterTestCase.MockKeep({})
 268         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 269                              keep_client=keep) as c:
 270             writer = c.open("count.txt", "r+")
 271             for b in xrange(0, 10):
 272                 writer.seek(0, os.SEEK_SET)
 273                 writer.write("0123456789")
 274
 275             self.assertEqual(writer.size(), 10)
 276             self.assertEqual("0123456789", writer.readfrom(0, 20))
 277             self.assertEqual(". 7a08b07e84641703e5f2c836aa59a170+100 90:10:count.txt\n", c.portable_manifest_text())
 278             writer.flush()
 279             self.assertEqual(writer.size(), 10)
 280             self.assertEqual("0123456789", writer.readfrom(0, 20))
 281             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.portable_manifest_text())
 282
 283     def test_rewrite_append_existing_file(self):
 284         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 285         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
 286                              keep_client=keep) as c:
 287             writer = c.open("count.txt", "r+")
 288             for b in xrange(0, 10):
 289                 writer.seek(10, os.SEEK_SET)
 290                 writer.write("abcdefghij")
 291
 292             self.assertEqual(writer.size(), 20)
 293             self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
 294             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:10:count.txt 100:10:count.txt\n", c.portable_manifest_text())
 295
 296             writer.arvadosfile.flush()
 297             self.assertEqual(writer.size(), 20)
 298             self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
 299             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.portable_manifest_text())
 300
 301     def test_rewrite_over_existing_file(self):
 302         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 303         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
 304                              keep_client=keep) as c:
 305             writer = c.open("count.txt", "r+")
 306             for b in xrange(0, 10):
 307                 writer.seek(5, os.SEEK_SET)
 308                 writer.write("abcdefghij")
 309
 310             self.assertEqual(writer.size(), 15)
 311             self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
 312             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:5:count.txt 100:10:count.txt\n", c.portable_manifest_text())
 313
 314             writer.arvadosfile.flush()
 315
 316             self.assertEqual(writer.size(), 15)
 317             self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
 318             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.portable_manifest_text())
 319
 320     def test_write_large_rewrite(self):
 321         keep = ArvadosFileWriterTestCase.MockKeep({})
 322         api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
 323                                                  "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n",
 324                                                  "replication_desired":None},
 325                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 326                                                  "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n"})
 327         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 328                              api_client=api, keep_client=keep) as c:
 329             writer = c.open("count.txt", "r+")
 330             text = ''.join(["0123456789" for a in xrange(0, 100)])
 331             for b in xrange(0, 100000):
 332                 writer.write(text)
 333             writer.seek(0, os.SEEK_SET)
 334             writer.write("foo")
 335             self.assertEqual(writer.size(), 100000000)
 336
 337             self.assertIsNone(c.manifest_locator())
 338             self.assertTrue(c.modified())
 339             c.save_new("test_write_large")
 340             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 341             self.assertFalse(c.modified())
 342
 343     def test_create(self):
 344         keep = ArvadosFileWriterTestCase.MockKeep({})
 345         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
 346                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 347                                                  "replication_desired":None},
 348                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 349                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
 350         with Collection(api_client=api, keep_client=keep) as c:
 351             writer = c.open("count.txt", "w+")
 352             self.assertEqual(writer.size(), 0)
 353             writer.write("01234567")
 354             self.assertEqual(writer.size(), 8)
 355
 356             self.assertIsNone(c.manifest_locator())
 357             self.assertTrue(c.modified())
 358             self.assertIsNone(keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 359             c.save_new("test_create")
 360             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 361             self.assertFalse(c.modified())
 362             self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 363
 364
 365     def test_create_subdir(self):
 366         keep = ArvadosFileWriterTestCase.MockKeep({})
 367         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
 368                                                  "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 369                                                  "replication_desired":None},
 370                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 371                                                  "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
 372         with Collection(api_client=api, keep_client=keep) as c:
 373             self.assertIsNone(c.api_response())
 374             writer = c.open("foo/bar/count.txt", "w+")
 375             writer.write("01234567")
 376             self.assertFalse(c.committed())
 377             c.save_new("test_create")
 378             self.assertTrue(c.committed())
 379             self.assertEqual(c.api_response(), api.response)
 380
 381     def test_overwrite(self):
 382         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 383         api = ArvadosFileWriterTestCase.MockApi({"name":"test_overwrite",
 384                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 385                                                  "replication_desired":None},
 386                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 387                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
 388         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 389                              api_client=api, keep_client=keep) as c:
 390             writer = c.open("count.txt", "w+")
 391             self.assertEqual(writer.size(), 0)
 392             writer.write("01234567")
 393             self.assertEqual(writer.size(), 8)
 394
 395             self.assertIsNone(c.manifest_locator())
 396             self.assertTrue(c.modified())
 397             c.save_new("test_overwrite")
 398             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 399             self.assertFalse(c.modified())
 400
 401     def test_file_not_found(self):
 402         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
 403             with self.assertRaises(IOError):
 404                 writer = c.open("nocount.txt", "r")
 405
 406     def test_cannot_open_directory(self):
 407         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
 408             with self.assertRaises(IOError):
 409                 writer = c.open(".", "r")
 410
 411     def test_create_multiple(self):
 412         keep = ArvadosFileWriterTestCase.MockKeep({})
 413         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create_multiple",
 414                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n",
 415                                                  "replication_desired":None},
 416                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 417                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n"})
 418         with Collection(api_client=api, keep_client=keep) as c:
 419             w1 = c.open("count1.txt", "w")
 420             w2 = c.open("count2.txt", "w")
 421             w1.write("01234567")
 422             w2.write("abcdefgh")
 423             self.assertEqual(w1.size(), 8)
 424             self.assertEqual(w2.size(), 8)
 425
 426             self.assertIsNone(c.manifest_locator())
 427             self.assertTrue(c.modified())
 428             self.assertIsNone(keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 429             c.save_new("test_create_multiple")
 430             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 431             self.assertFalse(c.modified())
 432             self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 433
 434
 435 class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
 436     class MockParent(object):
 437         class MockBlockMgr(object):
 438             def __init__(self, blocks, nocache):
 439                 self.blocks = blocks
 440                 self.nocache = nocache
 441
 442             def block_prefetch(self, loc):
 443                 pass
 444
 445             def get_block_contents(self, loc, num_retries=0, cache_only=False):
 446                 if self.nocache and cache_only:
 447                     return None
 448                 return self.blocks[loc]
 449
 450         def __init__(self, blocks, nocache):
 451             self.blocks = blocks
 452             self.nocache = nocache
 453             self.lock = arvados.arvfile.NoopLock()
 454
 455         def root_collection(self):
 456             return self
 457
 458         def _my_block_manager(self):
 459             return ArvadosFileReaderTestCase.MockParent.MockBlockMgr(self.blocks, self.nocache)
 460
 461
 462     def make_count_reader(self, nocache=False):
 463         stream = []
 464         n = 0
 465         blocks = {}
 466         for d in ['01234', '34567', '67890']:
 467             loc = tutil.str_keep_locator(d)
 468             blocks[loc] = d
 469             stream.append(Range(loc, n, len(d)))
 470             n += len(d)
 471         af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), "count.txt", stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
 472         return ArvadosFileReader(af)
 473
 474     def test_read_block_crossing_behavior(self):
 475         # read() needs to return all the data requested if possible, even if it
 476         # crosses uncached blocks: https://arvados.org/issues/5856
 477         sfile = self.make_count_reader(nocache=True)
 478         self.assertEqual('12345678', sfile.read(8))
 479
 480     def test_successive_reads(self):
 481         # Override StreamFileReaderTestCase.test_successive_reads
 482         sfile = self.make_count_reader(nocache=True)
 483         self.assertEqual('1234', sfile.read(4))
 484         self.assertEqual('5678', sfile.read(4))
 485         self.assertEqual('9', sfile.read(4))
 486         self.assertEqual('', sfile.read(4))
 487
 488     def test_tell_after_block_read(self):
 489         # Override StreamFileReaderTestCase.test_tell_after_block_read
 490         sfile = self.make_count_reader(nocache=True)
 491         self.assertEqual('12345678', sfile.read(8))
 492         self.assertEqual(8, sfile.tell())
 493
 494     def test_prefetch(self):
 495         keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"})
 496         with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
 497             r = c.open("count.txt", "r")
 498             self.assertEqual("0123", r.read(4))
 499         self.assertIn("2e9ec317e197819358fbc43afca7d837+8", keep.requests)
 500         self.assertIn("e8dc4081b13434b45189a720b77b6818+8", keep.requests)
 501
 502     def test__eq__from_manifest(self):
 503         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 504             with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
 505                 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
 506                 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
 507
 508     def test__eq__from_writes(self):
 509         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 510             with Collection() as c2:
 511                 f = c2.open("count1.txt", "w")
 512                 f.write("0123456789")
 513
 514                 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
 515                 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
 516
 517     def test__ne__(self):
 518         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 519             with Collection() as c2:
 520                 f = c2.open("count1.txt", "w")
 521                 f.write("1234567890")
 522
 523                 self.assertTrue(c1["count1.txt"] != c2["count1.txt"])
 524                 self.assertFalse(c1["count1.txt"] == c2["count1.txt"])
 525
 526
 527 class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
 528     def reader_for(self, coll_name, **kwargs):
 529         stream = []
 530         segments = []
 531         n = 0
 532         for d in self.manifest_for(coll_name).split():
 533             try:
 534                 k = KeepLocator(d)
 535                 segments.append(Range(n, n, k.size))
 536                 stream.append(Range(d, n, k.size))
 537                 n += k.size
 538             except ValueError:
 539                 pass
 540
 541         blockmanager = arvados.arvfile._BlockManager(self.keep_client())
 542         blockmanager.prefetch_enabled = False
 543         col = Collection(keep_client=self.keep_client(), block_manager=blockmanager)
 544         af = ArvadosFile(col, "test",
 545                          stream=stream,
 546                          segments=segments)
 547         return ArvadosFileReader(af, **kwargs)
 548
 549     def read_for_test(self, reader, byte_count, **kwargs):
 550         return reader.read(byte_count, **kwargs)
 551
 552
 553 class ArvadosFileReadFromTestCase(ArvadosFileReadTestCase):
 554     def read_for_test(self, reader, byte_count, **kwargs):
 555         return reader.readfrom(0, byte_count, **kwargs)
 556
 557
 558 class ArvadosFileReadAllTestCase(ArvadosFileReadTestCase):
 559     def read_for_test(self, reader, byte_count, **kwargs):
 560         return ''.join(reader.readall(**kwargs))
 561
 562
 563 class ArvadosFileReadAllDecompressedTestCase(ArvadosFileReadTestCase):
 564     def read_for_test(self, reader, byte_count, **kwargs):
 565         return ''.join(reader.readall_decompressed(**kwargs))
 566
 567
 568 class ArvadosFileReadlinesTestCase(ArvadosFileReadTestCase):
 569     def read_for_test(self, reader, byte_count, **kwargs):
 570         return ''.join(reader.readlines(**kwargs))
 571
 572 class BlockManagerTest(unittest.TestCase):
 573     def test_bufferblock_append(self):
 574         keep = ArvadosFileWriterTestCase.MockKeep({})
 575         with arvados.arvfile._BlockManager(keep) as blockmanager:
 576             bufferblock = blockmanager.alloc_bufferblock()
 577             bufferblock.append("foo")
 578
 579             self.assertEqual(bufferblock.size(), 3)
 580             self.assertEqual(bufferblock.buffer_view[0:3], "foo")
 581             self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 582
 583             bufferblock.append("bar")
 584
 585             self.assertEqual(bufferblock.size(), 6)
 586             self.assertEqual(bufferblock.buffer_view[0:6], "foobar")
 587             self.assertEqual(bufferblock.locator(), "3858f62230ac3c915f300c664312c63f+6")
 588
 589             bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
 590             with self.assertRaises(arvados.errors.AssertionError):
 591                 bufferblock.append("bar")
 592
 593     def test_bufferblock_dup(self):
 594         keep = ArvadosFileWriterTestCase.MockKeep({})
 595         with arvados.arvfile._BlockManager(keep) as blockmanager:
 596             bufferblock = blockmanager.alloc_bufferblock()
 597             bufferblock.append("foo")
 598
 599             self.assertEqual(bufferblock.size(), 3)
 600             self.assertEqual(bufferblock.buffer_view[0:3], "foo")
 601             self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 602             bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
 603
 604             bufferblock2 = blockmanager.dup_block(bufferblock, None)
 605             self.assertNotEqual(bufferblock.blockid, bufferblock2.blockid)
 606
 607             bufferblock2.append("bar")
 608
 609             self.assertEqual(bufferblock2.size(), 6)
 610             self.assertEqual(bufferblock2.buffer_view[0:6], "foobar")
 611             self.assertEqual(bufferblock2.locator(), "3858f62230ac3c915f300c664312c63f+6")
 612
 613             self.assertEqual(bufferblock.size(), 3)
 614             self.assertEqual(bufferblock.buffer_view[0:3], "foo")
 615             self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 616
 617     def test_bufferblock_get(self):
 618         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 619         with arvados.arvfile._BlockManager(keep) as blockmanager:
 620             bufferblock = blockmanager.alloc_bufferblock()
 621             bufferblock.append("foo")
 622
 623             self.assertEqual(blockmanager.get_block_contents("781e5e245d69b566979b86e28d23f2c7+10", 1), "0123456789")
 624             self.assertEqual(blockmanager.get_block_contents(bufferblock.blockid, 1), "foo")
 625
 626     def test_bufferblock_commit(self):
 627         mockkeep = mock.MagicMock()
 628         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
 629             bufferblock = blockmanager.alloc_bufferblock()
 630             bufferblock.owner = mock.MagicMock()
 631             def flush(sync=None):
 632                 blockmanager.commit_bufferblock(bufferblock, sync)
 633             bufferblock.owner.flush.side_effect = flush
 634             bufferblock.append("foo")
 635             blockmanager.commit_all()
 636             self.assertTrue(bufferblock.owner.flush.called)
 637             self.assertTrue(mockkeep.put.called)
 638             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
 639             self.assertIsNone(bufferblock.buffer_view)
 640
 641     def test_bufferblock_commit_pending(self):
 642         # Test for bug #7225
 643         mockkeep = mock.MagicMock()
 644         mockkeep.put.side_effect = lambda x: time.sleep(1)
 645         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
 646             bufferblock = blockmanager.alloc_bufferblock()
 647             bufferblock.append("foo")
 648
 649             blockmanager.commit_bufferblock(bufferblock, False)
 650             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.PENDING)
 651
 652             blockmanager.commit_bufferblock(bufferblock, True)
 653             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
 654
 655
 656     def test_bufferblock_commit_with_error(self):
 657         mockkeep = mock.MagicMock()
 658         mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail")
 659         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
 660             bufferblock = blockmanager.alloc_bufferblock()
 661             bufferblock.owner = mock.MagicMock()
 662             def flush(sync=None):
 663                 blockmanager.commit_bufferblock(bufferblock, sync)
 664             bufferblock.owner.flush.side_effect = flush
 665             bufferblock.append("foo")
 666             with self.assertRaises(arvados.errors.KeepWriteError) as err:
 667                 blockmanager.commit_all()
 668             self.assertTrue(bufferblock.owner.flush.called)
 669             self.assertEqual(str(err.exception), "Error writing some blocks: block acbd18db4cc2f85cedef654fccc4a4d8+3 raised KeepWriteError (fail)")
 670             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.ERROR)