sdk/python/tests/test_arvfile.py

   1 #!/usr/bin/env python
   2
   3 import bz2
   4 import gzip
   5 import io
   6 import mock
   7 import os
   8 import unittest
   9 import time
  10
  11 import arvados
  12 from arvados._ranges import Range
  13 from arvados.keep import KeepLocator
  14 from arvados.collection import Collection, CollectionReader
  15 from arvados.arvfile import ArvadosFile, ArvadosFileReader
  16
  17 import arvados_testutil as tutil
  18 from test_stream import StreamFileReaderTestCase, StreamRetryTestMixin
  19
  20 class ArvadosFileWriterTestCase(unittest.TestCase):
  21     class MockKeep(object):
  22         def __init__(self, blocks):
  23             self.blocks = blocks
  24             self.requests = []
  25         def get(self, locator, num_retries=0):
  26             self.requests.append(locator)
  27             return self.blocks.get(locator)
  28         def get_from_cache(self, locator):
  29             self.requests.append(locator)
  30             return self.blocks.get(locator)
  31         def put(self, data, num_retries=None, copies=None):
  32             pdh = tutil.str_keep_locator(data)
  33             self.blocks[pdh] = str(data)
  34             return pdh
  35
  36     class MockApi(object):
  37         def __init__(self, b, r):
  38             self.body = b
  39             self.response = r
  40             self._schema = ArvadosFileWriterTestCase.MockApi.MockSchema()
  41             self._rootDesc = {}
  42         class MockSchema(object):
  43             def __init__(self):
  44                 self.schemas = {'Collection': {'properties': {'replication_desired': {'type':'integer'}}}}
  45         class MockCollections(object):
  46             def __init__(self, b, r):
  47                 self.body = b
  48                 self.response = r
  49             class Execute(object):
  50                 def __init__(self, r):
  51                     self.response = r
  52                 def execute(self, num_retries=None):
  53                     return self.response
  54             def create(self, ensure_unique_name=False, body=None):
  55                 if body != self.body:
  56                     raise Exception("Body %s does not match expectation %s" % (body, self.body))
  57                 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
  58             def update(self, uuid=None, body=None):
  59                 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
  60         def collections(self):
  61             return ArvadosFileWriterTestCase.MockApi.MockCollections(self.body, self.response)
  62
  63
  64     def test_truncate(self):
  65         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
  66         api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate",
  67                                                  "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
  68                                                  "replication_desired":None},
  69                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
  70                                                  "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n"})
  71         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
  72                              api_client=api, keep_client=keep) as c:
  73             writer = c.open("count.txt", "r+")
  74             self.assertEqual(writer.size(), 10)
  75             self.assertEqual("0123456789", writer.read(12))
  76
  77             writer.truncate(8)
  78
  79             # Make sure reading off the end doesn't break
  80             self.assertEqual("", writer.read(12))
  81
  82             self.assertEqual(writer.size(), 8)
  83             writer.seek(0, os.SEEK_SET)
  84             self.assertEqual("01234567", writer.read(12))
  85
  86             self.assertIsNone(c.manifest_locator())
  87             self.assertTrue(c.modified())
  88             c.save_new("test_truncate")
  89             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
  90             self.assertFalse(c.modified())
  91
  92     def test_write_to_end(self):
  93         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
  94         api = ArvadosFileWriterTestCase.MockApi({"name":"test_append",
  95                                                  "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
  96                                                  "replication_desired":None},
  97                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
  98                                                  "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n"})
  99         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 100                              api_client=api, keep_client=keep) as c:
 101             writer = c.open("count.txt", "r+")
 102             self.assertEqual(writer.size(), 10)
 103
 104             writer.seek(5, os.SEEK_SET)
 105             self.assertEqual("56789", writer.read(8))
 106
 107             writer.seek(10, os.SEEK_SET)
 108             writer.write("foo")
 109             self.assertEqual(writer.size(), 13)
 110
 111             writer.seek(5, os.SEEK_SET)
 112             self.assertEqual("56789foo", writer.read(8))
 113
 114             self.assertIsNone(c.manifest_locator())
 115             self.assertTrue(c.modified())
 116             self.assertIsNone(keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
 117
 118             c.save_new("test_append")
 119             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 120             self.assertFalse(c.modified())
 121             self.assertEqual("foo", keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
 122
 123
 124     def test_append(self):
 125         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 126         c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
 127         writer = c.open("count.txt", "a+")
 128         self.assertEqual(writer.read(20), "0123456789")
 129         writer.seek(0, os.SEEK_SET)
 130
 131         writer.write("hello")
 132         self.assertEqual(writer.read(20), "0123456789hello")
 133         writer.seek(0, os.SEEK_SET)
 134
 135         writer.write("world")
 136         self.assertEqual(writer.read(20), "0123456789helloworld")
 137
 138         self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.portable_manifest_text())
 139
 140     def test_write_at_beginning(self):
 141         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 142         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 143                              keep_client=keep) as c:
 144             writer = c.open("count.txt", "r+")
 145             self.assertEqual("0123456789", writer.readfrom(0, 13))
 146             writer.seek(0, os.SEEK_SET)
 147             writer.write("foo")
 148             self.assertEqual(writer.size(), 10)
 149             self.assertEqual("foo3456789", writer.readfrom(0, 13))
 150             self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", c.portable_manifest_text())
 151
 152     def test_write_empty(self):
 153         keep = ArvadosFileWriterTestCase.MockKeep({})
 154         with Collection(keep_client=keep) as c:
 155             writer = c.open("count.txt", "w")
 156             self.assertEqual(writer.size(), 0)
 157             self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:count.txt\n", c.portable_manifest_text())
 158
 159     def test_save_manifest_text(self):
 160         keep = ArvadosFileWriterTestCase.MockKeep({})
 161         with Collection(keep_client=keep) as c:
 162             writer = c.open("count.txt", "w")
 163             writer.write("0123456789")
 164             self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
 165             self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 166
 167             self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.save_new(create_collection_record=False))
 168             self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 169
 170     def test_get_manifest_text_commits(self):
 171          keep = ArvadosFileWriterTestCase.MockKeep({})
 172          with Collection(keep_client=keep) as c:
 173              writer = c.open("count.txt", "w")
 174              writer.write("0123456789")
 175              self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
 176              self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 177              self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.manifest_text())
 178              self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
 179
 180
 181     def test_write_in_middle(self):
 182         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 183         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 184                              keep_client=keep) as c:
 185             writer = c.open("count.txt", "r+")
 186             self.assertEqual("0123456789", writer.readfrom(0, 13))
 187             writer.seek(3, os.SEEK_SET)
 188             writer.write("foo")
 189             self.assertEqual(writer.size(), 10)
 190             self.assertEqual("012foo6789", writer.readfrom(0, 13))
 191             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", c.portable_manifest_text())
 192
 193     def test_write_at_end(self):
 194         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 195         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 196                              keep_client=keep) as c:
 197             writer = c.open("count.txt", "r+")
 198             self.assertEqual("0123456789", writer.readfrom(0, 13))
 199             writer.seek(7, os.SEEK_SET)
 200             writer.write("foo")
 201             self.assertEqual(writer.size(), 10)
 202             self.assertEqual("0123456foo", writer.readfrom(0, 13))
 203             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", c.portable_manifest_text())
 204
 205     def test_write_across_segment_boundary(self):
 206         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 207         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
 208                              keep_client=keep) as c:
 209             writer = c.open("count.txt", "r+")
 210             self.assertEqual("012345678901234", writer.readfrom(0, 15))
 211             writer.seek(7, os.SEEK_SET)
 212             writer.write("foobar")
 213             self.assertEqual(writer.size(), 20)
 214             self.assertEqual("0123456foobar34", writer.readfrom(0, 15))
 215             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", c.portable_manifest_text())
 216
 217     def test_write_across_several_segments(self):
 218         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 219         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
 220                              keep_client=keep) as c:
 221             writer = c.open("count.txt", "r+")
 222             self.assertEqual("012301230123", writer.readfrom(0, 15))
 223             writer.seek(2, os.SEEK_SET)
 224             writer.write("abcdefg")
 225             self.assertEqual(writer.size(), 12)
 226             self.assertEqual("01abcdefg123", writer.readfrom(0, 15))
 227             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", c.portable_manifest_text())
 228
 229     def test_write_large(self):
 230         keep = ArvadosFileWriterTestCase.MockKeep({})
 231         api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
 232                                                  "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n",
 233                                                  "replication_desired":None},
 234                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 235                                                  "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n"})
 236         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 237                              api_client=api, keep_client=keep) as c:
 238             writer = c.open("count.txt", "r+")
 239             text = "0123456789" * 100
 240             for b in xrange(0, 100000):
 241                 writer.write(text)
 242             self.assertEqual(writer.size(), 100000000)
 243
 244             self.assertIsNone(c.manifest_locator())
 245             self.assertTrue(c.modified())
 246             c.save_new("test_write_large")
 247             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 248             self.assertFalse(c.modified())
 249
 250
 251     def test_large_write(self):
 252         keep = ArvadosFileWriterTestCase.MockKeep({})
 253         api = ArvadosFileWriterTestCase.MockApi({}, {})
 254         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 255                              api_client=api, keep_client=keep) as c:
 256             writer = c.open("count.txt", "r+")
 257             self.assertEqual(writer.size(), 0)
 258
 259             text = "0123456789"
 260             writer.write(text)
 261             text = "0123456789" * 9999999
 262             writer.write(text)
 263             self.assertEqual(writer.size(), 100000000)
 264
 265             self.assertEqual(c.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 48dd23ea1645fd47d789804d71b5bb8e+67108864 77c57dc6ac5a10bb2205caaa73187994+32891126 0:100000000:count.txt\n")
 266
 267     def test_rewrite_on_empty_file(self):
 268         keep = ArvadosFileWriterTestCase.MockKeep({})
 269         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 270                              keep_client=keep) as c:
 271             writer = c.open("count.txt", "r+")
 272             for b in xrange(0, 10):
 273                 writer.seek(0, os.SEEK_SET)
 274                 writer.write("0123456789")
 275
 276             self.assertEqual(writer.size(), 10)
 277             self.assertEqual("0123456789", writer.readfrom(0, 20))
 278             self.assertEqual(". 7a08b07e84641703e5f2c836aa59a170+100 90:10:count.txt\n", c.portable_manifest_text())
 279             writer.flush()
 280             self.assertEqual(writer.size(), 10)
 281             self.assertEqual("0123456789", writer.readfrom(0, 20))
 282             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.portable_manifest_text())
 283
 284     def test_rewrite_append_existing_file(self):
 285         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 286         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
 287                              keep_client=keep) as c:
 288             writer = c.open("count.txt", "r+")
 289             for b in xrange(0, 10):
 290                 writer.seek(10, os.SEEK_SET)
 291                 writer.write("abcdefghij")
 292
 293             self.assertEqual(writer.size(), 20)
 294             self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
 295             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:10:count.txt 100:10:count.txt\n", c.portable_manifest_text())
 296
 297             writer.arvadosfile.flush()
 298             self.assertEqual(writer.size(), 20)
 299             self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
 300             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.portable_manifest_text())
 301
 302     def test_rewrite_over_existing_file(self):
 303         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 304         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
 305                              keep_client=keep) as c:
 306             writer = c.open("count.txt", "r+")
 307             for b in xrange(0, 10):
 308                 writer.seek(5, os.SEEK_SET)
 309                 writer.write("abcdefghij")
 310
 311             self.assertEqual(writer.size(), 15)
 312             self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
 313             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:5:count.txt 100:10:count.txt\n", c.portable_manifest_text())
 314
 315             writer.arvadosfile.flush()
 316
 317             self.assertEqual(writer.size(), 15)
 318             self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
 319             self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.portable_manifest_text())
 320
 321     def test_write_large_rewrite(self):
 322         keep = ArvadosFileWriterTestCase.MockKeep({})
 323         api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
 324                                                  "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n",
 325                                                  "replication_desired":None},
 326                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 327                                                  "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n"})
 328         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
 329                              api_client=api, keep_client=keep) as c:
 330             writer = c.open("count.txt", "r+")
 331             text = ''.join(["0123456789" for a in xrange(0, 100)])
 332             for b in xrange(0, 100000):
 333                 writer.write(text)
 334             writer.seek(0, os.SEEK_SET)
 335             writer.write("foo")
 336             self.assertEqual(writer.size(), 100000000)
 337
 338             self.assertIsNone(c.manifest_locator())
 339             self.assertTrue(c.modified())
 340             c.save_new("test_write_large")
 341             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 342             self.assertFalse(c.modified())
 343
 344     def test_create(self):
 345         keep = ArvadosFileWriterTestCase.MockKeep({})
 346         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
 347                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 348                                                  "replication_desired":None},
 349                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 350                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
 351         with Collection(api_client=api, keep_client=keep) as c:
 352             writer = c.open("count.txt", "w+")
 353             self.assertEqual(writer.size(), 0)
 354             writer.write("01234567")
 355             self.assertEqual(writer.size(), 8)
 356
 357             self.assertIsNone(c.manifest_locator())
 358             self.assertTrue(c.modified())
 359             self.assertIsNone(keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 360             c.save_new("test_create")
 361             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 362             self.assertFalse(c.modified())
 363             self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 364
 365
 366     def test_create_subdir(self):
 367         keep = ArvadosFileWriterTestCase.MockKeep({})
 368         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
 369                                                  "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 370                                                  "replication_desired":None},
 371                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 372                                                  "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
 373         with Collection(api_client=api, keep_client=keep) as c:
 374             self.assertIsNone(c.api_response())
 375             writer = c.open("foo/bar/count.txt", "w+")
 376             writer.write("01234567")
 377             self.assertFalse(c.committed())
 378             c.save_new("test_create")
 379             self.assertTrue(c.committed())
 380             self.assertEqual(c.api_response(), api.response)
 381
 382     def test_overwrite(self):
 383         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 384         api = ArvadosFileWriterTestCase.MockApi({"name":"test_overwrite",
 385                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
 386                                                  "replication_desired":None},
 387                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 388                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
 389         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
 390                              api_client=api, keep_client=keep) as c:
 391             writer = c.open("count.txt", "w+")
 392             self.assertEqual(writer.size(), 0)
 393             writer.write("01234567")
 394             self.assertEqual(writer.size(), 8)
 395
 396             self.assertIsNone(c.manifest_locator())
 397             self.assertTrue(c.modified())
 398             c.save_new("test_overwrite")
 399             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 400             self.assertFalse(c.modified())
 401
 402     def test_file_not_found(self):
 403         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
 404             with self.assertRaises(IOError):
 405                 writer = c.open("nocount.txt", "r")
 406
 407     def test_cannot_open_directory(self):
 408         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
 409             with self.assertRaises(IOError):
 410                 writer = c.open(".", "r")
 411
 412     def test_create_multiple(self):
 413         keep = ArvadosFileWriterTestCase.MockKeep({})
 414         api = ArvadosFileWriterTestCase.MockApi({"name":"test_create_multiple",
 415                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n",
 416                                                  "replication_desired":None},
 417                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
 418                                                  "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n"})
 419         with Collection(api_client=api, keep_client=keep) as c:
 420             w1 = c.open("count1.txt", "w")
 421             w2 = c.open("count2.txt", "w")
 422             w1.write("01234567")
 423             w2.write("abcdefgh")
 424             self.assertEqual(w1.size(), 8)
 425             self.assertEqual(w2.size(), 8)
 426
 427             self.assertIsNone(c.manifest_locator())
 428             self.assertTrue(c.modified())
 429             self.assertIsNone(keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 430             c.save_new("test_create_multiple")
 431             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
 432             self.assertFalse(c.modified())
 433             self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 434
 435
 436 class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
 437     class MockParent(object):
 438         class MockBlockMgr(object):
 439             def __init__(self, blocks, nocache):
 440                 self.blocks = blocks
 441                 self.nocache = nocache
 442
 443             def block_prefetch(self, loc):
 444                 pass
 445
 446             def get_block_contents(self, loc, num_retries=0, cache_only=False):
 447                 if self.nocache and cache_only:
 448                     return None
 449                 return self.blocks[loc]
 450
 451         def __init__(self, blocks, nocache):
 452             self.blocks = blocks
 453             self.nocache = nocache
 454             self.lock = arvados.arvfile.NoopLock()
 455
 456         def root_collection(self):
 457             return self
 458
 459         def _my_block_manager(self):
 460             return ArvadosFileReaderTestCase.MockParent.MockBlockMgr(self.blocks, self.nocache)
 461
 462
 463     def make_count_reader(self, nocache=False):
 464         stream = []
 465         n = 0
 466         blocks = {}
 467         for d in ['01234', '34567', '67890']:
 468             loc = tutil.str_keep_locator(d)
 469             blocks[loc] = d
 470             stream.append(Range(loc, n, len(d)))
 471             n += len(d)
 472         af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), "count.txt", stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
 473         return ArvadosFileReader(af)
 474
 475     def test_read_block_crossing_behavior(self):
 476         # read() needs to return all the data requested if possible, even if it
 477         # crosses uncached blocks: https://arvados.org/issues/5856
 478         sfile = self.make_count_reader(nocache=True)
 479         self.assertEqual('12345678', sfile.read(8))
 480
 481     def test_successive_reads(self):
 482         # Override StreamFileReaderTestCase.test_successive_reads
 483         sfile = self.make_count_reader(nocache=True)
 484         self.assertEqual('1234', sfile.read(4))
 485         self.assertEqual('5678', sfile.read(4))
 486         self.assertEqual('9', sfile.read(4))
 487         self.assertEqual('', sfile.read(4))
 488
 489     def test_tell_after_block_read(self):
 490         # Override StreamFileReaderTestCase.test_tell_after_block_read
 491         sfile = self.make_count_reader(nocache=True)
 492         self.assertEqual('12345678', sfile.read(8))
 493         self.assertEqual(8, sfile.tell())
 494
 495     def test_prefetch(self):
 496         keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"})
 497         with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
 498             r = c.open("count.txt", "r")
 499             self.assertEqual("0123", r.read(4))
 500         self.assertIn("2e9ec317e197819358fbc43afca7d837+8", keep.requests)
 501         self.assertIn("e8dc4081b13434b45189a720b77b6818+8", keep.requests)
 502
 503     def test__eq__from_manifest(self):
 504         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 505             with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
 506                 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
 507                 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
 508
 509     def test__eq__from_writes(self):
 510         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 511             with Collection() as c2:
 512                 f = c2.open("count1.txt", "w")
 513                 f.write("0123456789")
 514
 515                 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
 516                 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
 517
 518     def test__ne__(self):
 519         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
 520             with Collection() as c2:
 521                 f = c2.open("count1.txt", "w")
 522                 f.write("1234567890")
 523
 524                 self.assertTrue(c1["count1.txt"] != c2["count1.txt"])
 525                 self.assertFalse(c1["count1.txt"] == c2["count1.txt"])
 526
 527
 528 class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
 529     def reader_for(self, coll_name, **kwargs):
 530         stream = []
 531         segments = []
 532         n = 0
 533         for d in self.manifest_for(coll_name).split():
 534             try:
 535                 k = KeepLocator(d)
 536                 segments.append(Range(n, n, k.size))
 537                 stream.append(Range(d, n, k.size))
 538                 n += k.size
 539             except ValueError:
 540                 pass
 541
 542         blockmanager = arvados.arvfile._BlockManager(self.keep_client())
 543         blockmanager.prefetch_enabled = False
 544         col = Collection(keep_client=self.keep_client(), block_manager=blockmanager)
 545         af = ArvadosFile(col, "test",
 546                          stream=stream,
 547                          segments=segments)
 548         return ArvadosFileReader(af, **kwargs)
 549
 550     def read_for_test(self, reader, byte_count, **kwargs):
 551         return reader.read(byte_count, **kwargs)
 552
 553
 554 class ArvadosFileReadFromTestCase(ArvadosFileReadTestCase):
 555     def read_for_test(self, reader, byte_count, **kwargs):
 556         return reader.readfrom(0, byte_count, **kwargs)
 557
 558
 559 class ArvadosFileReadAllTestCase(ArvadosFileReadTestCase):
 560     def read_for_test(self, reader, byte_count, **kwargs):
 561         return ''.join(reader.readall(**kwargs))
 562
 563
 564 class ArvadosFileReadAllDecompressedTestCase(ArvadosFileReadTestCase):
 565     def read_for_test(self, reader, byte_count, **kwargs):
 566         return ''.join(reader.readall_decompressed(**kwargs))
 567
 568
 569 class ArvadosFileReadlinesTestCase(ArvadosFileReadTestCase):
 570     def read_for_test(self, reader, byte_count, **kwargs):
 571         return ''.join(reader.readlines(**kwargs))
 572
 573 class BlockManagerTest(unittest.TestCase):
 574     def test_bufferblock_append(self):
 575         keep = ArvadosFileWriterTestCase.MockKeep({})
 576         with arvados.arvfile._BlockManager(keep) as blockmanager:
 577             bufferblock = blockmanager.alloc_bufferblock()
 578             bufferblock.append("foo")
 579
 580             self.assertEqual(bufferblock.size(), 3)
 581             self.assertEqual(bufferblock.buffer_view[0:3], "foo")
 582             self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 583
 584             bufferblock.append("bar")
 585
 586             self.assertEqual(bufferblock.size(), 6)
 587             self.assertEqual(bufferblock.buffer_view[0:6], "foobar")
 588             self.assertEqual(bufferblock.locator(), "3858f62230ac3c915f300c664312c63f+6")
 589
 590             bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
 591             with self.assertRaises(arvados.errors.AssertionError):
 592                 bufferblock.append("bar")
 593
 594     def test_bufferblock_dup(self):
 595         keep = ArvadosFileWriterTestCase.MockKeep({})
 596         with arvados.arvfile._BlockManager(keep) as blockmanager:
 597             bufferblock = blockmanager.alloc_bufferblock()
 598             bufferblock.append("foo")
 599
 600             self.assertEqual(bufferblock.size(), 3)
 601             self.assertEqual(bufferblock.buffer_view[0:3], "foo")
 602             self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 603             bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
 604
 605             bufferblock2 = blockmanager.dup_block(bufferblock, None)
 606             self.assertNotEqual(bufferblock.blockid, bufferblock2.blockid)
 607
 608             bufferblock2.append("bar")
 609
 610             self.assertEqual(bufferblock2.size(), 6)
 611             self.assertEqual(bufferblock2.buffer_view[0:6], "foobar")
 612             self.assertEqual(bufferblock2.locator(), "3858f62230ac3c915f300c664312c63f+6")
 613
 614             self.assertEqual(bufferblock.size(), 3)
 615             self.assertEqual(bufferblock.buffer_view[0:3], "foo")
 616             self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
 617
 618     def test_bufferblock_get(self):
 619         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 620         with arvados.arvfile._BlockManager(keep) as blockmanager:
 621             bufferblock = blockmanager.alloc_bufferblock()
 622             bufferblock.append("foo")
 623
 624             self.assertEqual(blockmanager.get_block_contents("781e5e245d69b566979b86e28d23f2c7+10", 1), "0123456789")
 625             self.assertEqual(blockmanager.get_block_contents(bufferblock.blockid, 1), "foo")
 626
 627     def test_bufferblock_commit(self):
 628         mockkeep = mock.MagicMock()
 629         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
 630             bufferblock = blockmanager.alloc_bufferblock()
 631             bufferblock.owner = mock.MagicMock()
 632             def flush(sync=None):
 633                 blockmanager.commit_bufferblock(bufferblock, sync)
 634             bufferblock.owner.flush.side_effect = flush
 635             bufferblock.append("foo")
 636             blockmanager.commit_all()
 637             self.assertTrue(bufferblock.owner.flush.called)
 638             self.assertTrue(mockkeep.put.called)
 639             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
 640             self.assertIsNone(bufferblock.buffer_view)
 641
 642     def test_bufferblock_commit_pending(self):
 643         # Test for bug #7225
 644         mockkeep = mock.MagicMock()
 645         mockkeep.put.side_effect = lambda x: time.sleep(1)
 646         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
 647             bufferblock = blockmanager.alloc_bufferblock()
 648             bufferblock.append("foo")
 649
 650             blockmanager.commit_bufferblock(bufferblock, False)
 651             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.PENDING)
 652
 653             blockmanager.commit_bufferblock(bufferblock, True)
 654             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
 655
 656
 657     def test_bufferblock_commit_with_error(self):
 658         mockkeep = mock.MagicMock()
 659         mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail")
 660         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
 661             bufferblock = blockmanager.alloc_bufferblock()
 662             bufferblock.owner = mock.MagicMock()
 663             def flush(sync=None):
 664                 blockmanager.commit_bufferblock(bufferblock, sync)
 665             bufferblock.owner.flush.side_effect = flush
 666             bufferblock.append("foo")
 667             with self.assertRaises(arvados.errors.KeepWriteError) as err:
 668                 blockmanager.commit_all()
 669             self.assertTrue(bufferblock.owner.flush.called)
 670             self.assertEqual(str(err.exception), "Error writing some blocks: block acbd18db4cc2f85cedef654fccc4a4d8+3 raised KeepWriteError (fail)")
 671             self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.ERROR)