import arvados
import bz2
import copy
+import hashlib
import mock
import os
import pprint
+import re
import subprocess
import tempfile
import unittest
""". bd19836ddb62c11c55ab251ccaca5645+2 0:2:f1
./d1 50170217e5b04312024aa5cd42934494+13 0:8:d2/f3 8:5:f2\n""")
+ def test_write_directory_tree_with_zero_recursion(self):
+ cwriter = arvados.CollectionWriter(self.api_client)
+ content = 'd1/d2/f3d1/f2f1'
+ blockhash = hashlib.md5(content).hexdigest() + '+' + str(len(content))
+ cwriter.write_directory_tree(
+ self.build_directory_tree(['f1', 'd1/f2', 'd1/d2/f3']),
+ max_manifest_depth=0)
+ self.assertEqual(
+ cwriter.manifest_text(),
+ ". {} 0:8:d1/d2/f3 8:5:d1/f2 13:2:f1\n".format(blockhash))
+
def test_write_one_file(self):
cwriter = arvados.CollectionWriter(self.api_client)
with self.make_test_file() as testfile:
def test_locator_init(self):
client = self.api_client_mock(200)
# Ensure Keep will not return anything if asked.
- with tutil.mock_responses(None, 404):
+ with tutil.mock_get_responses(None, 404):
reader = arvados.CollectionReader(self.DEFAULT_DATA_HASH,
api_client=client)
self.assertEqual(self.DEFAULT_MANIFEST, reader.manifest_text())
# crunch-job needs this to read manifests that have only ever
# been written to Keep.
client = self.api_client_mock(200)
- with tutil.mock_responses(self.DEFAULT_MANIFEST, 404, 200):
+ self.mock_get_collection(client, 404, None)
+ with tutil.mock_get_responses(self.DEFAULT_MANIFEST, 200):
reader = arvados.CollectionReader(self.DEFAULT_DATA_HASH,
api_client=client)
self.assertEqual(self.DEFAULT_MANIFEST, reader.manifest_text())
client = self.api_client_mock(404)
reader = arvados.CollectionReader(self.DEFAULT_UUID,
api_client=client)
- with tutil.mock_responses(self.DEFAULT_MANIFEST, 200):
+ with tutil.mock_get_responses(self.DEFAULT_MANIFEST, 200):
with self.assertRaises(arvados.errors.ApiError):
reader.manifest_text()
# To verify that CollectionReader tries Keep first here, we
# mock API server to return the wrong data.
client = self.api_client_mock(200)
- with tutil.mock_responses(self.ALT_MANIFEST, 200):
+ with tutil.mock_get_responses(self.ALT_MANIFEST, 200):
self.assertEqual(
self.ALT_MANIFEST,
arvados.CollectionReader(
client = self.api_client_mock(200)
reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client,
num_retries=3)
- with tutil.mock_responses('foo', 500, 500, 200):
+ with tutil.mock_get_responses('foo', 500, 500, 200):
self.assertEqual('foo',
''.join(f.read(9) for f in reader.all_files()))
# client should be able to use CollectionReader on a manifest without normalizing it
client = self.api_client_mock(500)
nonnormal = ". acbd18db4cc2f85cedef654fccc4a4d8+3+Aabadbadbee@abeebdee 0:3:foo.txt 1:0:bar.txt 0:3:foo.txt\n"
+ reader = arvados.CollectionReader(
+ nonnormal,
+ api_client=client, num_retries=0)
+ # Ensure stripped_manifest() doesn't mangle our manifest in
+ # any way other than stripping hints.
self.assertEqual(
- ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 1:0:bar.txt 0:3:foo.txt\n",
- arvados.CollectionReader(
- nonnormal,
- api_client=client, num_retries=0).stripped_manifest())
+ re.sub('\+[^\d\s\+]+', '', nonnormal),
+ reader.stripped_manifest())
+ # Ensure stripped_manifest() didn't mutate our reader.
+ self.assertEqual(nonnormal, reader.manifest_text())
+ # Ensure the files appear in the order given in the manifest.
self.assertEqual(
[[6, '.', 'foo.txt'],
[0, '.', 'bar.txt']],
[[f.size(), f.stream_name(), f.name()]
- for f in
- arvados.CollectionReader(
- nonnormal,
- api_client=client, num_retries=0).all_streams()[0].all_files()])
+ for f in reader.all_streams()[0].all_files()])
@tutil.skip_sleep
class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin):
def mock_keep(self, body, *codes, **headers):
headers.setdefault('x-keep-replicas-stored', 2)
- return tutil.mock_responses(body, *codes, **headers)
+ return tutil.mock_put_responses(body, *codes, **headers)
def foo_writer(self, **kwargs):
api_client = self.api_client_mock()