+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
from __future__ import absolute_import
from future.utils import listitems
import io
+import mock
+import os
+import re
import shutil
import tempfile
from . import run_test_server
from . import arvados_testutil as tutil
+from .arvados_testutil import ArvadosBaseTestCase
class ArvadosGetTestCase(run_test_server.TestCaseWithServers,
- tutil.VersionChecker):
+ tutil.VersionChecker,
+ ArvadosBaseTestCase):
MAIN_SERVER = {}
KEEP_SERVER = {}
shutil.rmtree(self.tempdir)
def write_test_collection(self,
+ strip_manifest=False,
contents = {
'foo.txt' : 'foo',
'bar.txt' : 'bar',
with c.open(path, 'wb') as f:
f.write(data)
c.save_new()
- return (c.manifest_locator(), c.portable_data_hash(), c.manifest_text())
+
+ return (c.manifest_locator(),
+ c.portable_data_hash(),
+ c.manifest_text(strip=strip_manifest))
def run_get(self, args):
self.stdout = tutil.BytesIO()
# Download the entire collection to the temp directory
r = self.run_get(["{}/".format(self.col_loc), self.tempdir])
self.assertEqual(0, r)
- with open("{}/foo.txt".format(self.tempdir), "r") as f:
+ with open(os.path.join(self.tempdir, "foo.txt"), "r") as f:
self.assertEqual("foo", f.read())
- with open("{}/bar.txt".format(self.tempdir), "r") as f:
+ with open(os.path.join(self.tempdir, "bar.txt"), "r") as f:
self.assertEqual("bar", f.read())
- with open("{}/subdir/baz.txt".format(self.tempdir), "r") as f:
+ with open(os.path.join(self.tempdir, "subdir", "baz.txt"), "r") as f:
self.assertEqual("baz", f.read())
- def test_get_collection_manifest(self):
- # Get the collection manifest
+ def test_get_collection_unstripped_manifest(self):
+ dummy_token = "+Axxxxxxx"
+ # Get the collection manifest by UUID
r = self.run_get([self.col_loc, self.tempdir])
self.assertEqual(0, r)
- with open("{}/{}".format(self.tempdir, self.col_loc), "r") as f:
- self.assertEqual(self.col_manifest, f.read())
+ m_from_collection = re.sub(r"\+A[0-9a-f@]+", dummy_token, self.col_manifest)
+ with open(os.path.join(self.tempdir, self.col_loc), "r") as f:
+ # Replace manifest tokens before comparison to avoid races
+ m_from_file = re.sub(r"\+A[0-9a-f@]+", dummy_token, f.read())
+ self.assertEqual(m_from_collection, m_from_file)
+ # Get the collection manifest by PDH
+ r = self.run_get([self.col_pdh, self.tempdir])
+ self.assertEqual(0, r)
+ with open(os.path.join(self.tempdir, self.col_pdh), "r") as f:
+ # Replace manifest tokens before comparison to avoid races
+ m_from_file = re.sub(r"\+A[0-9a-f@]+", dummy_token, f.read())
+ self.assertEqual(m_from_collection, m_from_file)
+
+ def test_get_collection_stripped_manifest(self):
+ col_loc, col_pdh, col_manifest = self.write_test_collection(
+ strip_manifest=True)
+ # Get the collection manifest by UUID
+ r = self.run_get(['--strip-manifest', col_loc, self.tempdir])
+ self.assertEqual(0, r)
+ with open(os.path.join(self.tempdir, col_loc), "r") as f:
+ self.assertEqual(col_manifest, f.read())
+ # Get the collection manifest by PDH
+ r = self.run_get(['--strip-manifest', col_pdh, self.tempdir])
+ self.assertEqual(0, r)
+ with open(os.path.join(self.tempdir, col_pdh), "r") as f:
+ self.assertEqual(col_manifest, f.read())
def test_invalid_collection(self):
# Asking for an invalid collection should generate an error.
def test_preexistent_destination(self):
# Asking to place a file with the same path as a local one should
# generate an error and avoid overwrites.
- with open("{}/foo.txt".format(self.tempdir), "w") as f:
+ with open(os.path.join(self.tempdir, "foo.txt"), "w") as f:
f.write("another foo")
r = self.run_get(["{}/foo.txt".format(self.col_loc), self.tempdir])
self.assertNotEqual(0, r)
- with open("{}/foo.txt".format(self.tempdir), "r") as f:
+ with open(os.path.join(self.tempdir, "foo.txt"), "r") as f:
self.assertEqual("another foo", f.read())
+ def test_no_progress_when_stderr_not_a_tty(self):
+ # Create a collection with a big file (>64MB) to force the progress
+ # to be printed
+ c = collection.Collection()
+ with c.open('bigfile.txt', 'wb') as f:
+ for _ in range(65):
+ f.write("x" * 1024 * 1024)
+ c.save_new()
+ tmpdir = self.make_tmpdir()
+ # Simulate a TTY stderr
+ stderr = mock.MagicMock()
+ stdout = tutil.BytesIO()
+
+ # Confirm that progress is written to stderr when is a tty
+ stderr.isatty.return_value = True
+ r = arv_get.main(['{}/bigfile.txt'.format(c.manifest_locator()),
+ '{}/bigfile.txt'.format(tmpdir)],
+ stdout, stderr)
+ self.assertEqual(0, r)
+ self.assertEqual(b'', stdout.getvalue())
+ self.assertTrue(stderr.write.called)
+
+ # Clean up and reset stderr mock
+ os.remove('{}/bigfile.txt'.format(tmpdir))
+ stderr = mock.MagicMock()
+ stdout = tutil.BytesIO()
+
+ # Confirm that progress is not written to stderr when isn't a tty
+ stderr.isatty.return_value = False
+ r = arv_get.main(['{}/bigfile.txt'.format(c.manifest_locator()),
+ '{}/bigfile.txt'.format(tmpdir)],
+ stdout, stderr)
+ self.assertEqual(0, r)
+ self.assertEqual(b'', stdout.getvalue())
+ self.assertFalse(stderr.write.called)
projects / arvados.git / blobdiff