1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
5 from __future__ import absolute_import
6 from future.utils import listitems
15 import arvados.collection as collection
16 import arvados.commands.get as arv_get
17 from . import run_test_server
19 from . import arvados_testutil as tutil
20 from .arvados_testutil import ArvadosBaseTestCase
22 class ArvadosGetTestCase(run_test_server.TestCaseWithServers,
29 super(ArvadosGetTestCase, self).setUp()
30 self.tempdir = tempfile.mkdtemp()
31 self.col_loc, self.col_pdh, self.col_manifest = self.write_test_collection()
34 super(ArvadosGetTestCase, self).tearDown()
35 shutil.rmtree(self.tempdir)
37 def write_test_collection(self,
42 'subdir/baz.txt' : 'baz',
44 c = collection.Collection()
45 for path, data in listitems(contents):
46 with c.open(path, 'wb') as f:
50 return (c.manifest_locator(),
51 c.portable_data_hash(),
52 c.manifest_text(strip=strip_manifest))
54 def run_get(self, args):
55 self.stdout = tutil.BytesIO()
56 self.stderr = tutil.StringIO()
57 return arv_get.main(args, self.stdout, self.stderr)
59 def test_version_argument(self):
60 with tutil.redirected_streams(
61 stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err):
62 with self.assertRaises(SystemExit):
63 self.run_get(['--version'])
64 self.assertVersionOutput(out, err)
66 def test_get_single_file(self):
67 # Get the file using the collection's locator
68 r = self.run_get(["{}/subdir/baz.txt".format(self.col_loc), '-'])
69 self.assertEqual(0, r)
70 self.assertEqual(b'baz', self.stdout.getvalue())
72 r = self.run_get(["{}/subdir/baz.txt".format(self.col_pdh), '-'])
73 self.assertEqual(0, r)
74 self.assertEqual(b'baz', self.stdout.getvalue())
76 def test_get_block(self):
77 # Get raw data using a block locator
78 blk = re.search(' (acbd18\S+\+A\S+) ', self.col_manifest).group(1)
79 r = self.run_get([blk, '-'])
80 self.assertEqual(0, r)
81 self.assertEqual(b'foo', self.stdout.getvalue())
83 def test_get_multiple_files(self):
84 # Download the entire collection to the temp directory
85 r = self.run_get(["{}/".format(self.col_loc), self.tempdir])
86 self.assertEqual(0, r)
87 with open(os.path.join(self.tempdir, "foo.txt"), "r") as f:
88 self.assertEqual("foo", f.read())
89 with open(os.path.join(self.tempdir, "bar.txt"), "r") as f:
90 self.assertEqual("bar", f.read())
91 with open(os.path.join(self.tempdir, "subdir", "baz.txt"), "r") as f:
92 self.assertEqual("baz", f.read())
94 def test_get_collection_unstripped_manifest(self):
95 dummy_token = "+Axxxxxxx"
96 # Get the collection manifest by UUID
97 r = self.run_get([self.col_loc, self.tempdir])
98 self.assertEqual(0, r)
99 m_from_collection = re.sub(r"\+A[0-9a-f@]+", dummy_token, self.col_manifest)
100 with open(os.path.join(self.tempdir, self.col_loc), "r") as f:
101 # Replace manifest tokens before comparison to avoid races
102 m_from_file = re.sub(r"\+A[0-9a-f@]+", dummy_token, f.read())
103 self.assertEqual(m_from_collection, m_from_file)
104 # Get the collection manifest by PDH
105 r = self.run_get([self.col_pdh, self.tempdir])
106 self.assertEqual(0, r)
107 with open(os.path.join(self.tempdir, self.col_pdh), "r") as f:
108 # Replace manifest tokens before comparison to avoid races
109 m_from_file = re.sub(r"\+A[0-9a-f@]+", dummy_token, f.read())
110 self.assertEqual(m_from_collection, m_from_file)
112 def test_get_collection_stripped_manifest(self):
113 col_loc, col_pdh, col_manifest = self.write_test_collection(
115 # Get the collection manifest by UUID
116 r = self.run_get(['--strip-manifest', col_loc, self.tempdir])
117 self.assertEqual(0, r)
118 with open(os.path.join(self.tempdir, col_loc), "r") as f:
119 self.assertEqual(col_manifest, f.read())
120 # Get the collection manifest by PDH
121 r = self.run_get(['--strip-manifest', col_pdh, self.tempdir])
122 self.assertEqual(0, r)
123 with open(os.path.join(self.tempdir, col_pdh), "r") as f:
124 self.assertEqual(col_manifest, f.read())
126 def test_invalid_collection(self):
127 # Asking for an invalid collection should generate an error.
128 r = self.run_get(['this-uuid-seems-to-be-fake', self.tempdir])
129 self.assertNotEqual(0, r)
131 def test_invalid_file_request(self):
132 # Asking for an inexistant file within a collection should generate an error.
133 r = self.run_get(["{}/im-not-here.txt".format(self.col_loc), self.tempdir])
134 self.assertNotEqual(0, r)
136 def test_invalid_destination(self):
137 # Asking to place the collection's files on a non existant directory
138 # should generate an error.
139 r = self.run_get([self.col_loc, "/fake/subdir/"])
140 self.assertNotEqual(0, r)
142 def test_preexistent_destination(self):
143 # Asking to place a file with the same path as a local one should
144 # generate an error and avoid overwrites.
145 with open(os.path.join(self.tempdir, "foo.txt"), "w") as f:
146 f.write("another foo")
147 r = self.run_get(["{}/foo.txt".format(self.col_loc), self.tempdir])
148 self.assertNotEqual(0, r)
149 with open(os.path.join(self.tempdir, "foo.txt"), "r") as f:
150 self.assertEqual("another foo", f.read())
152 def test_no_progress_when_stderr_not_a_tty(self):
153 # Create a collection with a big file (>64MB) to force the progress
155 c = collection.Collection()
156 with c.open('bigfile.txt', 'wb') as f:
158 f.write("x" * 1024 * 1024)
160 tmpdir = self.make_tmpdir()
161 # Simulate a TTY stderr
162 stderr = mock.MagicMock()
163 stdout = tutil.BytesIO()
165 # Confirm that progress is written to stderr when is a tty
166 stderr.isatty.return_value = True
167 r = arv_get.main(['{}/bigfile.txt'.format(c.manifest_locator()),
168 '{}/bigfile.txt'.format(tmpdir)],
170 self.assertEqual(0, r)
171 self.assertEqual(b'', stdout.getvalue())
172 self.assertTrue(stderr.write.called)
174 # Clean up and reset stderr mock
175 os.remove('{}/bigfile.txt'.format(tmpdir))
176 stderr = mock.MagicMock()
177 stdout = tutil.BytesIO()
179 # Confirm that progress is not written to stderr when isn't a tty
180 stderr.isatty.return_value = False
181 r = arv_get.main(['{}/bigfile.txt'.format(c.manifest_locator()),
182 '{}/bigfile.txt'.format(tmpdir)],
184 self.assertEqual(0, r)
185 self.assertEqual(b'', stdout.getvalue())
186 self.assertFalse(stderr.write.called)