X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/151df8c3b177e4971bfbdf68c87d89599dbe0812..3fa6aa4043286ad61e5f29c136d3cc2942e8750d:/sdk/python/tests/test_arv_get.py diff --git a/sdk/python/tests/test_arv_get.py b/sdk/python/tests/test_arv_get.py index b7f5e2b9c6..73ef2475b9 100644 --- a/sdk/python/tests/test_arv_get.py +++ b/sdk/python/tests/test_arv_get.py @@ -1,7 +1,12 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +from __future__ import absolute_import +from future.utils import listitems import io +import logging +import mock import os import re import shutil @@ -10,11 +15,14 @@ import tempfile import arvados import arvados.collection as collection import arvados.commands.get as arv_get -import run_test_server +from . import run_test_server -from arvados_testutil import redirected_streams +from . import arvados_testutil as tutil +from .arvados_testutil import ArvadosBaseTestCase -class ArvadosGetTestCase(run_test_server.TestCaseWithServers): +class ArvadosGetTestCase(run_test_server.TestCaseWithServers, + tutil.VersionChecker, + ArvadosBaseTestCase): MAIN_SERVER = {} KEEP_SERVER = {} @@ -23,7 +31,14 @@ class ArvadosGetTestCase(run_test_server.TestCaseWithServers): self.tempdir = tempfile.mkdtemp() self.col_loc, self.col_pdh, self.col_manifest = self.write_test_collection() + self.stdout = tutil.BytesIO() + self.stderr = tutil.StringIO() + self.loggingHandler = logging.StreamHandler(self.stderr) + self.loggingHandler.setFormatter(logging.Formatter('%(levelname)s: %(message)s')) + logging.getLogger().addHandler(self.loggingHandler) + def tearDown(self): + logging.getLogger().removeHandler(self.loggingHandler) super(ArvadosGetTestCase, self).tearDown() shutil.rmtree(self.tempdir) @@ -34,38 +49,49 @@ class ArvadosGetTestCase(run_test_server.TestCaseWithServers): 'bar.txt' : 'bar', 'subdir/baz.txt' : 'baz', }): - c = collection.Collection() - for path, data in contents.items(): - with c.open(path, 'w') as f: + api = arvados.api() + c = collection.Collection(api_client=api) + for path, data in listitems(contents): + with c.open(path, 'wb') as f: f.write(data) c.save_new() + + api.close_connections() + return (c.manifest_locator(), c.portable_data_hash(), c.manifest_text(strip=strip_manifest)) - + def run_get(self, args): - self.stdout = io.BytesIO() - self.stderr = io.BytesIO() + self.stdout.seek(0, 0) + self.stdout.truncate(0) + self.stderr.seek(0, 0) + self.stderr.truncate(0) return arv_get.main(args, self.stdout, self.stderr) def test_version_argument(self): - err = io.BytesIO() - out = io.BytesIO() - with redirected_streams(stdout=out, stderr=err): + with tutil.redirected_streams( + stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err): with self.assertRaises(SystemExit): self.run_get(['--version']) - self.assertEqual(out.getvalue(), '') - self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+") + self.assertVersionOutput(out, err) def test_get_single_file(self): # Get the file using the collection's locator r = self.run_get(["{}/subdir/baz.txt".format(self.col_loc), '-']) self.assertEqual(0, r) - self.assertEqual('baz', self.stdout.getvalue()) + self.assertEqual(b'baz', self.stdout.getvalue()) # Then, try by PDH r = self.run_get(["{}/subdir/baz.txt".format(self.col_pdh), '-']) self.assertEqual(0, r) - self.assertEqual('baz', self.stdout.getvalue()) + self.assertEqual(b'baz', self.stdout.getvalue()) + + def test_get_block(self): + # Get raw data using a block locator + blk = re.search(' (acbd18\S+\+A\S+) ', self.col_manifest).group(1) + r = self.run_get([blk, '-']) + self.assertEqual(0, r) + self.assertEqual(b'foo', self.stdout.getvalue()) def test_get_multiple_files(self): # Download the entire collection to the temp directory @@ -97,7 +123,8 @@ class ArvadosGetTestCase(run_test_server.TestCaseWithServers): self.assertEqual(m_from_collection, m_from_file) def test_get_collection_stripped_manifest(self): - col_loc, col_pdh, col_manifest = self.write_test_collection(strip_manifest=True) + col_loc, col_pdh, col_manifest = self.write_test_collection( + strip_manifest=True) # Get the collection manifest by UUID r = self.run_get(['--strip-manifest', col_loc, self.tempdir]) self.assertEqual(0, r) @@ -135,3 +162,50 @@ class ArvadosGetTestCase(run_test_server.TestCaseWithServers): with open(os.path.join(self.tempdir, "foo.txt"), "r") as f: self.assertEqual("another foo", f.read()) + def test_no_progress_when_stderr_not_a_tty(self): + # Create a collection with a big file (>64MB) to force the progress + # to be printed + c = collection.Collection() + with c.open('bigfile.txt', 'wb') as f: + for _ in range(65): + f.write("x" * 1024 * 1024) + c.save_new() + tmpdir = self.make_tmpdir() + # Simulate a TTY stderr + stderr = mock.MagicMock() + stdout = tutil.BytesIO() + + # Confirm that progress is written to stderr when is a tty + stderr.isatty.return_value = True + r = arv_get.main(['{}/bigfile.txt'.format(c.manifest_locator()), + '{}/bigfile.txt'.format(tmpdir)], + stdout, stderr) + self.assertEqual(0, r) + self.assertEqual(b'', stdout.getvalue()) + self.assertTrue(stderr.write.called) + + # Clean up and reset stderr mock + os.remove('{}/bigfile.txt'.format(tmpdir)) + stderr = mock.MagicMock() + stdout = tutil.BytesIO() + + # Confirm that progress is not written to stderr when isn't a tty + stderr.isatty.return_value = False + r = arv_get.main(['{}/bigfile.txt'.format(c.manifest_locator()), + '{}/bigfile.txt'.format(tmpdir)], + stdout, stderr) + self.assertEqual(0, r) + self.assertEqual(b'', stdout.getvalue()) + self.assertFalse(stderr.write.called) + + request_id_regex = r'INFO: X-Request-Id: req-[a-z0-9]{20}\n' + + def test_request_id_logging_on(self): + r = self.run_get(["-v", "{}/".format(self.col_loc), self.tempdir]) + self.assertEqual(0, r) + self.assertRegex(self.stderr.getvalue(), self.request_id_regex) + + def test_request_id_logging_off(self): + r = self.run_get(["{}/".format(self.col_loc), self.tempdir]) + self.assertEqual(0, r) + self.assertNotRegex(self.stderr.getvalue(), self.request_id_regex)