sdk/python/tests/test_arv_put.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from __future__ import absolute_import
   6 from __future__ import division
   7 from future import standard_library
   8 standard_library.install_aliases()
   9 from builtins import str
  10 from builtins import range
  11 import apiclient
  12 import datetime
  13 import hashlib
  14 import json
  15 import logging
  16 import mock
  17 import os
  18 import pwd
  19 import random
  20 import re
  21 import shutil
  22 import signal
  23 import subprocess
  24 import sys
  25 import tempfile
  26 import threading
  27 import time
  28 import unittest
  29 import uuid
  30 import yaml
  31
  32 import arvados
  33 import arvados.commands.put as arv_put
  34 from . import arvados_testutil as tutil
  35
  36 from .arvados_testutil import ArvadosBaseTestCase, fake_httplib2_response
  37 from . import run_test_server
  38
  39 class ArvadosPutResumeCacheTest(ArvadosBaseTestCase):
  40     CACHE_ARGSET = [
  41         [],
  42         ['/dev/null'],
  43         ['/dev/null', '--filename', 'empty'],
  44         ['/tmp']
  45         ]
  46
  47     def tearDown(self):
  48         super(ArvadosPutResumeCacheTest, self).tearDown()
  49         try:
  50             self.last_cache.destroy()
  51         except AttributeError:
  52             pass
  53
  54     def cache_path_from_arglist(self, arglist):
  55         return arv_put.ResumeCache.make_path(arv_put.parse_arguments(arglist))
  56
  57     def test_cache_names_stable(self):
  58         for argset in self.CACHE_ARGSET:
  59             self.assertEqual(self.cache_path_from_arglist(argset),
  60                               self.cache_path_from_arglist(argset),
  61                               "cache name changed for {}".format(argset))
  62
  63     def test_cache_names_unique(self):
  64         results = []
  65         for argset in self.CACHE_ARGSET:
  66             path = self.cache_path_from_arglist(argset)
  67             self.assertNotIn(path, results)
  68             results.append(path)
  69
  70     def test_cache_names_simple(self):
  71         # The goal here is to make sure the filename doesn't use characters
  72         # reserved by the filesystem.  Feel free to adjust this regexp as
  73         # long as it still does that.
  74         bad_chars = re.compile(r'[^-\.\w]')
  75         for argset in self.CACHE_ARGSET:
  76             path = self.cache_path_from_arglist(argset)
  77             self.assertFalse(bad_chars.search(os.path.basename(path)),
  78                              "path too exotic: {}".format(path))
  79
  80     def test_cache_names_ignore_argument_order(self):
  81         self.assertEqual(
  82             self.cache_path_from_arglist(['a', 'b', 'c']),
  83             self.cache_path_from_arglist(['c', 'a', 'b']))
  84         self.assertEqual(
  85             self.cache_path_from_arglist(['-', '--filename', 'stdin']),
  86             self.cache_path_from_arglist(['--filename', 'stdin', '-']))
  87
  88     def test_cache_names_differ_for_similar_paths(self):
  89         # This test needs names at / that don't exist on the real filesystem.
  90         self.assertNotEqual(
  91             self.cache_path_from_arglist(['/_arvputtest1', '/_arvputtest2']),
  92             self.cache_path_from_arglist(['/_arvputtest1/_arvputtest2']))
  93
  94     def test_cache_names_ignore_irrelevant_arguments(self):
  95         # Workaround: parse_arguments bails on --filename with a directory.
  96         path1 = self.cache_path_from_arglist(['/tmp'])
  97         args = arv_put.parse_arguments(['/tmp'])
  98         args.filename = 'tmp'
  99         path2 = arv_put.ResumeCache.make_path(args)
 100         self.assertEqual(path1, path2,
 101                          "cache path considered --filename for directory")
 102         self.assertEqual(
 103             self.cache_path_from_arglist(['-']),
 104             self.cache_path_from_arglist(['-', '--max-manifest-depth', '1']),
 105             "cache path considered --max-manifest-depth for file")
 106
 107     def test_cache_names_treat_negative_manifest_depths_identically(self):
 108         base_args = ['/tmp', '--max-manifest-depth']
 109         self.assertEqual(
 110             self.cache_path_from_arglist(base_args + ['-1']),
 111             self.cache_path_from_arglist(base_args + ['-2']))
 112
 113     def test_cache_names_treat_stdin_consistently(self):
 114         self.assertEqual(
 115             self.cache_path_from_arglist(['-', '--filename', 'test']),
 116             self.cache_path_from_arglist(['/dev/stdin', '--filename', 'test']))
 117
 118     def test_cache_names_identical_for_synonymous_names(self):
 119         self.assertEqual(
 120             self.cache_path_from_arglist(['.']),
 121             self.cache_path_from_arglist([os.path.realpath('.')]))
 122         testdir = self.make_tmpdir()
 123         looplink = os.path.join(testdir, 'loop')
 124         os.symlink(testdir, looplink)
 125         self.assertEqual(
 126             self.cache_path_from_arglist([testdir]),
 127             self.cache_path_from_arglist([looplink]))
 128
 129     def test_cache_names_different_by_api_host(self):
 130         config = arvados.config.settings()
 131         orig_host = config.get('ARVADOS_API_HOST')
 132         try:
 133             name1 = self.cache_path_from_arglist(['.'])
 134             config['ARVADOS_API_HOST'] = 'x' + (orig_host or 'localhost')
 135             self.assertNotEqual(name1, self.cache_path_from_arglist(['.']))
 136         finally:
 137             if orig_host is None:
 138                 del config['ARVADOS_API_HOST']
 139             else:
 140                 config['ARVADOS_API_HOST'] = orig_host
 141
 142     @mock.patch('arvados.keep.KeepClient.head')
 143     def test_resume_cache_with_current_stream_locators(self, keep_client_head):
 144         keep_client_head.side_effect = [True]
 145         thing = {}
 146         thing['_current_stream_locators'] = ['098f6bcd4621d373cade4e832627b4f6+4', '1f253c60a2306e0ee12fb6ce0c587904+6']
 147         with tempfile.NamedTemporaryFile() as cachefile:
 148             self.last_cache = arv_put.ResumeCache(cachefile.name)
 149         self.last_cache.save(thing)
 150         self.last_cache.close()
 151         resume_cache = arv_put.ResumeCache(self.last_cache.filename)
 152         self.assertNotEqual(None, resume_cache)
 153
 154     @mock.patch('arvados.keep.KeepClient.head')
 155     def test_resume_cache_with_finished_streams(self, keep_client_head):
 156         keep_client_head.side_effect = [True]
 157         thing = {}
 158         thing['_finished_streams'] = [['.', ['098f6bcd4621d373cade4e832627b4f6+4', '1f253c60a2306e0ee12fb6ce0c587904+6']]]
 159         with tempfile.NamedTemporaryFile() as cachefile:
 160             self.last_cache = arv_put.ResumeCache(cachefile.name)
 161         self.last_cache.save(thing)
 162         self.last_cache.close()
 163         resume_cache = arv_put.ResumeCache(self.last_cache.filename)
 164         self.assertNotEqual(None, resume_cache)
 165
 166     @mock.patch('arvados.keep.KeepClient.head')
 167     def test_resume_cache_with_finished_streams_error_on_head(self, keep_client_head):
 168         keep_client_head.side_effect = Exception('Locator not found')
 169         thing = {}
 170         thing['_finished_streams'] = [['.', ['098f6bcd4621d373cade4e832627b4f6+4', '1f253c60a2306e0ee12fb6ce0c587904+6']]]
 171         with tempfile.NamedTemporaryFile() as cachefile:
 172             self.last_cache = arv_put.ResumeCache(cachefile.name)
 173         self.last_cache.save(thing)
 174         self.last_cache.close()
 175         resume_cache = arv_put.ResumeCache(self.last_cache.filename)
 176         self.assertNotEqual(None, resume_cache)
 177         resume_cache.check_cache()
 178
 179     def test_basic_cache_storage(self):
 180         thing = ['test', 'list']
 181         with tempfile.NamedTemporaryFile() as cachefile:
 182             self.last_cache = arv_put.ResumeCache(cachefile.name)
 183         self.last_cache.save(thing)
 184         self.assertEqual(thing, self.last_cache.load())
 185
 186     def test_empty_cache(self):
 187         with tempfile.NamedTemporaryFile() as cachefile:
 188             cache = arv_put.ResumeCache(cachefile.name)
 189         self.assertRaises(ValueError, cache.load)
 190
 191     def test_cache_persistent(self):
 192         thing = ['test', 'list']
 193         path = os.path.join(self.make_tmpdir(), 'cache')
 194         cache = arv_put.ResumeCache(path)
 195         cache.save(thing)
 196         cache.close()
 197         self.last_cache = arv_put.ResumeCache(path)
 198         self.assertEqual(thing, self.last_cache.load())
 199
 200     def test_multiple_cache_writes(self):
 201         thing = ['short', 'list']
 202         with tempfile.NamedTemporaryFile() as cachefile:
 203             self.last_cache = arv_put.ResumeCache(cachefile.name)
 204         # Start writing an object longer than the one we test, to make
 205         # sure the cache file gets truncated.
 206         self.last_cache.save(['long', 'long', 'list'])
 207         self.last_cache.save(thing)
 208         self.assertEqual(thing, self.last_cache.load())
 209
 210     def test_cache_is_locked(self):
 211         with tempfile.NamedTemporaryFile() as cachefile:
 212             cache = arv_put.ResumeCache(cachefile.name)
 213             self.assertRaises(arv_put.ResumeCacheConflict,
 214                               arv_put.ResumeCache, cachefile.name)
 215
 216     def test_cache_stays_locked(self):
 217         with tempfile.NamedTemporaryFile() as cachefile:
 218             self.last_cache = arv_put.ResumeCache(cachefile.name)
 219             path = cachefile.name
 220         self.last_cache.save('test')
 221         self.assertRaises(arv_put.ResumeCacheConflict,
 222                           arv_put.ResumeCache, path)
 223
 224     def test_destroy_cache(self):
 225         cachefile = tempfile.NamedTemporaryFile(delete=False)
 226         try:
 227             cache = arv_put.ResumeCache(cachefile.name)
 228             cache.save('test')
 229             cache.destroy()
 230             try:
 231                 arv_put.ResumeCache(cachefile.name)
 232             except arv_put.ResumeCacheConflict:
 233                 self.fail("could not load cache after destroying it")
 234             self.assertRaises(ValueError, cache.load)
 235         finally:
 236             if os.path.exists(cachefile.name):
 237                 os.unlink(cachefile.name)
 238
 239     def test_restart_cache(self):
 240         path = os.path.join(self.make_tmpdir(), 'cache')
 241         cache = arv_put.ResumeCache(path)
 242         cache.save('test')
 243         cache.restart()
 244         self.assertRaises(ValueError, cache.load)
 245         self.assertRaises(arv_put.ResumeCacheConflict,
 246                           arv_put.ResumeCache, path)
 247
 248
 249 class ArvPutUploadJobTest(run_test_server.TestCaseWithServers,
 250                           ArvadosBaseTestCase):
 251
 252     def setUp(self):
 253         super(ArvPutUploadJobTest, self).setUp()
 254         run_test_server.authorize_with('active')
 255         # Temp files creation
 256         self.tempdir = tempfile.mkdtemp()
 257         subdir = os.path.join(self.tempdir, 'subdir')
 258         os.mkdir(subdir)
 259         data = "x" * 1024 # 1 KB
 260         for i in range(1, 5):
 261             with open(os.path.join(self.tempdir, str(i)), 'w') as f:
 262                 f.write(data * i)
 263         with open(os.path.join(subdir, 'otherfile'), 'w') as f:
 264             f.write(data * 5)
 265         # Large temp file for resume test
 266         _, self.large_file_name = tempfile.mkstemp()
 267         fileobj = open(self.large_file_name, 'w')
 268         # Make sure to write just a little more than one block
 269         for _ in range((arvados.config.KEEP_BLOCK_SIZE>>20)+1):
 270             data = random.choice(['x', 'y', 'z']) * 1024 * 1024 # 1 MiB
 271             fileobj.write(data)
 272         fileobj.close()
 273         # Temp dir containing small files to be repacked
 274         self.small_files_dir = tempfile.mkdtemp()
 275         data = 'y' * 1024 * 1024 # 1 MB
 276         for i in range(1, 70):
 277             with open(os.path.join(self.small_files_dir, str(i)), 'w') as f:
 278                 f.write(data + str(i))
 279         self.arvfile_write = getattr(arvados.arvfile.ArvadosFileWriter, 'write')
 280         # Temp dir to hold a symlink to other temp dir
 281         self.tempdir_with_symlink = tempfile.mkdtemp()
 282         os.symlink(self.tempdir, os.path.join(self.tempdir_with_symlink, 'linkeddir'))
 283         os.symlink(os.path.join(self.tempdir, '1'),
 284                    os.path.join(self.tempdir_with_symlink, 'linkedfile'))
 285
 286     def tearDown(self):
 287         super(ArvPutUploadJobTest, self).tearDown()
 288         shutil.rmtree(self.tempdir)
 289         os.unlink(self.large_file_name)
 290         shutil.rmtree(self.small_files_dir)
 291         shutil.rmtree(self.tempdir_with_symlink)
 292
 293     def test_symlinks_are_followed_by_default(self):
 294         cwriter = arv_put.ArvPutUploadJob([self.tempdir_with_symlink])
 295         cwriter.start(save_collection=False)
 296         self.assertIn('linkeddir', cwriter.manifest_text())
 297         self.assertIn('linkedfile', cwriter.manifest_text())
 298         cwriter.destroy_cache()
 299
 300     def test_symlinks_are_not_followed_when_requested(self):
 301         cwriter = arv_put.ArvPutUploadJob([self.tempdir_with_symlink],
 302                                           follow_links=False)
 303         cwriter.start(save_collection=False)
 304         self.assertNotIn('linkeddir', cwriter.manifest_text())
 305         self.assertNotIn('linkedfile', cwriter.manifest_text())
 306         cwriter.destroy_cache()
 307
 308     def test_passing_nonexistant_path_raise_exception(self):
 309         uuid_str = str(uuid.uuid4())
 310         with self.assertRaises(arv_put.PathDoesNotExistError):
 311             cwriter = arv_put.ArvPutUploadJob(["/this/path/does/not/exist/{}".format(uuid_str)])
 312
 313     def test_writer_works_without_cache(self):
 314         cwriter = arv_put.ArvPutUploadJob(['/dev/null'], resume=False)
 315         cwriter.start(save_collection=False)
 316         self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text())
 317
 318     def test_writer_works_with_cache(self):
 319         with tempfile.NamedTemporaryFile() as f:
 320             f.write(b'foo')
 321             f.flush()
 322             cwriter = arv_put.ArvPutUploadJob([f.name])
 323             cwriter.start(save_collection=False)
 324             self.assertEqual(0, cwriter.bytes_skipped)
 325             self.assertEqual(3, cwriter.bytes_written)
 326             # Don't destroy the cache, and start another upload
 327             cwriter_new = arv_put.ArvPutUploadJob([f.name])
 328             cwriter_new.start(save_collection=False)
 329             cwriter_new.destroy_cache()
 330             self.assertEqual(3, cwriter_new.bytes_skipped)
 331             self.assertEqual(3, cwriter_new.bytes_written)
 332
 333     def make_progress_tester(self):
 334         progression = []
 335         def record_func(written, expected):
 336             progression.append((written, expected))
 337         return progression, record_func
 338
 339     def test_progress_reporting(self):
 340         with tempfile.NamedTemporaryFile() as f:
 341             f.write(b'foo')
 342             f.flush()
 343             for expect_count in (None, 8):
 344                 progression, reporter = self.make_progress_tester()
 345                 cwriter = arv_put.ArvPutUploadJob([f.name],
 346                                                   reporter=reporter)
 347                 cwriter.bytes_expected = expect_count
 348                 cwriter.start(save_collection=False)
 349                 cwriter.destroy_cache()
 350                 self.assertIn((3, expect_count), progression)
 351
 352     def test_writer_upload_directory(self):
 353         cwriter = arv_put.ArvPutUploadJob([self.tempdir])
 354         cwriter.start(save_collection=False)
 355         cwriter.destroy_cache()
 356         self.assertEqual(1024*(1+2+3+4+5), cwriter.bytes_written)
 357
 358     def test_resume_large_file_upload(self):
 359         def wrapped_write(*args, **kwargs):
 360             data = args[1]
 361             # Exit only on last block
 362             if len(data) < arvados.config.KEEP_BLOCK_SIZE:
 363                 # Simulate a checkpoint before quitting. Ensure block commit.
 364                 self.writer._update(final=True)
 365                 raise SystemExit("Simulated error")
 366             return self.arvfile_write(*args, **kwargs)
 367
 368         with mock.patch('arvados.arvfile.ArvadosFileWriter.write',
 369                         autospec=True) as mocked_write:
 370             mocked_write.side_effect = wrapped_write
 371             writer = arv_put.ArvPutUploadJob([self.large_file_name],
 372                                              replication_desired=1)
 373             # We'll be accessing from inside the wrapper
 374             self.writer = writer
 375             with self.assertRaises(SystemExit):
 376                 writer.start(save_collection=False)
 377             # Confirm that the file was partially uploaded
 378             self.assertGreater(writer.bytes_written, 0)
 379             self.assertLess(writer.bytes_written,
 380                             os.path.getsize(self.large_file_name))
 381         # Retry the upload
 382         writer2 = arv_put.ArvPutUploadJob([self.large_file_name],
 383                                           replication_desired=1)
 384         writer2.start(save_collection=False)
 385         self.assertEqual(writer.bytes_written + writer2.bytes_written - writer2.bytes_skipped,
 386                          os.path.getsize(self.large_file_name))
 387         writer2.destroy_cache()
 388         del(self.writer)
 389
 390     # Test for bug #11002
 391     def test_graceful_exit_while_repacking_small_blocks(self):
 392         def wrapped_commit(*args, **kwargs):
 393             raise SystemExit("Simulated error")
 394
 395         with mock.patch('arvados.arvfile._BlockManager.commit_bufferblock',
 396                         autospec=True) as mocked_commit:
 397             mocked_commit.side_effect = wrapped_commit
 398             # Upload a little more than 1 block, wrapped_commit will make the first block
 399             # commit to fail.
 400             # arv-put should not exit with an exception by trying to commit the collection
 401             # as it's in an inconsistent state.
 402             writer = arv_put.ArvPutUploadJob([self.small_files_dir],
 403                                              replication_desired=1)
 404             try:
 405                 with self.assertRaises(SystemExit):
 406                     writer.start(save_collection=False)
 407             except arvados.arvfile.UnownedBlockError:
 408                 self.fail("arv-put command is trying to use a corrupted BlockManager. See https://dev.arvados.org/issues/11002")
 409         writer.destroy_cache()
 410
 411     def test_no_resume_when_asked(self):
 412         def wrapped_write(*args, **kwargs):
 413             data = args[1]
 414             # Exit only on last block
 415             if len(data) < arvados.config.KEEP_BLOCK_SIZE:
 416                 # Simulate a checkpoint before quitting.
 417                 self.writer._update()
 418                 raise SystemExit("Simulated error")
 419             return self.arvfile_write(*args, **kwargs)
 420
 421         with mock.patch('arvados.arvfile.ArvadosFileWriter.write',
 422                         autospec=True) as mocked_write:
 423             mocked_write.side_effect = wrapped_write
 424             writer = arv_put.ArvPutUploadJob([self.large_file_name],
 425                                              replication_desired=1)
 426             # We'll be accessing from inside the wrapper
 427             self.writer = writer
 428             with self.assertRaises(SystemExit):
 429                 writer.start(save_collection=False)
 430             # Confirm that the file was partially uploaded
 431             self.assertGreater(writer.bytes_written, 0)
 432             self.assertLess(writer.bytes_written,
 433                             os.path.getsize(self.large_file_name))
 434         # Retry the upload, this time without resume
 435         writer2 = arv_put.ArvPutUploadJob([self.large_file_name],
 436                                           replication_desired=1,
 437                                           resume=False)
 438         writer2.start(save_collection=False)
 439         self.assertEqual(writer2.bytes_skipped, 0)
 440         self.assertEqual(writer2.bytes_written,
 441                          os.path.getsize(self.large_file_name))
 442         writer2.destroy_cache()
 443         del(self.writer)
 444
 445     def test_no_resume_when_no_cache(self):
 446         def wrapped_write(*args, **kwargs):
 447             data = args[1]
 448             # Exit only on last block
 449             if len(data) < arvados.config.KEEP_BLOCK_SIZE:
 450                 # Simulate a checkpoint before quitting.
 451                 self.writer._update()
 452                 raise SystemExit("Simulated error")
 453             return self.arvfile_write(*args, **kwargs)
 454
 455         with mock.patch('arvados.arvfile.ArvadosFileWriter.write',
 456                         autospec=True) as mocked_write:
 457             mocked_write.side_effect = wrapped_write
 458             writer = arv_put.ArvPutUploadJob([self.large_file_name],
 459                                              replication_desired=1)
 460             # We'll be accessing from inside the wrapper
 461             self.writer = writer
 462             with self.assertRaises(SystemExit):
 463                 writer.start(save_collection=False)
 464             # Confirm that the file was partially uploaded
 465             self.assertGreater(writer.bytes_written, 0)
 466             self.assertLess(writer.bytes_written,
 467                             os.path.getsize(self.large_file_name))
 468         # Retry the upload, this time without cache usage
 469         writer2 = arv_put.ArvPutUploadJob([self.large_file_name],
 470                                           replication_desired=1,
 471                                           resume=False,
 472                                           use_cache=False)
 473         writer2.start(save_collection=False)
 474         self.assertEqual(writer2.bytes_skipped, 0)
 475         self.assertEqual(writer2.bytes_written,
 476                          os.path.getsize(self.large_file_name))
 477         writer2.destroy_cache()
 478         del(self.writer)
 479
 480     def test_dry_run_feature(self):
 481         def wrapped_write(*args, **kwargs):
 482             data = args[1]
 483             # Exit only on last block
 484             if len(data) < arvados.config.KEEP_BLOCK_SIZE:
 485                 # Simulate a checkpoint before quitting.
 486                 self.writer._update()
 487                 raise SystemExit("Simulated error")
 488             return self.arvfile_write(*args, **kwargs)
 489
 490         with mock.patch('arvados.arvfile.ArvadosFileWriter.write',
 491                         autospec=True) as mocked_write:
 492             mocked_write.side_effect = wrapped_write
 493             writer = arv_put.ArvPutUploadJob([self.large_file_name],
 494                                              replication_desired=1)
 495             # We'll be accessing from inside the wrapper
 496             self.writer = writer
 497             with self.assertRaises(SystemExit):
 498                 writer.start(save_collection=False)
 499             # Confirm that the file was partially uploaded
 500             self.assertGreater(writer.bytes_written, 0)
 501             self.assertLess(writer.bytes_written,
 502                             os.path.getsize(self.large_file_name))
 503         with self.assertRaises(arv_put.ArvPutUploadIsPending):
 504             # Retry the upload using dry_run to check if there is a pending upload
 505             writer2 = arv_put.ArvPutUploadJob([self.large_file_name],
 506                                               replication_desired=1,
 507                                               dry_run=True)
 508         # Complete the pending upload
 509         writer3 = arv_put.ArvPutUploadJob([self.large_file_name],
 510                                           replication_desired=1)
 511         writer3.start(save_collection=False)
 512         with self.assertRaises(arv_put.ArvPutUploadNotPending):
 513             # Confirm there's no pending upload with dry_run=True
 514             writer4 = arv_put.ArvPutUploadJob([self.large_file_name],
 515                                               replication_desired=1,
 516                                               dry_run=True)
 517         # Test obvious cases
 518         with self.assertRaises(arv_put.ArvPutUploadIsPending):
 519             arv_put.ArvPutUploadJob([self.large_file_name],
 520                                     replication_desired=1,
 521                                     dry_run=True,
 522                                     resume=False,
 523                                     use_cache=False)
 524         with self.assertRaises(arv_put.ArvPutUploadIsPending):
 525             arv_put.ArvPutUploadJob([self.large_file_name],
 526                                     replication_desired=1,
 527                                     dry_run=True,
 528                                     resume=False)
 529         del(self.writer)
 530
 531 class ArvadosExpectedBytesTest(ArvadosBaseTestCase):
 532     TEST_SIZE = os.path.getsize(__file__)
 533
 534     def test_expected_bytes_for_file(self):
 535         writer = arv_put.ArvPutUploadJob([__file__])
 536         self.assertEqual(self.TEST_SIZE,
 537                          writer.bytes_expected)
 538
 539     def test_expected_bytes_for_tree(self):
 540         tree = self.make_tmpdir()
 541         shutil.copyfile(__file__, os.path.join(tree, 'one'))
 542         shutil.copyfile(__file__, os.path.join(tree, 'two'))
 543
 544         writer = arv_put.ArvPutUploadJob([tree])
 545         self.assertEqual(self.TEST_SIZE * 2,
 546                          writer.bytes_expected)
 547         writer = arv_put.ArvPutUploadJob([tree, __file__])
 548         self.assertEqual(self.TEST_SIZE * 3,
 549                          writer.bytes_expected)
 550
 551     def test_expected_bytes_for_device(self):
 552         writer = arv_put.ArvPutUploadJob(['/dev/null'])
 553         self.assertIsNone(writer.bytes_expected)
 554         writer = arv_put.ArvPutUploadJob([__file__, '/dev/null'])
 555         self.assertIsNone(writer.bytes_expected)
 556
 557
 558 class ArvadosPutReportTest(ArvadosBaseTestCase):
 559     def test_machine_progress(self):
 560         for count, total in [(0, 1), (0, None), (1, None), (235, 9283)]:
 561             expect = ": {} written {} total\n".format(
 562                 count, -1 if (total is None) else total)
 563             self.assertTrue(
 564                 arv_put.machine_progress(count, total).endswith(expect))
 565
 566     def test_known_human_progress(self):
 567         for count, total in [(0, 1), (2, 4), (45, 60)]:
 568             expect = '{:.1%}'.format(1.0*count/total)
 569             actual = arv_put.human_progress(count, total)
 570             self.assertTrue(actual.startswith('\r'))
 571             self.assertIn(expect, actual)
 572
 573     def test_unknown_human_progress(self):
 574         for count in [1, 20, 300, 4000, 50000]:
 575             self.assertTrue(re.search(r'\b{}\b'.format(count),
 576                                       arv_put.human_progress(count, None)))
 577
 578
 579 class ArvPutLogFormatterTest(ArvadosBaseTestCase):
 580     matcher = r'\(X-Request-Id: req-[a-z0-9]{20}\)'
 581
 582     def setUp(self):
 583         super(ArvPutLogFormatterTest, self).setUp()
 584         self.stderr = tutil.StringIO()
 585         self.loggingHandler = logging.StreamHandler(self.stderr)
 586         self.loggingHandler.setFormatter(
 587             arv_put.ArvPutLogFormatter(arvados.util.new_request_id()))
 588         self.logger = logging.getLogger()
 589         self.logger.addHandler(self.loggingHandler)
 590         self.logger.setLevel(logging.DEBUG)
 591
 592     def tearDown(self):
 593         self.logger.removeHandler(self.loggingHandler)
 594         self.stderr.close()
 595         self.stderr = None
 596         super(ArvPutLogFormatterTest, self).tearDown()
 597
 598     def test_request_id_logged_only_once_on_error(self):
 599         self.logger.error('Ooops, something bad happened.')
 600         self.logger.error('Another bad thing just happened.')
 601         log_lines = self.stderr.getvalue().split('\n')[:-1]
 602         self.assertEqual(2, len(log_lines))
 603         self.assertRegex(log_lines[0], self.matcher)
 604         self.assertNotRegex(log_lines[1], self.matcher)
 605
 606     def test_request_id_logged_only_once_on_debug(self):
 607         self.logger.debug('This is just a debug message.')
 608         self.logger.debug('Another message, move along.')
 609         log_lines = self.stderr.getvalue().split('\n')[:-1]
 610         self.assertEqual(2, len(log_lines))
 611         self.assertRegex(log_lines[0], self.matcher)
 612         self.assertNotRegex(log_lines[1], self.matcher)
 613
 614     def test_request_id_not_logged_on_info(self):
 615         self.logger.info('This should be a useful message')
 616         log_lines = self.stderr.getvalue().split('\n')[:-1]
 617         self.assertEqual(1, len(log_lines))
 618         self.assertNotRegex(log_lines[0], self.matcher)
 619
 620 class ArvadosPutTest(run_test_server.TestCaseWithServers,
 621                      ArvadosBaseTestCase,
 622                      tutil.VersionChecker):
 623     MAIN_SERVER = {}
 624     Z_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
 625
 626     def call_main_with_args(self, args):
 627         self.main_stdout.seek(0, 0)
 628         self.main_stdout.truncate(0)
 629         self.main_stderr.seek(0, 0)
 630         self.main_stderr.truncate(0)
 631         return arv_put.main(args, self.main_stdout, self.main_stderr)
 632
 633     def call_main_on_test_file(self, args=[]):
 634         with self.make_test_file() as testfile:
 635             path = testfile.name
 636             self.call_main_with_args(['--stream', '--no-progress'] + args + [path])
 637         self.assertTrue(
 638             os.path.exists(os.path.join(os.environ['KEEP_LOCAL_STORE'],
 639                                         '098f6bcd4621d373cade4e832627b4f6')),
 640             "did not find file stream in Keep store")
 641
 642     def setUp(self):
 643         super(ArvadosPutTest, self).setUp()
 644         run_test_server.authorize_with('active')
 645         arv_put.api_client = None
 646         self.main_stdout = tutil.StringIO()
 647         self.main_stderr = tutil.StringIO()
 648         self.loggingHandler = logging.StreamHandler(self.main_stderr)
 649         self.loggingHandler.setFormatter(
 650             arv_put.ArvPutLogFormatter(arvados.util.new_request_id()))
 651         logging.getLogger().addHandler(self.loggingHandler)
 652
 653     def tearDown(self):
 654         logging.getLogger().removeHandler(self.loggingHandler)
 655         for outbuf in ['main_stdout', 'main_stderr']:
 656             if hasattr(self, outbuf):
 657                 getattr(self, outbuf).close()
 658                 delattr(self, outbuf)
 659         super(ArvadosPutTest, self).tearDown()
 660
 661     def test_version_argument(self):
 662         with tutil.redirected_streams(
 663                 stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err):
 664             with self.assertRaises(SystemExit):
 665                 self.call_main_with_args(['--version'])
 666         self.assertVersionOutput(out, err)
 667
 668     def test_simple_file_put(self):
 669         self.call_main_on_test_file()
 670
 671     def test_put_with_unwriteable_cache_dir(self):
 672         orig_cachedir = arv_put.ResumeCache.CACHE_DIR
 673         cachedir = self.make_tmpdir()
 674         os.chmod(cachedir, 0o0)
 675         arv_put.ResumeCache.CACHE_DIR = cachedir
 676         try:
 677             self.call_main_on_test_file()
 678         finally:
 679             arv_put.ResumeCache.CACHE_DIR = orig_cachedir
 680             os.chmod(cachedir, 0o700)
 681
 682     def test_put_with_unwritable_cache_subdir(self):
 683         orig_cachedir = arv_put.ResumeCache.CACHE_DIR
 684         cachedir = self.make_tmpdir()
 685         os.chmod(cachedir, 0o0)
 686         arv_put.ResumeCache.CACHE_DIR = os.path.join(cachedir, 'cachedir')
 687         try:
 688             self.call_main_on_test_file()
 689         finally:
 690             arv_put.ResumeCache.CACHE_DIR = orig_cachedir
 691             os.chmod(cachedir, 0o700)
 692
 693     def test_put_block_replication(self):
 694         self.call_main_on_test_file()
 695         with mock.patch('arvados.collection.KeepClient.local_store_put') as put_mock:
 696             put_mock.return_value = 'acbd18db4cc2f85cedef654fccc4a4d8+3'
 697             self.call_main_on_test_file(['--replication', '1'])
 698             self.call_main_on_test_file(['--replication', '4'])
 699             self.call_main_on_test_file(['--replication', '5'])
 700             self.assertEqual(
 701                 [x[-1].get('copies') for x in put_mock.call_args_list],
 702                 [1, 4, 5])
 703
 704     def test_normalize(self):
 705         testfile1 = self.make_test_file()
 706         testfile2 = self.make_test_file()
 707         test_paths = [testfile1.name, testfile2.name]
 708         # Reverse-sort the paths, so normalization must change their order.
 709         test_paths.sort(reverse=True)
 710         self.call_main_with_args(['--stream', '--no-progress', '--normalize'] +
 711                                  test_paths)
 712         manifest = self.main_stdout.getvalue()
 713         # Assert the second file we specified appears first in the manifest.
 714         file_indices = [manifest.find(':' + os.path.basename(path))
 715                         for path in test_paths]
 716         self.assertGreater(*file_indices)
 717
 718     def test_error_name_without_collection(self):
 719         self.assertRaises(SystemExit, self.call_main_with_args,
 720                           ['--name', 'test without Collection',
 721                            '--stream', '/dev/null'])
 722
 723     def test_error_when_project_not_found(self):
 724         self.assertRaises(SystemExit,
 725                           self.call_main_with_args,
 726                           ['--project-uuid', self.Z_UUID])
 727
 728     def test_error_bad_project_uuid(self):
 729         self.assertRaises(SystemExit,
 730                           self.call_main_with_args,
 731                           ['--project-uuid', self.Z_UUID, '--stream'])
 732
 733     def test_error_when_excluding_absolute_path(self):
 734         tmpdir = self.make_tmpdir()
 735         self.assertRaises(SystemExit,
 736                           self.call_main_with_args,
 737                           ['--exclude', '/some/absolute/path/*',
 738                            tmpdir])
 739
 740     def test_api_error_handling(self):
 741         coll_save_mock = mock.Mock(name='arv.collection.Collection().save_new()')
 742         coll_save_mock.side_effect = arvados.errors.ApiError(
 743             fake_httplib2_response(403), b'{}')
 744         with mock.patch('arvados.collection.Collection.save_new',
 745                         new=coll_save_mock):
 746             with self.assertRaises(SystemExit) as exc_test:
 747                 self.call_main_with_args(['/dev/null'])
 748             self.assertLess(0, exc_test.exception.args[0])
 749             self.assertLess(0, coll_save_mock.call_count)
 750             self.assertEqual("", self.main_stdout.getvalue())
 751
 752     def test_request_id_logging_on_error(self):
 753         matcher = r'\(X-Request-Id: req-[a-z0-9]{20}\)\n'
 754         coll_save_mock = mock.Mock(name='arv.collection.Collection().save_new()')
 755         coll_save_mock.side_effect = arvados.errors.ApiError(
 756             fake_httplib2_response(403), b'{}')
 757         with mock.patch('arvados.collection.Collection.save_new',
 758                         new=coll_save_mock):
 759             with self.assertRaises(SystemExit) as exc_test:
 760                 self.call_main_with_args(['/dev/null'])
 761             self.assertRegex(
 762                 self.main_stderr.getvalue(), matcher)
 763
 764
 765 class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
 766                             ArvadosBaseTestCase):
 767     def _getKeepServerConfig():
 768         for config_file, mandatory in [
 769                 ['application.yml', False], ['application.default.yml', True]]:
 770             path = os.path.join(run_test_server.SERVICES_SRC_DIR,
 771                                 "api", "config", config_file)
 772             if not mandatory and not os.path.exists(path):
 773                 continue
 774             with open(path) as f:
 775                 rails_config = yaml.load(f.read())
 776                 for config_section in ['test', 'common']:
 777                     try:
 778                         key = rails_config[config_section]["blob_signing_key"]
 779                     except (KeyError, TypeError):
 780                         pass
 781                     else:
 782                         return {'blob_signing_key': key,
 783                                 'enforce_permissions': True}
 784         return {'blog_signing_key': None, 'enforce_permissions': False}
 785
 786     MAIN_SERVER = {}
 787     KEEP_SERVER = _getKeepServerConfig()
 788     PROJECT_UUID = run_test_server.fixture('groups')['aproject']['uuid']
 789
 790     @classmethod
 791     def setUpClass(cls):
 792         super(ArvPutIntegrationTest, cls).setUpClass()
 793         cls.ENVIRON = os.environ.copy()
 794         cls.ENVIRON['PYTHONPATH'] = ':'.join(sys.path)
 795
 796     def datetime_to_hex(self, dt):
 797         return hex(int(time.mktime(dt.timetuple())))[2:]
 798
 799     def setUp(self):
 800         super(ArvPutIntegrationTest, self).setUp()
 801         arv_put.api_client = None
 802
 803     def authorize_with(self, token_name):
 804         run_test_server.authorize_with(token_name)
 805         for v in ["ARVADOS_API_HOST",
 806                   "ARVADOS_API_HOST_INSECURE",
 807                   "ARVADOS_API_TOKEN"]:
 808             self.ENVIRON[v] = arvados.config.settings()[v]
 809         arv_put.api_client = arvados.api('v1')
 810
 811     def current_user(self):
 812         return arv_put.api_client.users().current().execute()
 813
 814     def test_check_real_project_found(self):
 815         self.authorize_with('active')
 816         self.assertTrue(arv_put.desired_project_uuid(arv_put.api_client, self.PROJECT_UUID, 0),
 817                         "did not correctly find test fixture project")
 818
 819     def test_check_error_finding_nonexistent_uuid(self):
 820         BAD_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
 821         self.authorize_with('active')
 822         try:
 823             result = arv_put.desired_project_uuid(arv_put.api_client, BAD_UUID,
 824                                                   0)
 825         except ValueError as error:
 826             self.assertIn(BAD_UUID, str(error))
 827         else:
 828             self.assertFalse(result, "incorrectly found nonexistent project")
 829
 830     def test_check_error_finding_nonexistent_project(self):
 831         BAD_UUID = 'zzzzz-tpzed-zzzzzzzzzzzzzzz'
 832         self.authorize_with('active')
 833         with self.assertRaises(apiclient.errors.HttpError):
 834             result = arv_put.desired_project_uuid(arv_put.api_client, BAD_UUID,
 835                                                   0)
 836
 837     def test_short_put_from_stdin(self):
 838         # Have to run this as an integration test since arv-put can't
 839         # read from the tests' stdin.
 840         # arv-put usually can't stat(os.path.realpath('/dev/stdin')) in this
 841         # case, because the /proc entry is already gone by the time it tries.
 842         pipe = subprocess.Popen(
 843             [sys.executable, arv_put.__file__, '--stream'],
 844             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
 845             stderr=subprocess.STDOUT, env=self.ENVIRON)
 846         pipe.stdin.write(b'stdin test\n')
 847         pipe.stdin.close()
 848         deadline = time.time() + 5
 849         while (pipe.poll() is None) and (time.time() < deadline):
 850             time.sleep(.1)
 851         returncode = pipe.poll()
 852         if returncode is None:
 853             pipe.terminate()
 854             self.fail("arv-put did not PUT from stdin within 5 seconds")
 855         elif returncode != 0:
 856             sys.stdout.write(pipe.stdout.read())
 857             self.fail("arv-put returned exit code {}".format(returncode))
 858         self.assertIn('4a9c8b735dce4b5fa3acf221a0b13628+11',
 859                       pipe.stdout.read().decode())
 860
 861     def test_sigint_logs_request_id(self):
 862         # Connect 'yes' command output to arv-put, wait for a second and
 863         # send SIGINT to arv-put's process, then check if its output includes
 864         # the X-Request-Id.
 865         input_stream = subprocess.Popen(
 866             'yes', stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 867         pipe = subprocess.Popen(
 868             [sys.executable, arv_put.__file__, '--stream'],
 869             stdin=input_stream.stdout, stdout=subprocess.PIPE,
 870             stderr=subprocess.STDOUT, env=self.ENVIRON)
 871         time.sleep(1)
 872         pipe.send_signal(signal.SIGINT)
 873         deadline = time.time() + 5
 874         while (pipe.poll() is None) and (time.time() < deadline):
 875             time.sleep(.1)
 876         returncode = pipe.poll()
 877         input_stream.terminate()
 878         if returncode is None:
 879             pipe.terminate()
 880             self.fail("arv-put did not exited within 5 seconds")
 881         self.assertRegex(pipe.stdout.read().decode(), r'\(X-Request-Id: req-[a-z0-9]{20}\)')
 882
 883     def test_ArvPutSignedManifest(self):
 884         # ArvPutSignedManifest runs "arv-put foo" and then attempts to get
 885         # the newly created manifest from the API server, testing to confirm
 886         # that the block locators in the returned manifest are signed.
 887         self.authorize_with('active')
 888
 889         # Before doing anything, demonstrate that the collection
 890         # we're about to create is not present in our test fixture.
 891         manifest_uuid = "00b4e9f40ac4dd432ef89749f1c01e74+47"
 892         with self.assertRaises(apiclient.errors.HttpError):
 893             notfound = arv_put.api_client.collections().get(
 894                 uuid=manifest_uuid).execute()
 895
 896         datadir = self.make_tmpdir()
 897         with open(os.path.join(datadir, "foo"), "w") as f:
 898             f.write("The quick brown fox jumped over the lazy dog")
 899         p = subprocess.Popen([sys.executable, arv_put.__file__,
 900                               os.path.join(datadir, 'foo')],
 901                              stdout=subprocess.PIPE,
 902                              stderr=subprocess.PIPE,
 903                              env=self.ENVIRON)
 904         (out, err) = p.communicate()
 905         self.assertRegex(err.decode(), r'INFO: Collection saved as ')
 906         self.assertEqual(p.returncode, 0)
 907
 908         # The manifest text stored in the API server under the same
 909         # manifest UUID must use signed locators.
 910         c = arv_put.api_client.collections().get(uuid=manifest_uuid).execute()
 911         self.assertRegex(
 912             c['manifest_text'],
 913             r'^\. 08a008a01d498c404b0c30852b39d3b8\+44\+A[0-9a-f]+@[0-9a-f]+ 0:44:foo\n')
 914
 915         os.remove(os.path.join(datadir, "foo"))
 916         os.rmdir(datadir)
 917
 918     def run_and_find_collection(self, text, extra_args=[]):
 919         self.authorize_with('active')
 920         pipe = subprocess.Popen(
 921             [sys.executable, arv_put.__file__] + extra_args,
 922             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
 923             stderr=subprocess.PIPE, env=self.ENVIRON)
 924         stdout, stderr = pipe.communicate(text.encode())
 925         self.assertRegex(stderr.decode(), r'INFO: Collection (updated:|saved as)')
 926         search_key = ('portable_data_hash'
 927                       if '--portable-data-hash' in extra_args else 'uuid')
 928         collection_list = arvados.api('v1').collections().list(
 929             filters=[[search_key, '=', stdout.decode().strip()]]
 930         ).execute().get('items', [])
 931         self.assertEqual(1, len(collection_list))
 932         return collection_list[0]
 933
 934     def test_expired_token_invalidates_cache(self):
 935         self.authorize_with('active')
 936         tmpdir = self.make_tmpdir()
 937         with open(os.path.join(tmpdir, 'somefile.txt'), 'w') as f:
 938             f.write('foo')
 939         # Upload a directory and get the cache file name
 940         p = subprocess.Popen([sys.executable, arv_put.__file__, tmpdir],
 941                              stdout=subprocess.PIPE,
 942                              stderr=subprocess.PIPE,
 943                              env=self.ENVIRON)
 944         (out, err) = p.communicate()
 945         self.assertRegex(err.decode(), r'INFO: Creating new cache file at ')
 946         self.assertEqual(p.returncode, 0)
 947         cache_filepath = re.search(r'INFO: Creating new cache file at (.*)',
 948                                    err.decode()).groups()[0]
 949         self.assertTrue(os.path.isfile(cache_filepath))
 950         # Load the cache file contents and modify the manifest to simulate
 951         # an expired access token
 952         with open(cache_filepath, 'r') as c:
 953             cache = json.load(c)
 954         self.assertRegex(cache['manifest'], r'\+A\S+\@')
 955         a_month_ago = datetime.datetime.now() - datetime.timedelta(days=30)
 956         cache['manifest'] = re.sub(
 957             r'\@.*? ',
 958             "@{} ".format(self.datetime_to_hex(a_month_ago)),
 959             cache['manifest'])
 960         with open(cache_filepath, 'w') as c:
 961             c.write(json.dumps(cache))
 962         # Re-run the upload and expect to get an invalid cache message
 963         p = subprocess.Popen([sys.executable, arv_put.__file__, tmpdir],
 964                              stdout=subprocess.PIPE,
 965                              stderr=subprocess.PIPE,
 966                              env=self.ENVIRON)
 967         (out, err) = p.communicate()
 968         self.assertRegex(
 969             err.decode(),
 970             r'WARNING: Uploaded file .* access token expired, will re-upload it from scratch')
 971         self.assertEqual(p.returncode, 0)
 972         # Confirm that the resulting cache is different from the last run.
 973         with open(cache_filepath, 'r') as c2:
 974             new_cache = json.load(c2)
 975         self.assertNotEqual(cache['manifest'], new_cache['manifest'])
 976
 977     def test_put_collection_with_later_update(self):
 978         tmpdir = self.make_tmpdir()
 979         with open(os.path.join(tmpdir, 'file1'), 'w') as f:
 980             f.write('Relaxing in basins at the end of inlets terminates the endless tests from the box')
 981         col = self.run_and_find_collection("", ['--no-progress', tmpdir])
 982         self.assertNotEqual(None, col['uuid'])
 983         # Add a new file to the directory
 984         with open(os.path.join(tmpdir, 'file2'), 'w') as f:
 985             f.write('The quick brown fox jumped over the lazy dog')
 986         updated_col = self.run_and_find_collection("", ['--no-progress', '--update-collection', col['uuid'], tmpdir])
 987         self.assertEqual(col['uuid'], updated_col['uuid'])
 988         # Get the manifest and check that the new file is being included
 989         c = arv_put.api_client.collections().get(uuid=updated_col['uuid']).execute()
 990         self.assertRegex(c['manifest_text'], r'^\..* .*:44:file2\n')
 991
 992     def test_upload_directory_reference_without_trailing_slash(self):
 993         tmpdir1 = self.make_tmpdir()
 994         tmpdir2 = self.make_tmpdir()
 995         with open(os.path.join(tmpdir1, 'foo'), 'w') as f:
 996             f.write('This is foo')
 997         with open(os.path.join(tmpdir2, 'bar'), 'w') as f:
 998             f.write('This is not foo')
 999         # Upload one directory and one file
1000         col = self.run_and_find_collection("", ['--no-progress',
1001                                                 tmpdir1,
1002                                                 os.path.join(tmpdir2, 'bar')])
1003         self.assertNotEqual(None, col['uuid'])
1004         c = arv_put.api_client.collections().get(uuid=col['uuid']).execute()
1005         # Check that 'foo' was written inside a subcollection
1006         # OTOH, 'bar' should have been directly uploaded on the root collection
1007         self.assertRegex(c['manifest_text'], r'^\. .*:15:bar\n\./.+ .*:11:foo\n')
1008
1009     def test_upload_directory_reference_with_trailing_slash(self):
1010         tmpdir1 = self.make_tmpdir()
1011         tmpdir2 = self.make_tmpdir()
1012         with open(os.path.join(tmpdir1, 'foo'), 'w') as f:
1013             f.write('This is foo')
1014         with open(os.path.join(tmpdir2, 'bar'), 'w') as f:
1015             f.write('This is not foo')
1016         # Upload one directory (with trailing slash) and one file
1017         col = self.run_and_find_collection("", ['--no-progress',
1018                                                 tmpdir1 + os.sep,
1019                                                 os.path.join(tmpdir2, 'bar')])
1020         self.assertNotEqual(None, col['uuid'])
1021         c = arv_put.api_client.collections().get(uuid=col['uuid']).execute()
1022         # Check that 'foo' and 'bar' were written at the same level
1023         self.assertRegex(c['manifest_text'], r'^\. .*:15:bar .*:11:foo\n')
1024
1025     def test_put_collection_with_high_redundancy(self):
1026         # Write empty data: we're not testing CollectionWriter, just
1027         # making sure collections.create tells the API server what our
1028         # desired replication level is.
1029         collection = self.run_and_find_collection("", ['--replication', '4'])
1030         self.assertEqual(4, collection['replication_desired'])
1031
1032     def test_put_collection_with_default_redundancy(self):
1033         collection = self.run_and_find_collection("")
1034         self.assertEqual(None, collection['replication_desired'])
1035
1036     def test_put_collection_with_unnamed_project_link(self):
1037         link = self.run_and_find_collection(
1038             "Test unnamed collection",
1039             ['--portable-data-hash', '--project-uuid', self.PROJECT_UUID])
1040         username = pwd.getpwuid(os.getuid()).pw_name
1041         self.assertRegex(
1042             link['name'],
1043             r'^Saved at .* by {}@'.format(re.escape(username)))
1044
1045     def test_put_collection_with_name_and_no_project(self):
1046         link_name = 'Test Collection Link in home project'
1047         collection = self.run_and_find_collection(
1048             "Test named collection in home project",
1049             ['--portable-data-hash', '--name', link_name])
1050         self.assertEqual(link_name, collection['name'])
1051         my_user_uuid = self.current_user()['uuid']
1052         self.assertEqual(my_user_uuid, collection['owner_uuid'])
1053
1054     def test_put_collection_with_named_project_link(self):
1055         link_name = 'Test auto Collection Link'
1056         collection = self.run_and_find_collection("Test named collection",
1057                                       ['--portable-data-hash',
1058                                        '--name', link_name,
1059                                        '--project-uuid', self.PROJECT_UUID])
1060         self.assertEqual(link_name, collection['name'])
1061
1062     def test_exclude_filename_pattern(self):
1063         tmpdir = self.make_tmpdir()
1064         tmpsubdir = os.path.join(tmpdir, 'subdir')
1065         os.mkdir(tmpsubdir)
1066         for fname in ['file1', 'file2', 'file3']:
1067             with open(os.path.join(tmpdir, "%s.txt" % fname), 'w') as f:
1068                 f.write("This is %s" % fname)
1069             with open(os.path.join(tmpsubdir, "%s.txt" % fname), 'w') as f:
1070                 f.write("This is %s" % fname)
1071         col = self.run_and_find_collection("", ['--no-progress',
1072                                                 '--exclude', '*2.txt',
1073                                                 '--exclude', 'file3.*',
1074                                                  tmpdir])
1075         self.assertNotEqual(None, col['uuid'])
1076         c = arv_put.api_client.collections().get(uuid=col['uuid']).execute()
1077         # None of the file2.txt & file3.txt should have been uploaded
1078         self.assertRegex(c['manifest_text'], r'^.*:file1.txt')
1079         self.assertNotRegex(c['manifest_text'], r'^.*:file2.txt')
1080         self.assertNotRegex(c['manifest_text'], r'^.*:file3.txt')
1081
1082     def test_exclude_filepath_pattern(self):
1083         tmpdir = self.make_tmpdir()
1084         tmpsubdir = os.path.join(tmpdir, 'subdir')
1085         os.mkdir(tmpsubdir)
1086         for fname in ['file1', 'file2', 'file3']:
1087             with open(os.path.join(tmpdir, "%s.txt" % fname), 'w') as f:
1088                 f.write("This is %s" % fname)
1089             with open(os.path.join(tmpsubdir, "%s.txt" % fname), 'w') as f:
1090                 f.write("This is %s" % fname)
1091         col = self.run_and_find_collection("", ['--no-progress',
1092                                                 '--exclude', 'subdir/*2.txt',
1093                                                 '--exclude', './file1.*',
1094                                                  tmpdir])
1095         self.assertNotEqual(None, col['uuid'])
1096         c = arv_put.api_client.collections().get(uuid=col['uuid']).execute()
1097         # Only tmpdir/file1.txt & tmpdir/subdir/file2.txt should have been excluded
1098         self.assertNotRegex(c['manifest_text'],
1099                             r'^\./%s.*:file1.txt' % os.path.basename(tmpdir))
1100         self.assertNotRegex(c['manifest_text'],
1101                             r'^\./%s/subdir.*:file2.txt' % os.path.basename(tmpdir))
1102         self.assertRegex(c['manifest_text'],
1103                          r'^\./%s.*:file2.txt' % os.path.basename(tmpdir))
1104         self.assertRegex(c['manifest_text'], r'^.*:file3.txt')
1105
1106     def test_silent_mode_no_errors(self):
1107         self.authorize_with('active')
1108         tmpdir = self.make_tmpdir()
1109         with open(os.path.join(tmpdir, 'test.txt'), 'w') as f:
1110             f.write('hello world')
1111         pipe = subprocess.Popen(
1112             [sys.executable, arv_put.__file__] + ['--silent', tmpdir],
1113             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1114             stderr=subprocess.PIPE, env=self.ENVIRON)
1115         stdout, stderr = pipe.communicate()
1116         # No console output should occur on normal operations
1117         self.assertNotRegex(stderr.decode(), r'.+')
1118         self.assertNotRegex(stdout.decode(), r'.+')
1119
1120     def test_silent_mode_does_not_avoid_error_messages(self):
1121         self.authorize_with('active')
1122         pipe = subprocess.Popen(
1123             [sys.executable, arv_put.__file__] + ['--silent',
1124                                                   '/path/not/existant'],
1125             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1126             stderr=subprocess.PIPE, env=self.ENVIRON)
1127         stdout, stderr = pipe.communicate()
1128         # Error message should be displayed when errors happen
1129         self.assertRegex(stderr.decode(), r'.*ERROR:.*')
1130         self.assertNotRegex(stdout.decode(), r'.+')
1131
1132
1133 if __name__ == '__main__':
1134     unittest.main()