sdk/python/tests/test_arv_put.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 import apiclient
   5 import mock
   6 import os
   7 import pwd
   8 import re
   9 import shutil
  10 import subprocess
  11 import sys
  12 import tempfile
  13 import time
  14 import unittest
  15 import yaml
  16
  17 from cStringIO import StringIO
  18
  19 import arvados
  20 import arvados.commands.put as arv_put
  21
  22 from arvados_testutil import ArvadosBaseTestCase
  23 import run_test_server
  24
  25 class ArvadosPutResumeCacheTest(ArvadosBaseTestCase):
  26     CACHE_ARGSET = [
  27         [],
  28         ['/dev/null'],
  29         ['/dev/null', '--filename', 'empty'],
  30         ['/tmp'],
  31         ['/tmp', '--max-manifest-depth', '0'],
  32         ['/tmp', '--max-manifest-depth', '1']
  33         ]
  34
  35     def tearDown(self):
  36         super(ArvadosPutResumeCacheTest, self).tearDown()
  37         try:
  38             self.last_cache.destroy()
  39         except AttributeError:
  40             pass
  41
  42     def cache_path_from_arglist(self, arglist):
  43         return arv_put.ResumeCache.make_path(arv_put.parse_arguments(arglist))
  44
  45     def test_cache_names_stable(self):
  46         for argset in self.CACHE_ARGSET:
  47             self.assertEqual(self.cache_path_from_arglist(argset),
  48                               self.cache_path_from_arglist(argset),
  49                               "cache name changed for {}".format(argset))
  50
  51     def test_cache_names_unique(self):
  52         results = []
  53         for argset in self.CACHE_ARGSET:
  54             path = self.cache_path_from_arglist(argset)
  55             self.assertNotIn(path, results)
  56             results.append(path)
  57
  58     def test_cache_names_simple(self):
  59         # The goal here is to make sure the filename doesn't use characters
  60         # reserved by the filesystem.  Feel free to adjust this regexp as
  61         # long as it still does that.
  62         bad_chars = re.compile(r'[^-\.\w]')
  63         for argset in self.CACHE_ARGSET:
  64             path = self.cache_path_from_arglist(argset)
  65             self.assertFalse(bad_chars.search(os.path.basename(path)),
  66                              "path too exotic: {}".format(path))
  67
  68     def test_cache_names_ignore_argument_order(self):
  69         self.assertEqual(
  70             self.cache_path_from_arglist(['a', 'b', 'c']),
  71             self.cache_path_from_arglist(['c', 'a', 'b']))
  72         self.assertEqual(
  73             self.cache_path_from_arglist(['-', '--filename', 'stdin']),
  74             self.cache_path_from_arglist(['--filename', 'stdin', '-']))
  75
  76     def test_cache_names_differ_for_similar_paths(self):
  77         # This test needs names at / that don't exist on the real filesystem.
  78         self.assertNotEqual(
  79             self.cache_path_from_arglist(['/_arvputtest1', '/_arvputtest2']),
  80             self.cache_path_from_arglist(['/_arvputtest1/_arvputtest2']))
  81
  82     def test_cache_names_ignore_irrelevant_arguments(self):
  83         # Workaround: parse_arguments bails on --filename with a directory.
  84         path1 = self.cache_path_from_arglist(['/tmp'])
  85         args = arv_put.parse_arguments(['/tmp'])
  86         args.filename = 'tmp'
  87         path2 = arv_put.ResumeCache.make_path(args)
  88         self.assertEqual(path1, path2,
  89                          "cache path considered --filename for directory")
  90         self.assertEqual(
  91             self.cache_path_from_arglist(['-']),
  92             self.cache_path_from_arglist(['-', '--max-manifest-depth', '1']),
  93             "cache path considered --max-manifest-depth for file")
  94
  95     def test_cache_names_treat_negative_manifest_depths_identically(self):
  96         base_args = ['/tmp', '--max-manifest-depth']
  97         self.assertEqual(
  98             self.cache_path_from_arglist(base_args + ['-1']),
  99             self.cache_path_from_arglist(base_args + ['-2']))
 100
 101     def test_cache_names_treat_stdin_consistently(self):
 102         self.assertEqual(
 103             self.cache_path_from_arglist(['-', '--filename', 'test']),
 104             self.cache_path_from_arglist(['/dev/stdin', '--filename', 'test']))
 105
 106     def test_cache_names_identical_for_synonymous_names(self):
 107         self.assertEqual(
 108             self.cache_path_from_arglist(['.']),
 109             self.cache_path_from_arglist([os.path.realpath('.')]))
 110         testdir = self.make_tmpdir()
 111         looplink = os.path.join(testdir, 'loop')
 112         os.symlink(testdir, looplink)
 113         self.assertEqual(
 114             self.cache_path_from_arglist([testdir]),
 115             self.cache_path_from_arglist([looplink]))
 116
 117     def test_cache_names_different_by_api_host(self):
 118         config = arvados.config.settings()
 119         orig_host = config.get('ARVADOS_API_HOST')
 120         try:
 121             name1 = self.cache_path_from_arglist(['.'])
 122             config['ARVADOS_API_HOST'] = 'x' + (orig_host or 'localhost')
 123             self.assertNotEqual(name1, self.cache_path_from_arglist(['.']))
 124         finally:
 125             if orig_host is None:
 126                 del config['ARVADOS_API_HOST']
 127             else:
 128                 config['ARVADOS_API_HOST'] = orig_host
 129
 130     @mock.patch('arvados.keep.KeepClient.head')
 131     def test_resume_cache_with_current_stream_locators(self, keep_client_head):
 132         keep_client_head.side_effect = [True]
 133         thing = {}
 134         thing['_current_stream_locators'] = ['098f6bcd4621d373cade4e832627b4f6+4', '1f253c60a2306e0ee12fb6ce0c587904+6']
 135         with tempfile.NamedTemporaryFile() as cachefile:
 136             self.last_cache = arv_put.ResumeCache(cachefile.name)
 137         self.last_cache.save(thing)
 138         self.last_cache.close()
 139         resume_cache = arv_put.ResumeCache(self.last_cache.filename)
 140         self.assertNotEqual(None, resume_cache)
 141
 142     @mock.patch('arvados.keep.KeepClient.head')
 143     def test_resume_cache_with_finished_streams(self, keep_client_head):
 144         keep_client_head.side_effect = [True]
 145         thing = {}
 146         thing['_finished_streams'] = [['.', ['098f6bcd4621d373cade4e832627b4f6+4', '1f253c60a2306e0ee12fb6ce0c587904+6']]]
 147         with tempfile.NamedTemporaryFile() as cachefile:
 148             self.last_cache = arv_put.ResumeCache(cachefile.name)
 149         self.last_cache.save(thing)
 150         self.last_cache.close()
 151         resume_cache = arv_put.ResumeCache(self.last_cache.filename)
 152         self.assertNotEqual(None, resume_cache)
 153
 154     @mock.patch('arvados.keep.KeepClient.head')
 155     def test_resume_cache_with_finished_streams_error_on_head(self, keep_client_head):
 156         keep_client_head.side_effect = Exception('Locator not found')
 157         thing = {}
 158         thing['_finished_streams'] = [['.', ['098f6bcd4621d373cade4e832627b4f6+4', '1f253c60a2306e0ee12fb6ce0c587904+6']]]
 159         with tempfile.NamedTemporaryFile() as cachefile:
 160             self.last_cache = arv_put.ResumeCache(cachefile.name)
 161         self.last_cache.save(thing)
 162         self.last_cache.close()
 163         with self.assertRaises(arvados.errors.KeepRequestError):
 164             arv_put.ResumeCache(self.last_cache.filename)
 165
 166     def test_basic_cache_storage(self):
 167         thing = ['test', 'list']
 168         with tempfile.NamedTemporaryFile() as cachefile:
 169             self.last_cache = arv_put.ResumeCache(cachefile.name)
 170         self.last_cache.save(thing)
 171         self.assertEqual(thing, self.last_cache.load())
 172
 173     def test_empty_cache(self):
 174         with tempfile.NamedTemporaryFile() as cachefile:
 175             cache = arv_put.ResumeCache(cachefile.name)
 176         self.assertRaises(ValueError, cache.load)
 177
 178     def test_cache_persistent(self):
 179         thing = ['test', 'list']
 180         path = os.path.join(self.make_tmpdir(), 'cache')
 181         cache = arv_put.ResumeCache(path)
 182         cache.save(thing)
 183         cache.close()
 184         self.last_cache = arv_put.ResumeCache(path)
 185         self.assertEqual(thing, self.last_cache.load())
 186
 187     def test_multiple_cache_writes(self):
 188         thing = ['short', 'list']
 189         with tempfile.NamedTemporaryFile() as cachefile:
 190             self.last_cache = arv_put.ResumeCache(cachefile.name)
 191         # Start writing an object longer than the one we test, to make
 192         # sure the cache file gets truncated.
 193         self.last_cache.save(['long', 'long', 'list'])
 194         self.last_cache.save(thing)
 195         self.assertEqual(thing, self.last_cache.load())
 196
 197     def test_cache_is_locked(self):
 198         with tempfile.NamedTemporaryFile() as cachefile:
 199             cache = arv_put.ResumeCache(cachefile.name)
 200             self.assertRaises(arv_put.ResumeCacheConflict,
 201                               arv_put.ResumeCache, cachefile.name)
 202
 203     def test_cache_stays_locked(self):
 204         with tempfile.NamedTemporaryFile() as cachefile:
 205             self.last_cache = arv_put.ResumeCache(cachefile.name)
 206             path = cachefile.name
 207         self.last_cache.save('test')
 208         self.assertRaises(arv_put.ResumeCacheConflict,
 209                           arv_put.ResumeCache, path)
 210
 211     def test_destroy_cache(self):
 212         cachefile = tempfile.NamedTemporaryFile(delete=False)
 213         try:
 214             cache = arv_put.ResumeCache(cachefile.name)
 215             cache.save('test')
 216             cache.destroy()
 217             try:
 218                 arv_put.ResumeCache(cachefile.name)
 219             except arv_put.ResumeCacheConflict:
 220                 self.fail("could not load cache after destroying it")
 221             self.assertRaises(ValueError, cache.load)
 222         finally:
 223             if os.path.exists(cachefile.name):
 224                 os.unlink(cachefile.name)
 225
 226     def test_restart_cache(self):
 227         path = os.path.join(self.make_tmpdir(), 'cache')
 228         cache = arv_put.ResumeCache(path)
 229         cache.save('test')
 230         cache.restart()
 231         self.assertRaises(ValueError, cache.load)
 232         self.assertRaises(arv_put.ResumeCacheConflict,
 233                           arv_put.ResumeCache, path)
 234
 235
 236 class ArvadosPutCollectionWriterTest(run_test_server.TestCaseWithServers,
 237                                      ArvadosBaseTestCase):
 238     def setUp(self):
 239         super(ArvadosPutCollectionWriterTest, self).setUp()
 240         run_test_server.authorize_with('active')
 241         with tempfile.NamedTemporaryFile(delete=False) as cachefile:
 242             self.cache = arv_put.ResumeCache(cachefile.name)
 243             self.cache_filename = cachefile.name
 244
 245     def tearDown(self):
 246         super(ArvadosPutCollectionWriterTest, self).tearDown()
 247         if os.path.exists(self.cache_filename):
 248             self.cache.destroy()
 249         self.cache.close()
 250
 251     def test_writer_caches(self):
 252         cwriter = arv_put.ArvPutCollectionWriter(self.cache)
 253         cwriter.write_file('/dev/null')
 254         cwriter.cache_state()
 255         self.assertTrue(self.cache.load())
 256         self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text())
 257
 258     def test_writer_works_without_cache(self):
 259         cwriter = arv_put.ArvPutCollectionWriter()
 260         cwriter.write_file('/dev/null')
 261         self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text())
 262
 263     def test_writer_resumes_from_cache(self):
 264         cwriter = arv_put.ArvPutCollectionWriter(self.cache)
 265         with self.make_test_file() as testfile:
 266             cwriter.write_file(testfile.name, 'test')
 267             cwriter.cache_state()
 268             new_writer = arv_put.ArvPutCollectionWriter.from_cache(
 269                 self.cache)
 270             self.assertEqual(
 271                 ". 098f6bcd4621d373cade4e832627b4f6+4 0:4:test\n",
 272                 new_writer.manifest_text())
 273
 274     def test_new_writer_from_stale_cache(self):
 275         cwriter = arv_put.ArvPutCollectionWriter(self.cache)
 276         with self.make_test_file() as testfile:
 277             cwriter.write_file(testfile.name, 'test')
 278         new_writer = arv_put.ArvPutCollectionWriter.from_cache(self.cache)
 279         new_writer.write_file('/dev/null')
 280         self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", new_writer.manifest_text())
 281
 282     def test_new_writer_from_empty_cache(self):
 283         cwriter = arv_put.ArvPutCollectionWriter.from_cache(self.cache)
 284         cwriter.write_file('/dev/null')
 285         self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text())
 286
 287     def test_writer_resumable_after_arbitrary_bytes(self):
 288         cwriter = arv_put.ArvPutCollectionWriter(self.cache)
 289         # These bytes are intentionally not valid UTF-8.
 290         with self.make_test_file('\x00\x07\xe2') as testfile:
 291             cwriter.write_file(testfile.name, 'test')
 292             cwriter.cache_state()
 293             new_writer = arv_put.ArvPutCollectionWriter.from_cache(
 294                 self.cache)
 295         self.assertEqual(cwriter.manifest_text(), new_writer.manifest_text())
 296
 297     def make_progress_tester(self):
 298         progression = []
 299         def record_func(written, expected):
 300             progression.append((written, expected))
 301         return progression, record_func
 302
 303     def test_progress_reporting(self):
 304         for expect_count in (None, 8):
 305             progression, reporter = self.make_progress_tester()
 306             cwriter = arv_put.ArvPutCollectionWriter(
 307                 reporter=reporter, bytes_expected=expect_count)
 308             with self.make_test_file() as testfile:
 309                 cwriter.write_file(testfile.name, 'test')
 310             cwriter.finish_current_stream()
 311             self.assertIn((4, expect_count), progression)
 312
 313     def test_resume_progress(self):
 314         cwriter = arv_put.ArvPutCollectionWriter(self.cache, bytes_expected=4)
 315         with self.make_test_file() as testfile:
 316             # Set up a writer with some flushed bytes.
 317             cwriter.write_file(testfile.name, 'test')
 318             cwriter.finish_current_stream()
 319             cwriter.cache_state()
 320             new_writer = arv_put.ArvPutCollectionWriter.from_cache(self.cache)
 321             self.assertEqual(new_writer.bytes_written, 4)
 322
 323
 324 class ArvadosExpectedBytesTest(ArvadosBaseTestCase):
 325     TEST_SIZE = os.path.getsize(__file__)
 326
 327     def test_expected_bytes_for_file(self):
 328         self.assertEqual(self.TEST_SIZE,
 329                           arv_put.expected_bytes_for([__file__]))
 330
 331     def test_expected_bytes_for_tree(self):
 332         tree = self.make_tmpdir()
 333         shutil.copyfile(__file__, os.path.join(tree, 'one'))
 334         shutil.copyfile(__file__, os.path.join(tree, 'two'))
 335         self.assertEqual(self.TEST_SIZE * 2,
 336                           arv_put.expected_bytes_for([tree]))
 337         self.assertEqual(self.TEST_SIZE * 3,
 338                           arv_put.expected_bytes_for([tree, __file__]))
 339
 340     def test_expected_bytes_for_device(self):
 341         self.assertIsNone(arv_put.expected_bytes_for(['/dev/null']))
 342         self.assertIsNone(arv_put.expected_bytes_for([__file__, '/dev/null']))
 343
 344
 345 class ArvadosPutReportTest(ArvadosBaseTestCase):
 346     def test_machine_progress(self):
 347         for count, total in [(0, 1), (0, None), (1, None), (235, 9283)]:
 348             expect = ": {} written {} total\n".format(
 349                 count, -1 if (total is None) else total)
 350             self.assertTrue(
 351                 arv_put.machine_progress(count, total).endswith(expect))
 352
 353     def test_known_human_progress(self):
 354         for count, total in [(0, 1), (2, 4), (45, 60)]:
 355             expect = '{:.1%}'.format(float(count) / total)
 356             actual = arv_put.human_progress(count, total)
 357             self.assertTrue(actual.startswith('\r'))
 358             self.assertIn(expect, actual)
 359
 360     def test_unknown_human_progress(self):
 361         for count in [1, 20, 300, 4000, 50000]:
 362             self.assertTrue(re.search(r'\b{}\b'.format(count),
 363                                       arv_put.human_progress(count, None)))
 364
 365
 366 class ArvadosPutTest(run_test_server.TestCaseWithServers, ArvadosBaseTestCase):
 367     MAIN_SERVER = {}
 368     Z_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
 369
 370     def call_main_with_args(self, args):
 371         self.main_stdout = StringIO()
 372         self.main_stderr = StringIO()
 373         return arv_put.main(args, self.main_stdout, self.main_stderr)
 374
 375     def call_main_on_test_file(self, args=[]):
 376         with self.make_test_file() as testfile:
 377             path = testfile.name
 378             self.call_main_with_args(['--stream', '--no-progress'] + args + [path])
 379         self.assertTrue(
 380             os.path.exists(os.path.join(os.environ['KEEP_LOCAL_STORE'],
 381                                         '098f6bcd4621d373cade4e832627b4f6')),
 382             "did not find file stream in Keep store")
 383
 384     def setUp(self):
 385         super(ArvadosPutTest, self).setUp()
 386         run_test_server.authorize_with('active')
 387         arv_put.api_client = None
 388
 389     def tearDown(self):
 390         for outbuf in ['main_stdout', 'main_stderr']:
 391             if hasattr(self, outbuf):
 392                 getattr(self, outbuf).close()
 393                 delattr(self, outbuf)
 394         super(ArvadosPutTest, self).tearDown()
 395
 396     def test_simple_file_put(self):
 397         self.call_main_on_test_file()
 398
 399     def test_put_with_unwriteable_cache_dir(self):
 400         orig_cachedir = arv_put.ResumeCache.CACHE_DIR
 401         cachedir = self.make_tmpdir()
 402         os.chmod(cachedir, 0o0)
 403         arv_put.ResumeCache.CACHE_DIR = cachedir
 404         try:
 405             self.call_main_on_test_file()
 406         finally:
 407             arv_put.ResumeCache.CACHE_DIR = orig_cachedir
 408             os.chmod(cachedir, 0o700)
 409
 410     def test_put_with_unwritable_cache_subdir(self):
 411         orig_cachedir = arv_put.ResumeCache.CACHE_DIR
 412         cachedir = self.make_tmpdir()
 413         os.chmod(cachedir, 0o0)
 414         arv_put.ResumeCache.CACHE_DIR = os.path.join(cachedir, 'cachedir')
 415         try:
 416             self.call_main_on_test_file()
 417         finally:
 418             arv_put.ResumeCache.CACHE_DIR = orig_cachedir
 419             os.chmod(cachedir, 0o700)
 420
 421     def test_put_block_replication(self):
 422         with mock.patch('arvados.collection.KeepClient.local_store_put') as put_mock, \
 423              mock.patch('arvados.commands.put.ResumeCache.load') as cache_mock:
 424             cache_mock.side_effect = ValueError
 425             put_mock.return_value = 'acbd18db4cc2f85cedef654fccc4a4d8+3'
 426             self.call_main_on_test_file(['--replication', '1'])
 427             self.call_main_on_test_file(['--replication', '4'])
 428             self.call_main_on_test_file(['--replication', '5'])
 429             self.assertEqual(
 430                 [x[-1].get('copies') for x in put_mock.call_args_list],
 431                 [1, 4, 5])
 432
 433     def test_normalize(self):
 434         testfile1 = self.make_test_file()
 435         testfile2 = self.make_test_file()
 436         test_paths = [testfile1.name, testfile2.name]
 437         # Reverse-sort the paths, so normalization must change their order.
 438         test_paths.sort(reverse=True)
 439         self.call_main_with_args(['--stream', '--no-progress', '--normalize'] +
 440                                  test_paths)
 441         manifest = self.main_stdout.getvalue()
 442         # Assert the second file we specified appears first in the manifest.
 443         file_indices = [manifest.find(':' + os.path.basename(path))
 444                         for path in test_paths]
 445         self.assertGreater(*file_indices)
 446
 447     def test_error_name_without_collection(self):
 448         self.assertRaises(SystemExit, self.call_main_with_args,
 449                           ['--name', 'test without Collection',
 450                            '--stream', '/dev/null'])
 451
 452     def test_error_when_project_not_found(self):
 453         self.assertRaises(SystemExit,
 454                           self.call_main_with_args,
 455                           ['--project-uuid', self.Z_UUID])
 456
 457     def test_error_bad_project_uuid(self):
 458         self.assertRaises(SystemExit,
 459                           self.call_main_with_args,
 460                           ['--project-uuid', self.Z_UUID, '--stream'])
 461
 462 class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
 463                             ArvadosBaseTestCase):
 464     def _getKeepServerConfig():
 465         for config_file, mandatory in [
 466                 ['application.yml', False], ['application.default.yml', True]]:
 467             path = os.path.join(run_test_server.SERVICES_SRC_DIR,
 468                                 "api", "config", config_file)
 469             if not mandatory and not os.path.exists(path):
 470                 continue
 471             with open(path) as f:
 472                 rails_config = yaml.load(f.read())
 473                 for config_section in ['test', 'common']:
 474                     try:
 475                         key = rails_config[config_section]["blob_signing_key"]
 476                     except (KeyError, TypeError):
 477                         pass
 478                     else:
 479                         return {'blob_signing_key': key,
 480                                 'enforce_permissions': True}
 481         return {'blog_signing_key': None, 'enforce_permissions': False}
 482
 483     MAIN_SERVER = {}
 484     KEEP_SERVER = _getKeepServerConfig()
 485     PROJECT_UUID = run_test_server.fixture('groups')['aproject']['uuid']
 486
 487     @classmethod
 488     def setUpClass(cls):
 489         super(ArvPutIntegrationTest, cls).setUpClass()
 490         cls.ENVIRON = os.environ.copy()
 491         cls.ENVIRON['PYTHONPATH'] = ':'.join(sys.path)
 492
 493     def setUp(self):
 494         super(ArvPutIntegrationTest, self).setUp()
 495         arv_put.api_client = None
 496
 497     def authorize_with(self, token_name):
 498         run_test_server.authorize_with(token_name)
 499         for v in ["ARVADOS_API_HOST",
 500                   "ARVADOS_API_HOST_INSECURE",
 501                   "ARVADOS_API_TOKEN"]:
 502             self.ENVIRON[v] = arvados.config.settings()[v]
 503         arv_put.api_client = arvados.api('v1')
 504
 505     def current_user(self):
 506         return arv_put.api_client.users().current().execute()
 507
 508     def test_check_real_project_found(self):
 509         self.authorize_with('active')
 510         self.assertTrue(arv_put.desired_project_uuid(arv_put.api_client, self.PROJECT_UUID, 0),
 511                         "did not correctly find test fixture project")
 512
 513     def test_check_error_finding_nonexistent_uuid(self):
 514         BAD_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
 515         self.authorize_with('active')
 516         try:
 517             result = arv_put.desired_project_uuid(arv_put.api_client, BAD_UUID,
 518                                                   0)
 519         except ValueError as error:
 520             self.assertIn(BAD_UUID, error.message)
 521         else:
 522             self.assertFalse(result, "incorrectly found nonexistent project")
 523
 524     def test_check_error_finding_nonexistent_project(self):
 525         BAD_UUID = 'zzzzz-tpzed-zzzzzzzzzzzzzzz'
 526         self.authorize_with('active')
 527         with self.assertRaises(apiclient.errors.HttpError):
 528             result = arv_put.desired_project_uuid(arv_put.api_client, BAD_UUID,
 529                                                   0)
 530
 531     def test_short_put_from_stdin(self):
 532         # Have to run this as an integration test since arv-put can't
 533         # read from the tests' stdin.
 534         # arv-put usually can't stat(os.path.realpath('/dev/stdin')) in this
 535         # case, because the /proc entry is already gone by the time it tries.
 536         pipe = subprocess.Popen(
 537             [sys.executable, arv_put.__file__, '--stream'],
 538             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
 539             stderr=subprocess.STDOUT, env=self.ENVIRON)
 540         pipe.stdin.write('stdin test\n')
 541         pipe.stdin.close()
 542         deadline = time.time() + 5
 543         while (pipe.poll() is None) and (time.time() < deadline):
 544             time.sleep(.1)
 545         returncode = pipe.poll()
 546         if returncode is None:
 547             pipe.terminate()
 548             self.fail("arv-put did not PUT from stdin within 5 seconds")
 549         elif returncode != 0:
 550             sys.stdout.write(pipe.stdout.read())
 551             self.fail("arv-put returned exit code {}".format(returncode))
 552         self.assertIn('4a9c8b735dce4b5fa3acf221a0b13628+11', pipe.stdout.read())
 553
 554     def test_ArvPutSignedManifest(self):
 555         # ArvPutSignedManifest runs "arv-put foo" and then attempts to get
 556         # the newly created manifest from the API server, testing to confirm
 557         # that the block locators in the returned manifest are signed.
 558         self.authorize_with('active')
 559
 560         # Before doing anything, demonstrate that the collection
 561         # we're about to create is not present in our test fixture.
 562         manifest_uuid = "00b4e9f40ac4dd432ef89749f1c01e74+47"
 563         with self.assertRaises(apiclient.errors.HttpError):
 564             notfound = arv_put.api_client.collections().get(
 565                 uuid=manifest_uuid).execute()
 566
 567         datadir = self.make_tmpdir()
 568         with open(os.path.join(datadir, "foo"), "w") as f:
 569             f.write("The quick brown fox jumped over the lazy dog")
 570         p = subprocess.Popen([sys.executable, arv_put.__file__, datadir],
 571                              stdout=subprocess.PIPE, env=self.ENVIRON)
 572         (arvout, arverr) = p.communicate()
 573         self.assertEqual(arverr, None)
 574         self.assertEqual(p.returncode, 0)
 575
 576         # The manifest text stored in the API server under the same
 577         # manifest UUID must use signed locators.
 578         c = arv_put.api_client.collections().get(uuid=manifest_uuid).execute()
 579         self.assertRegexpMatches(
 580             c['manifest_text'],
 581             r'^\. 08a008a01d498c404b0c30852b39d3b8\+44\+A[0-9a-f]+@[0-9a-f]+ 0:44:foo\n')
 582
 583         os.remove(os.path.join(datadir, "foo"))
 584         os.rmdir(datadir)
 585
 586     def run_and_find_collection(self, text, extra_args=[]):
 587         self.authorize_with('active')
 588         pipe = subprocess.Popen(
 589             [sys.executable, arv_put.__file__] + extra_args,
 590             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
 591             stderr=subprocess.PIPE, env=self.ENVIRON)
 592         stdout, stderr = pipe.communicate(text)
 593         search_key = ('portable_data_hash'
 594                       if '--portable-data-hash' in extra_args else 'uuid')
 595         collection_list = arvados.api('v1').collections().list(
 596             filters=[[search_key, '=', stdout.strip()]]).execute().get('items', [])
 597         self.assertEqual(1, len(collection_list))
 598         return collection_list[0]
 599
 600     def test_put_collection_with_high_redundancy(self):
 601         # Write empty data: we're not testing CollectionWriter, just
 602         # making sure collections.create tells the API server what our
 603         # desired replication level is.
 604         collection = self.run_and_find_collection("", ['--replication', '4'])
 605         self.assertEqual(4, collection['replication_desired'])
 606
 607     def test_put_collection_with_default_redundancy(self):
 608         collection = self.run_and_find_collection("")
 609         self.assertEqual(None, collection['replication_desired'])
 610
 611     def test_put_collection_with_unnamed_project_link(self):
 612         link = self.run_and_find_collection(
 613             "Test unnamed collection",
 614             ['--portable-data-hash', '--project-uuid', self.PROJECT_UUID])
 615         username = pwd.getpwuid(os.getuid()).pw_name
 616         self.assertRegexpMatches(
 617             link['name'],
 618             r'^Saved at .* by {}@'.format(re.escape(username)))
 619
 620     def test_put_collection_with_name_and_no_project(self):
 621         link_name = 'Test Collection Link in home project'
 622         collection = self.run_and_find_collection(
 623             "Test named collection in home project",
 624             ['--portable-data-hash', '--name', link_name])
 625         self.assertEqual(link_name, collection['name'])
 626         my_user_uuid = self.current_user()['uuid']
 627         self.assertEqual(my_user_uuid, collection['owner_uuid'])
 628
 629     def test_put_collection_with_named_project_link(self):
 630         link_name = 'Test auto Collection Link'
 631         collection = self.run_and_find_collection("Test named collection",
 632                                       ['--portable-data-hash',
 633                                        '--name', link_name,
 634                                        '--project-uuid', self.PROJECT_UUID])
 635         self.assertEqual(link_name, collection['name'])
 636
 637
 638 if __name__ == '__main__':
 639     unittest.main()