sdk/python/tests/test_arv_put.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 import apiclient
   5 import mock
   6 import os
   7 import pwd
   8 import re
   9 import shutil
  10 import subprocess
  11 import sys
  12 import tempfile
  13 import time
  14 import unittest
  15 import yaml
  16
  17 from cStringIO import StringIO
  18
  19 import arvados
  20 import arvados.commands.put as arv_put
  21
  22 from arvados_testutil import ArvadosBaseTestCase
  23 import run_test_server
  24
  25 class ArvadosPutResumeCacheTest(ArvadosBaseTestCase):
  26     CACHE_ARGSET = [
  27         [],
  28         ['/dev/null'],
  29         ['/dev/null', '--filename', 'empty'],
  30         ['/tmp'],
  31         ['/tmp', '--max-manifest-depth', '0'],
  32         ['/tmp', '--max-manifest-depth', '1']
  33         ]
  34
  35     def tearDown(self):
  36         super(ArvadosPutResumeCacheTest, self).tearDown()
  37         try:
  38             self.last_cache.destroy()
  39         except AttributeError:
  40             pass
  41
  42     def cache_path_from_arglist(self, arglist):
  43         return arv_put.ResumeCache.make_path(arv_put.parse_arguments(arglist))
  44
  45     def test_cache_names_stable(self):
  46         for argset in self.CACHE_ARGSET:
  47             self.assertEqual(self.cache_path_from_arglist(argset),
  48                               self.cache_path_from_arglist(argset),
  49                               "cache name changed for {}".format(argset))
  50
  51     def test_cache_names_unique(self):
  52         results = []
  53         for argset in self.CACHE_ARGSET:
  54             path = self.cache_path_from_arglist(argset)
  55             self.assertNotIn(path, results)
  56             results.append(path)
  57
  58     def test_cache_names_simple(self):
  59         # The goal here is to make sure the filename doesn't use characters
  60         # reserved by the filesystem.  Feel free to adjust this regexp as
  61         # long as it still does that.
  62         bad_chars = re.compile(r'[^-\.\w]')
  63         for argset in self.CACHE_ARGSET:
  64             path = self.cache_path_from_arglist(argset)
  65             self.assertFalse(bad_chars.search(os.path.basename(path)),
  66                              "path too exotic: {}".format(path))
  67
  68     def test_cache_names_ignore_argument_order(self):
  69         self.assertEqual(
  70             self.cache_path_from_arglist(['a', 'b', 'c']),
  71             self.cache_path_from_arglist(['c', 'a', 'b']))
  72         self.assertEqual(
  73             self.cache_path_from_arglist(['-', '--filename', 'stdin']),
  74             self.cache_path_from_arglist(['--filename', 'stdin', '-']))
  75
  76     def test_cache_names_differ_for_similar_paths(self):
  77         # This test needs names at / that don't exist on the real filesystem.
  78         self.assertNotEqual(
  79             self.cache_path_from_arglist(['/_arvputtest1', '/_arvputtest2']),
  80             self.cache_path_from_arglist(['/_arvputtest1/_arvputtest2']))
  81
  82     def test_cache_names_ignore_irrelevant_arguments(self):
  83         # Workaround: parse_arguments bails on --filename with a directory.
  84         path1 = self.cache_path_from_arglist(['/tmp'])
  85         args = arv_put.parse_arguments(['/tmp'])
  86         args.filename = 'tmp'
  87         path2 = arv_put.ResumeCache.make_path(args)
  88         self.assertEqual(path1, path2,
  89                          "cache path considered --filename for directory")
  90         self.assertEqual(
  91             self.cache_path_from_arglist(['-']),
  92             self.cache_path_from_arglist(['-', '--max-manifest-depth', '1']),
  93             "cache path considered --max-manifest-depth for file")
  94
  95     def test_cache_names_treat_negative_manifest_depths_identically(self):
  96         base_args = ['/tmp', '--max-manifest-depth']
  97         self.assertEqual(
  98             self.cache_path_from_arglist(base_args + ['-1']),
  99             self.cache_path_from_arglist(base_args + ['-2']))
 100
 101     def test_cache_names_treat_stdin_consistently(self):
 102         self.assertEqual(
 103             self.cache_path_from_arglist(['-', '--filename', 'test']),
 104             self.cache_path_from_arglist(['/dev/stdin', '--filename', 'test']))
 105
 106     def test_cache_names_identical_for_synonymous_names(self):
 107         self.assertEqual(
 108             self.cache_path_from_arglist(['.']),
 109             self.cache_path_from_arglist([os.path.realpath('.')]))
 110         testdir = self.make_tmpdir()
 111         looplink = os.path.join(testdir, 'loop')
 112         os.symlink(testdir, looplink)
 113         self.assertEqual(
 114             self.cache_path_from_arglist([testdir]),
 115             self.cache_path_from_arglist([looplink]))
 116
 117     def test_cache_names_different_by_api_host(self):
 118         config = arvados.config.settings()
 119         orig_host = config.get('ARVADOS_API_HOST')
 120         try:
 121             name1 = self.cache_path_from_arglist(['.'])
 122             config['ARVADOS_API_HOST'] = 'x' + (orig_host or 'localhost')
 123             self.assertNotEqual(name1, self.cache_path_from_arglist(['.']))
 124         finally:
 125             if orig_host is None:
 126                 del config['ARVADOS_API_HOST']
 127             else:
 128                 config['ARVADOS_API_HOST'] = orig_host
 129
 130     @mock.patch('arvados.keep.KeepClient.head')
 131     def test_resume_cache_with_current_stream_locators(self, keep_client_head):
 132         keep_client_head.side_effect = [True]
 133         thing = {}
 134         thing['_current_stream_locators'] = ['098f6bcd4621d373cade4e832627b4f6+4', '1f253c60a2306e0ee12fb6ce0c587904+6']
 135         with tempfile.NamedTemporaryFile() as cachefile:
 136             self.last_cache = arv_put.ResumeCache(cachefile.name)
 137         self.last_cache.save(thing)
 138         self.last_cache.close()
 139         resume_cache = arv_put.ResumeCache(self.last_cache.filename)
 140         self.assertNotEqual(None, resume_cache)
 141
 142     @mock.patch('arvados.keep.KeepClient.head')
 143     def test_resume_cache_with_finished_streams(self, keep_client_head):
 144         keep_client_head.side_effect = [True]
 145         thing = {}
 146         thing['_finished_streams'] = [['.', ['098f6bcd4621d373cade4e832627b4f6+4', '1f253c60a2306e0ee12fb6ce0c587904+6']]]
 147         with tempfile.NamedTemporaryFile() as cachefile:
 148             self.last_cache = arv_put.ResumeCache(cachefile.name)
 149         self.last_cache.save(thing)
 150         self.last_cache.close()
 151         resume_cache = arv_put.ResumeCache(self.last_cache.filename)
 152         self.assertNotEqual(None, resume_cache)
 153
 154     @mock.patch('arvados.keep.KeepClient.head')
 155     def test_resume_cache_with_finished_streams_error_on_head(self, keep_client_head):
 156         keep_client_head.side_effect = Exception('Locator not found')
 157         thing = {}
 158         thing['_finished_streams'] = [['.', ['098f6bcd4621d373cade4e832627b4f6+4', '1f253c60a2306e0ee12fb6ce0c587904+6']]]
 159         with tempfile.NamedTemporaryFile() as cachefile:
 160             self.last_cache = arv_put.ResumeCache(cachefile.name)
 161         self.last_cache.save(thing)
 162         self.last_cache.close()
 163         resume_cache = arv_put.ResumeCache(self.last_cache.filename)
 164         self.assertNotEqual(None, resume_cache)
 165         self.assertRaises(None, resume_cache.check_cache())
 166
 167     def test_basic_cache_storage(self):
 168         thing = ['test', 'list']
 169         with tempfile.NamedTemporaryFile() as cachefile:
 170             self.last_cache = arv_put.ResumeCache(cachefile.name)
 171         self.last_cache.save(thing)
 172         self.assertEqual(thing, self.last_cache.load())
 173
 174     def test_empty_cache(self):
 175         with tempfile.NamedTemporaryFile() as cachefile:
 176             cache = arv_put.ResumeCache(cachefile.name)
 177         self.assertRaises(ValueError, cache.load)
 178
 179     def test_cache_persistent(self):
 180         thing = ['test', 'list']
 181         path = os.path.join(self.make_tmpdir(), 'cache')
 182         cache = arv_put.ResumeCache(path)
 183         cache.save(thing)
 184         cache.close()
 185         self.last_cache = arv_put.ResumeCache(path)
 186         self.assertEqual(thing, self.last_cache.load())
 187
 188     def test_multiple_cache_writes(self):
 189         thing = ['short', 'list']
 190         with tempfile.NamedTemporaryFile() as cachefile:
 191             self.last_cache = arv_put.ResumeCache(cachefile.name)
 192         # Start writing an object longer than the one we test, to make
 193         # sure the cache file gets truncated.
 194         self.last_cache.save(['long', 'long', 'list'])
 195         self.last_cache.save(thing)
 196         self.assertEqual(thing, self.last_cache.load())
 197
 198     def test_cache_is_locked(self):
 199         with tempfile.NamedTemporaryFile() as cachefile:
 200             cache = arv_put.ResumeCache(cachefile.name)
 201             self.assertRaises(arv_put.ResumeCacheConflict,
 202                               arv_put.ResumeCache, cachefile.name)
 203
 204     def test_cache_stays_locked(self):
 205         with tempfile.NamedTemporaryFile() as cachefile:
 206             self.last_cache = arv_put.ResumeCache(cachefile.name)
 207             path = cachefile.name
 208         self.last_cache.save('test')
 209         self.assertRaises(arv_put.ResumeCacheConflict,
 210                           arv_put.ResumeCache, path)
 211
 212     def test_destroy_cache(self):
 213         cachefile = tempfile.NamedTemporaryFile(delete=False)
 214         try:
 215             cache = arv_put.ResumeCache(cachefile.name)
 216             cache.save('test')
 217             cache.destroy()
 218             try:
 219                 arv_put.ResumeCache(cachefile.name)
 220             except arv_put.ResumeCacheConflict:
 221                 self.fail("could not load cache after destroying it")
 222             self.assertRaises(ValueError, cache.load)
 223         finally:
 224             if os.path.exists(cachefile.name):
 225                 os.unlink(cachefile.name)
 226
 227     def test_restart_cache(self):
 228         path = os.path.join(self.make_tmpdir(), 'cache')
 229         cache = arv_put.ResumeCache(path)
 230         cache.save('test')
 231         cache.restart()
 232         self.assertRaises(ValueError, cache.load)
 233         self.assertRaises(arv_put.ResumeCacheConflict,
 234                           arv_put.ResumeCache, path)
 235
 236
 237 class ArvadosPutCollectionWriterTest(run_test_server.TestCaseWithServers,
 238                                      ArvadosBaseTestCase):
 239     def setUp(self):
 240         super(ArvadosPutCollectionWriterTest, self).setUp()
 241         run_test_server.authorize_with('active')
 242         with tempfile.NamedTemporaryFile(delete=False) as cachefile:
 243             self.cache = arv_put.ResumeCache(cachefile.name)
 244             self.cache_filename = cachefile.name
 245
 246     def tearDown(self):
 247         super(ArvadosPutCollectionWriterTest, self).tearDown()
 248         if os.path.exists(self.cache_filename):
 249             self.cache.destroy()
 250         self.cache.close()
 251
 252     def test_writer_caches(self):
 253         cwriter = arv_put.ArvPutCollectionWriter(self.cache)
 254         cwriter.write_file('/dev/null')
 255         cwriter.cache_state()
 256         self.assertTrue(self.cache.load())
 257         self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text())
 258
 259     def test_writer_works_without_cache(self):
 260         cwriter = arv_put.ArvPutCollectionWriter()
 261         cwriter.write_file('/dev/null')
 262         self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text())
 263
 264     def test_writer_resumes_from_cache(self):
 265         cwriter = arv_put.ArvPutCollectionWriter(self.cache)
 266         with self.make_test_file() as testfile:
 267             cwriter.write_file(testfile.name, 'test')
 268             cwriter.cache_state()
 269             new_writer = arv_put.ArvPutCollectionWriter.from_cache(
 270                 self.cache)
 271             self.assertEqual(
 272                 ". 098f6bcd4621d373cade4e832627b4f6+4 0:4:test\n",
 273                 new_writer.manifest_text())
 274
 275     def test_new_writer_from_stale_cache(self):
 276         cwriter = arv_put.ArvPutCollectionWriter(self.cache)
 277         with self.make_test_file() as testfile:
 278             cwriter.write_file(testfile.name, 'test')
 279         new_writer = arv_put.ArvPutCollectionWriter.from_cache(self.cache)
 280         new_writer.write_file('/dev/null')
 281         self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", new_writer.manifest_text())
 282
 283     def test_new_writer_from_empty_cache(self):
 284         cwriter = arv_put.ArvPutCollectionWriter.from_cache(self.cache)
 285         cwriter.write_file('/dev/null')
 286         self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text())
 287
 288     def test_writer_resumable_after_arbitrary_bytes(self):
 289         cwriter = arv_put.ArvPutCollectionWriter(self.cache)
 290         # These bytes are intentionally not valid UTF-8.
 291         with self.make_test_file('\x00\x07\xe2') as testfile:
 292             cwriter.write_file(testfile.name, 'test')
 293             cwriter.cache_state()
 294             new_writer = arv_put.ArvPutCollectionWriter.from_cache(
 295                 self.cache)
 296         self.assertEqual(cwriter.manifest_text(), new_writer.manifest_text())
 297
 298     def make_progress_tester(self):
 299         progression = []
 300         def record_func(written, expected):
 301             progression.append((written, expected))
 302         return progression, record_func
 303
 304     def test_progress_reporting(self):
 305         for expect_count in (None, 8):
 306             progression, reporter = self.make_progress_tester()
 307             cwriter = arv_put.ArvPutCollectionWriter(
 308                 reporter=reporter, bytes_expected=expect_count)
 309             with self.make_test_file() as testfile:
 310                 cwriter.write_file(testfile.name, 'test')
 311             cwriter.finish_current_stream()
 312             self.assertIn((4, expect_count), progression)
 313
 314     def test_resume_progress(self):
 315         cwriter = arv_put.ArvPutCollectionWriter(self.cache, bytes_expected=4)
 316         with self.make_test_file() as testfile:
 317             # Set up a writer with some flushed bytes.
 318             cwriter.write_file(testfile.name, 'test')
 319             cwriter.finish_current_stream()
 320             cwriter.cache_state()
 321             new_writer = arv_put.ArvPutCollectionWriter.from_cache(self.cache)
 322             self.assertEqual(new_writer.bytes_written, 4)
 323
 324
 325 class ArvadosExpectedBytesTest(ArvadosBaseTestCase):
 326     TEST_SIZE = os.path.getsize(__file__)
 327
 328     def test_expected_bytes_for_file(self):
 329         self.assertEqual(self.TEST_SIZE,
 330                           arv_put.expected_bytes_for([__file__]))
 331
 332     def test_expected_bytes_for_tree(self):
 333         tree = self.make_tmpdir()
 334         shutil.copyfile(__file__, os.path.join(tree, 'one'))
 335         shutil.copyfile(__file__, os.path.join(tree, 'two'))
 336         self.assertEqual(self.TEST_SIZE * 2,
 337                           arv_put.expected_bytes_for([tree]))
 338         self.assertEqual(self.TEST_SIZE * 3,
 339                           arv_put.expected_bytes_for([tree, __file__]))
 340
 341     def test_expected_bytes_for_device(self):
 342         self.assertIsNone(arv_put.expected_bytes_for(['/dev/null']))
 343         self.assertIsNone(arv_put.expected_bytes_for([__file__, '/dev/null']))
 344
 345
 346 class ArvadosPutReportTest(ArvadosBaseTestCase):
 347     def test_machine_progress(self):
 348         for count, total in [(0, 1), (0, None), (1, None), (235, 9283)]:
 349             expect = ": {} written {} total\n".format(
 350                 count, -1 if (total is None) else total)
 351             self.assertTrue(
 352                 arv_put.machine_progress(count, total).endswith(expect))
 353
 354     def test_known_human_progress(self):
 355         for count, total in [(0, 1), (2, 4), (45, 60)]:
 356             expect = '{:.1%}'.format(float(count) / total)
 357             actual = arv_put.human_progress(count, total)
 358             self.assertTrue(actual.startswith('\r'))
 359             self.assertIn(expect, actual)
 360
 361     def test_unknown_human_progress(self):
 362         for count in [1, 20, 300, 4000, 50000]:
 363             self.assertTrue(re.search(r'\b{}\b'.format(count),
 364                                       arv_put.human_progress(count, None)))
 365
 366
 367 class ArvadosPutTest(run_test_server.TestCaseWithServers, ArvadosBaseTestCase):
 368     MAIN_SERVER = {}
 369     Z_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
 370
 371     def call_main_with_args(self, args):
 372         self.main_stdout = StringIO()
 373         self.main_stderr = StringIO()
 374         return arv_put.main(args, self.main_stdout, self.main_stderr)
 375
 376     def call_main_on_test_file(self, args=[]):
 377         with self.make_test_file() as testfile:
 378             path = testfile.name
 379             self.call_main_with_args(['--stream', '--no-progress'] + args + [path])
 380         self.assertTrue(
 381             os.path.exists(os.path.join(os.environ['KEEP_LOCAL_STORE'],
 382                                         '098f6bcd4621d373cade4e832627b4f6')),
 383             "did not find file stream in Keep store")
 384
 385     def setUp(self):
 386         super(ArvadosPutTest, self).setUp()
 387         run_test_server.authorize_with('active')
 388         arv_put.api_client = None
 389
 390     def tearDown(self):
 391         for outbuf in ['main_stdout', 'main_stderr']:
 392             if hasattr(self, outbuf):
 393                 getattr(self, outbuf).close()
 394                 delattr(self, outbuf)
 395         super(ArvadosPutTest, self).tearDown()
 396
 397     def test_simple_file_put(self):
 398         self.call_main_on_test_file()
 399
 400     def test_put_with_unwriteable_cache_dir(self):
 401         orig_cachedir = arv_put.ResumeCache.CACHE_DIR
 402         cachedir = self.make_tmpdir()
 403         os.chmod(cachedir, 0o0)
 404         arv_put.ResumeCache.CACHE_DIR = cachedir
 405         try:
 406             self.call_main_on_test_file()
 407         finally:
 408             arv_put.ResumeCache.CACHE_DIR = orig_cachedir
 409             os.chmod(cachedir, 0o700)
 410
 411     def test_put_with_unwritable_cache_subdir(self):
 412         orig_cachedir = arv_put.ResumeCache.CACHE_DIR
 413         cachedir = self.make_tmpdir()
 414         os.chmod(cachedir, 0o0)
 415         arv_put.ResumeCache.CACHE_DIR = os.path.join(cachedir, 'cachedir')
 416         try:
 417             self.call_main_on_test_file()
 418         finally:
 419             arv_put.ResumeCache.CACHE_DIR = orig_cachedir
 420             os.chmod(cachedir, 0o700)
 421
 422     def test_put_block_replication(self):
 423         with mock.patch('arvados.collection.KeepClient.local_store_put') as put_mock, \
 424              mock.patch('arvados.commands.put.ResumeCache.load') as cache_mock:
 425             cache_mock.side_effect = ValueError
 426             put_mock.return_value = 'acbd18db4cc2f85cedef654fccc4a4d8+3'
 427             self.call_main_on_test_file(['--replication', '1'])
 428             self.call_main_on_test_file(['--replication', '4'])
 429             self.call_main_on_test_file(['--replication', '5'])
 430             self.assertEqual(
 431                 [x[-1].get('copies') for x in put_mock.call_args_list],
 432                 [1, 4, 5])
 433
 434     def test_normalize(self):
 435         testfile1 = self.make_test_file()
 436         testfile2 = self.make_test_file()
 437         test_paths = [testfile1.name, testfile2.name]
 438         # Reverse-sort the paths, so normalization must change their order.
 439         test_paths.sort(reverse=True)
 440         self.call_main_with_args(['--stream', '--no-progress', '--normalize'] +
 441                                  test_paths)
 442         manifest = self.main_stdout.getvalue()
 443         # Assert the second file we specified appears first in the manifest.
 444         file_indices = [manifest.find(':' + os.path.basename(path))
 445                         for path in test_paths]
 446         self.assertGreater(*file_indices)
 447
 448     def test_error_name_without_collection(self):
 449         self.assertRaises(SystemExit, self.call_main_with_args,
 450                           ['--name', 'test without Collection',
 451                            '--stream', '/dev/null'])
 452
 453     def test_error_when_project_not_found(self):
 454         self.assertRaises(SystemExit,
 455                           self.call_main_with_args,
 456                           ['--project-uuid', self.Z_UUID])
 457
 458     def test_error_bad_project_uuid(self):
 459         self.assertRaises(SystemExit,
 460                           self.call_main_with_args,
 461                           ['--project-uuid', self.Z_UUID, '--stream'])
 462
 463 class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
 464                             ArvadosBaseTestCase):
 465     def _getKeepServerConfig():
 466         for config_file, mandatory in [
 467                 ['application.yml', False], ['application.default.yml', True]]:
 468             path = os.path.join(run_test_server.SERVICES_SRC_DIR,
 469                                 "api", "config", config_file)
 470             if not mandatory and not os.path.exists(path):
 471                 continue
 472             with open(path) as f:
 473                 rails_config = yaml.load(f.read())
 474                 for config_section in ['test', 'common']:
 475                     try:
 476                         key = rails_config[config_section]["blob_signing_key"]
 477                     except (KeyError, TypeError):
 478                         pass
 479                     else:
 480                         return {'blob_signing_key': key,
 481                                 'enforce_permissions': True}
 482         return {'blog_signing_key': None, 'enforce_permissions': False}
 483
 484     MAIN_SERVER = {}
 485     KEEP_SERVER = _getKeepServerConfig()
 486     PROJECT_UUID = run_test_server.fixture('groups')['aproject']['uuid']
 487
 488     @classmethod
 489     def setUpClass(cls):
 490         super(ArvPutIntegrationTest, cls).setUpClass()
 491         cls.ENVIRON = os.environ.copy()
 492         cls.ENVIRON['PYTHONPATH'] = ':'.join(sys.path)
 493
 494     def setUp(self):
 495         super(ArvPutIntegrationTest, self).setUp()
 496         arv_put.api_client = None
 497
 498     def authorize_with(self, token_name):
 499         run_test_server.authorize_with(token_name)
 500         for v in ["ARVADOS_API_HOST",
 501                   "ARVADOS_API_HOST_INSECURE",
 502                   "ARVADOS_API_TOKEN"]:
 503             self.ENVIRON[v] = arvados.config.settings()[v]
 504         arv_put.api_client = arvados.api('v1')
 505
 506     def current_user(self):
 507         return arv_put.api_client.users().current().execute()
 508
 509     def test_check_real_project_found(self):
 510         self.authorize_with('active')
 511         self.assertTrue(arv_put.desired_project_uuid(arv_put.api_client, self.PROJECT_UUID, 0),
 512                         "did not correctly find test fixture project")
 513
 514     def test_check_error_finding_nonexistent_uuid(self):
 515         BAD_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
 516         self.authorize_with('active')
 517         try:
 518             result = arv_put.desired_project_uuid(arv_put.api_client, BAD_UUID,
 519                                                   0)
 520         except ValueError as error:
 521             self.assertIn(BAD_UUID, error.message)
 522         else:
 523             self.assertFalse(result, "incorrectly found nonexistent project")
 524
 525     def test_check_error_finding_nonexistent_project(self):
 526         BAD_UUID = 'zzzzz-tpzed-zzzzzzzzzzzzzzz'
 527         self.authorize_with('active')
 528         with self.assertRaises(apiclient.errors.HttpError):
 529             result = arv_put.desired_project_uuid(arv_put.api_client, BAD_UUID,
 530                                                   0)
 531
 532     def test_short_put_from_stdin(self):
 533         # Have to run this as an integration test since arv-put can't
 534         # read from the tests' stdin.
 535         # arv-put usually can't stat(os.path.realpath('/dev/stdin')) in this
 536         # case, because the /proc entry is already gone by the time it tries.
 537         pipe = subprocess.Popen(
 538             [sys.executable, arv_put.__file__, '--stream'],
 539             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
 540             stderr=subprocess.STDOUT, env=self.ENVIRON)
 541         pipe.stdin.write('stdin test\n')
 542         pipe.stdin.close()
 543         deadline = time.time() + 5
 544         while (pipe.poll() is None) and (time.time() < deadline):
 545             time.sleep(.1)
 546         returncode = pipe.poll()
 547         if returncode is None:
 548             pipe.terminate()
 549             self.fail("arv-put did not PUT from stdin within 5 seconds")
 550         elif returncode != 0:
 551             sys.stdout.write(pipe.stdout.read())
 552             self.fail("arv-put returned exit code {}".format(returncode))
 553         self.assertIn('4a9c8b735dce4b5fa3acf221a0b13628+11', pipe.stdout.read())
 554
 555     def test_ArvPutSignedManifest(self):
 556         # ArvPutSignedManifest runs "arv-put foo" and then attempts to get
 557         # the newly created manifest from the API server, testing to confirm
 558         # that the block locators in the returned manifest are signed.
 559         self.authorize_with('active')
 560
 561         # Before doing anything, demonstrate that the collection
 562         # we're about to create is not present in our test fixture.
 563         manifest_uuid = "00b4e9f40ac4dd432ef89749f1c01e74+47"
 564         with self.assertRaises(apiclient.errors.HttpError):
 565             notfound = arv_put.api_client.collections().get(
 566                 uuid=manifest_uuid).execute()
 567
 568         datadir = self.make_tmpdir()
 569         with open(os.path.join(datadir, "foo"), "w") as f:
 570             f.write("The quick brown fox jumped over the lazy dog")
 571         p = subprocess.Popen([sys.executable, arv_put.__file__, datadir],
 572                              stdout=subprocess.PIPE, env=self.ENVIRON)
 573         (arvout, arverr) = p.communicate()
 574         self.assertEqual(arverr, None)
 575         self.assertEqual(p.returncode, 0)
 576
 577         # The manifest text stored in the API server under the same
 578         # manifest UUID must use signed locators.
 579         c = arv_put.api_client.collections().get(uuid=manifest_uuid).execute()
 580         self.assertRegexpMatches(
 581             c['manifest_text'],
 582             r'^\. 08a008a01d498c404b0c30852b39d3b8\+44\+A[0-9a-f]+@[0-9a-f]+ 0:44:foo\n')
 583
 584         os.remove(os.path.join(datadir, "foo"))
 585         os.rmdir(datadir)
 586
 587     def run_and_find_collection(self, text, extra_args=[]):
 588         self.authorize_with('active')
 589         pipe = subprocess.Popen(
 590             [sys.executable, arv_put.__file__] + extra_args,
 591             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
 592             stderr=subprocess.PIPE, env=self.ENVIRON)
 593         stdout, stderr = pipe.communicate(text)
 594         search_key = ('portable_data_hash'
 595                       if '--portable-data-hash' in extra_args else 'uuid')
 596         collection_list = arvados.api('v1').collections().list(
 597             filters=[[search_key, '=', stdout.strip()]]).execute().get('items', [])
 598         self.assertEqual(1, len(collection_list))
 599         return collection_list[0]
 600
 601     def test_put_collection_with_high_redundancy(self):
 602         # Write empty data: we're not testing CollectionWriter, just
 603         # making sure collections.create tells the API server what our
 604         # desired replication level is.
 605         collection = self.run_and_find_collection("", ['--replication', '4'])
 606         self.assertEqual(4, collection['replication_desired'])
 607
 608     def test_put_collection_with_default_redundancy(self):
 609         collection = self.run_and_find_collection("")
 610         self.assertEqual(None, collection['replication_desired'])
 611
 612     def test_put_collection_with_unnamed_project_link(self):
 613         link = self.run_and_find_collection(
 614             "Test unnamed collection",
 615             ['--portable-data-hash', '--project-uuid', self.PROJECT_UUID])
 616         username = pwd.getpwuid(os.getuid()).pw_name
 617         self.assertRegexpMatches(
 618             link['name'],
 619             r'^Saved at .* by {}@'.format(re.escape(username)))
 620
 621     def test_put_collection_with_name_and_no_project(self):
 622         link_name = 'Test Collection Link in home project'
 623         collection = self.run_and_find_collection(
 624             "Test named collection in home project",
 625             ['--portable-data-hash', '--name', link_name])
 626         self.assertEqual(link_name, collection['name'])
 627         my_user_uuid = self.current_user()['uuid']
 628         self.assertEqual(my_user_uuid, collection['owner_uuid'])
 629
 630     def test_put_collection_with_named_project_link(self):
 631         link_name = 'Test auto Collection Link'
 632         collection = self.run_and_find_collection("Test named collection",
 633                                       ['--portable-data-hash',
 634                                        '--name', link_name,
 635                                        '--project-uuid', self.PROJECT_UUID])
 636         self.assertEqual(link_name, collection['name'])
 637
 638
 639 if __name__ == '__main__':
 640     unittest.main()