sdk/python/tests/test_arv_put.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 import apiclient
   5 import mock
   6 import os
   7 import pwd
   8 import re
   9 import shutil
  10 import subprocess
  11 import sys
  12 import tempfile
  13 import time
  14 import unittest
  15 import yaml
  16
  17 from cStringIO import StringIO
  18
  19 import arvados
  20 import arvados.commands.put as arv_put
  21
  22 from arvados_testutil import ArvadosBaseTestCase, fake_httplib2_response
  23 import run_test_server
  24
  25 class ArvadosPutResumeCacheTest(ArvadosBaseTestCase):
  26     CACHE_ARGSET = [
  27         [],
  28         ['/dev/null'],
  29         ['/dev/null', '--filename', 'empty'],
  30         ['/tmp'],
  31         ['/tmp', '--max-manifest-depth', '0'],
  32         ['/tmp', '--max-manifest-depth', '1']
  33         ]
  34
  35     def tearDown(self):
  36         super(ArvadosPutResumeCacheTest, self).tearDown()
  37         try:
  38             self.last_cache.destroy()
  39         except AttributeError:
  40             pass
  41
  42     def cache_path_from_arglist(self, arglist):
  43         return arv_put.ResumeCache.make_path(arv_put.parse_arguments(arglist))
  44
  45     def test_cache_names_stable(self):
  46         for argset in self.CACHE_ARGSET:
  47             self.assertEqual(self.cache_path_from_arglist(argset),
  48                               self.cache_path_from_arglist(argset),
  49                               "cache name changed for {}".format(argset))
  50
  51     def test_cache_names_unique(self):
  52         results = []
  53         for argset in self.CACHE_ARGSET:
  54             path = self.cache_path_from_arglist(argset)
  55             self.assertNotIn(path, results)
  56             results.append(path)
  57
  58     def test_cache_names_simple(self):
  59         # The goal here is to make sure the filename doesn't use characters
  60         # reserved by the filesystem.  Feel free to adjust this regexp as
  61         # long as it still does that.
  62         bad_chars = re.compile(r'[^-\.\w]')
  63         for argset in self.CACHE_ARGSET:
  64             path = self.cache_path_from_arglist(argset)
  65             self.assertFalse(bad_chars.search(os.path.basename(path)),
  66                              "path too exotic: {}".format(path))
  67
  68     def test_cache_names_ignore_argument_order(self):
  69         self.assertEqual(
  70             self.cache_path_from_arglist(['a', 'b', 'c']),
  71             self.cache_path_from_arglist(['c', 'a', 'b']))
  72         self.assertEqual(
  73             self.cache_path_from_arglist(['-', '--filename', 'stdin']),
  74             self.cache_path_from_arglist(['--filename', 'stdin', '-']))
  75
  76     def test_cache_names_differ_for_similar_paths(self):
  77         # This test needs names at / that don't exist on the real filesystem.
  78         self.assertNotEqual(
  79             self.cache_path_from_arglist(['/_arvputtest1', '/_arvputtest2']),
  80             self.cache_path_from_arglist(['/_arvputtest1/_arvputtest2']))
  81
  82     def test_cache_names_ignore_irrelevant_arguments(self):
  83         # Workaround: parse_arguments bails on --filename with a directory.
  84         path1 = self.cache_path_from_arglist(['/tmp'])
  85         args = arv_put.parse_arguments(['/tmp'])
  86         args.filename = 'tmp'
  87         path2 = arv_put.ResumeCache.make_path(args)
  88         self.assertEqual(path1, path2,
  89                          "cache path considered --filename for directory")
  90         self.assertEqual(
  91             self.cache_path_from_arglist(['-']),
  92             self.cache_path_from_arglist(['-', '--max-manifest-depth', '1']),
  93             "cache path considered --max-manifest-depth for file")
  94
  95     def test_cache_names_treat_negative_manifest_depths_identically(self):
  96         base_args = ['/tmp', '--max-manifest-depth']
  97         self.assertEqual(
  98             self.cache_path_from_arglist(base_args + ['-1']),
  99             self.cache_path_from_arglist(base_args + ['-2']))
 100
 101     def test_cache_names_treat_stdin_consistently(self):
 102         self.assertEqual(
 103             self.cache_path_from_arglist(['-', '--filename', 'test']),
 104             self.cache_path_from_arglist(['/dev/stdin', '--filename', 'test']))
 105
 106     def test_cache_names_identical_for_synonymous_names(self):
 107         self.assertEqual(
 108             self.cache_path_from_arglist(['.']),
 109             self.cache_path_from_arglist([os.path.realpath('.')]))
 110         testdir = self.make_tmpdir()
 111         looplink = os.path.join(testdir, 'loop')
 112         os.symlink(testdir, looplink)
 113         self.assertEqual(
 114             self.cache_path_from_arglist([testdir]),
 115             self.cache_path_from_arglist([looplink]))
 116
 117     def test_cache_names_different_by_api_host(self):
 118         config = arvados.config.settings()
 119         orig_host = config.get('ARVADOS_API_HOST')
 120         try:
 121             name1 = self.cache_path_from_arglist(['.'])
 122             config['ARVADOS_API_HOST'] = 'x' + (orig_host or 'localhost')
 123             self.assertNotEqual(name1, self.cache_path_from_arglist(['.']))
 124         finally:
 125             if orig_host is None:
 126                 del config['ARVADOS_API_HOST']
 127             else:
 128                 config['ARVADOS_API_HOST'] = orig_host
 129
 130     @mock.patch('arvados.keep.KeepClient.head')
 131     def test_resume_cache_with_current_stream_locators(self, keep_client_head):
 132         keep_client_head.side_effect = [True]
 133         thing = {}
 134         thing['_current_stream_locators'] = ['098f6bcd4621d373cade4e832627b4f6+4', '1f253c60a2306e0ee12fb6ce0c587904+6']
 135         with tempfile.NamedTemporaryFile() as cachefile:
 136             self.last_cache = arv_put.ResumeCache(cachefile.name)
 137         self.last_cache.save(thing)
 138         self.last_cache.close()
 139         resume_cache = arv_put.ResumeCache(self.last_cache.filename)
 140         self.assertNotEqual(None, resume_cache)
 141
 142     @mock.patch('arvados.keep.KeepClient.head')
 143     def test_resume_cache_with_finished_streams(self, keep_client_head):
 144         keep_client_head.side_effect = [True]
 145         thing = {}
 146         thing['_finished_streams'] = [['.', ['098f6bcd4621d373cade4e832627b4f6+4', '1f253c60a2306e0ee12fb6ce0c587904+6']]]
 147         with tempfile.NamedTemporaryFile() as cachefile:
 148             self.last_cache = arv_put.ResumeCache(cachefile.name)
 149         self.last_cache.save(thing)
 150         self.last_cache.close()
 151         resume_cache = arv_put.ResumeCache(self.last_cache.filename)
 152         self.assertNotEqual(None, resume_cache)
 153
 154     @mock.patch('arvados.keep.KeepClient.head')
 155     def test_resume_cache_with_finished_streams_error_on_head(self, keep_client_head):
 156         keep_client_head.side_effect = Exception('Locator not found')
 157         thing = {}
 158         thing['_finished_streams'] = [['.', ['098f6bcd4621d373cade4e832627b4f6+4', '1f253c60a2306e0ee12fb6ce0c587904+6']]]
 159         with tempfile.NamedTemporaryFile() as cachefile:
 160             self.last_cache = arv_put.ResumeCache(cachefile.name)
 161         self.last_cache.save(thing)
 162         self.last_cache.close()
 163         resume_cache = arv_put.ResumeCache(self.last_cache.filename)
 164         self.assertNotEqual(None, resume_cache)
 165         self.assertRaises(None, resume_cache.check_cache())
 166
 167     def test_basic_cache_storage(self):
 168         thing = ['test', 'list']
 169         with tempfile.NamedTemporaryFile() as cachefile:
 170             self.last_cache = arv_put.ResumeCache(cachefile.name)
 171         self.last_cache.save(thing)
 172         self.assertEqual(thing, self.last_cache.load())
 173
 174     def test_empty_cache(self):
 175         with tempfile.NamedTemporaryFile() as cachefile:
 176             cache = arv_put.ResumeCache(cachefile.name)
 177         self.assertRaises(ValueError, cache.load)
 178
 179     def test_cache_persistent(self):
 180         thing = ['test', 'list']
 181         path = os.path.join(self.make_tmpdir(), 'cache')
 182         cache = arv_put.ResumeCache(path)
 183         cache.save(thing)
 184         cache.close()
 185         self.last_cache = arv_put.ResumeCache(path)
 186         self.assertEqual(thing, self.last_cache.load())
 187
 188     def test_multiple_cache_writes(self):
 189         thing = ['short', 'list']
 190         with tempfile.NamedTemporaryFile() as cachefile:
 191             self.last_cache = arv_put.ResumeCache(cachefile.name)
 192         # Start writing an object longer than the one we test, to make
 193         # sure the cache file gets truncated.
 194         self.last_cache.save(['long', 'long', 'list'])
 195         self.last_cache.save(thing)
 196         self.assertEqual(thing, self.last_cache.load())
 197
 198     def test_cache_is_locked(self):
 199         with tempfile.NamedTemporaryFile() as cachefile:
 200             cache = arv_put.ResumeCache(cachefile.name)
 201             self.assertRaises(arv_put.ResumeCacheConflict,
 202                               arv_put.ResumeCache, cachefile.name)
 203
 204     def test_cache_stays_locked(self):
 205         with tempfile.NamedTemporaryFile() as cachefile:
 206             self.last_cache = arv_put.ResumeCache(cachefile.name)
 207             path = cachefile.name
 208         self.last_cache.save('test')
 209         self.assertRaises(arv_put.ResumeCacheConflict,
 210                           arv_put.ResumeCache, path)
 211
 212     def test_destroy_cache(self):
 213         cachefile = tempfile.NamedTemporaryFile(delete=False)
 214         try:
 215             cache = arv_put.ResumeCache(cachefile.name)
 216             cache.save('test')
 217             cache.destroy()
 218             try:
 219                 arv_put.ResumeCache(cachefile.name)
 220             except arv_put.ResumeCacheConflict:
 221                 self.fail("could not load cache after destroying it")
 222             self.assertRaises(ValueError, cache.load)
 223         finally:
 224             if os.path.exists(cachefile.name):
 225                 os.unlink(cachefile.name)
 226
 227     def test_restart_cache(self):
 228         path = os.path.join(self.make_tmpdir(), 'cache')
 229         cache = arv_put.ResumeCache(path)
 230         cache.save('test')
 231         cache.restart()
 232         self.assertRaises(ValueError, cache.load)
 233         self.assertRaises(arv_put.ResumeCacheConflict,
 234                           arv_put.ResumeCache, path)
 235
 236
 237 class ArvadosPutCollectionTest(run_test_server.TestCaseWithServers):
 238     MAIN_SERVER = {}
 239     KEEP_SERVER = {}
 240
 241     def test_write_files(self):
 242         c = arv_put.ArvPutCollection()
 243         data = 'a' * 1024 * 1024 # 1 MB
 244         tmpdir = tempfile.mkdtemp()
 245         for size in [1, 10, 64, 128]:
 246             with open(os.path.join(tmpdir, 'file_%d' % size), 'w') as f:
 247                 for _ in range(size):
 248                     f.write(data)
 249             c.write_file(f.name, os.path.basename(f.name))
 250             os.unlink(f.name)
 251         self.assertEqual(True, c.manifest())
 252
 253     def test_write_directory(self):
 254         c = arv_put.ArvPutCollection()
 255         data = 'b' * 1024 * 1024
 256         tmpdir = tempfile.mkdtemp()
 257         for size in [1, 5, 10, 70]:
 258             with open(os.path.join(tmpdir, 'file_%d' % size), 'w') as f:
 259                 for _ in range(size):
 260                     f.write(data)
 261         os.mkdir(os.path.join(tmpdir, 'subdir1'))
 262         for size in [2, 4, 6]:
 263             with open(os.path.join(tmpdir, 'subdir1', 'file_%d' % size), 'w') as f:
 264                 for _ in range(size):
 265                     f.write(data)
 266         c.write_directory_tree(tmpdir, os.path.join('.', os.path.basename(tmpdir)))
 267         self.assertEqual(True, c.manifest())
 268
 269
 270 class ArvadosPutCollectionWriterTest(run_test_server.TestCaseWithServers,
 271                                      ArvadosBaseTestCase):
 272     def setUp(self):
 273         super(ArvadosPutCollectionWriterTest, self).setUp()
 274         run_test_server.authorize_with('active')
 275         with tempfile.NamedTemporaryFile(delete=False) as cachefile:
 276             self.cache = arv_put.ResumeCache(cachefile.name)
 277             self.cache_filename = cachefile.name
 278
 279     def tearDown(self):
 280         super(ArvadosPutCollectionWriterTest, self).tearDown()
 281         if os.path.exists(self.cache_filename):
 282             self.cache.destroy()
 283         self.cache.close()
 284
 285     def test_writer_caches(self):
 286         cwriter = arv_put.ArvPutCollectionWriter(self.cache)
 287         cwriter.write_file('/dev/null')
 288         cwriter.cache_state()
 289         self.assertTrue(self.cache.load())
 290         self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text())
 291
 292     def test_writer_works_without_cache(self):
 293         cwriter = arv_put.ArvPutCollectionWriter()
 294         cwriter.write_file('/dev/null')
 295         self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text())
 296
 297     def test_writer_resumes_from_cache(self):
 298         cwriter = arv_put.ArvPutCollectionWriter(self.cache)
 299         with self.make_test_file() as testfile:
 300             cwriter.write_file(testfile.name, 'test')
 301             cwriter.cache_state()
 302             new_writer = arv_put.ArvPutCollectionWriter.from_cache(
 303                 self.cache)
 304             self.assertEqual(
 305                 ". 098f6bcd4621d373cade4e832627b4f6+4 0:4:test\n",
 306                 new_writer.manifest_text())
 307
 308     def test_new_writer_from_stale_cache(self):
 309         cwriter = arv_put.ArvPutCollectionWriter(self.cache)
 310         with self.make_test_file() as testfile:
 311             cwriter.write_file(testfile.name, 'test')
 312         new_writer = arv_put.ArvPutCollectionWriter.from_cache(self.cache)
 313         new_writer.write_file('/dev/null')
 314         self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", new_writer.manifest_text())
 315
 316     def test_new_writer_from_empty_cache(self):
 317         cwriter = arv_put.ArvPutCollectionWriter.from_cache(self.cache)
 318         cwriter.write_file('/dev/null')
 319         self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text())
 320
 321     def test_writer_resumable_after_arbitrary_bytes(self):
 322         cwriter = arv_put.ArvPutCollectionWriter(self.cache)
 323         # These bytes are intentionally not valid UTF-8.
 324         with self.make_test_file('\x00\x07\xe2') as testfile:
 325             cwriter.write_file(testfile.name, 'test')
 326             cwriter.cache_state()
 327             new_writer = arv_put.ArvPutCollectionWriter.from_cache(
 328                 self.cache)
 329         self.assertEqual(cwriter.manifest_text(), new_writer.manifest_text())
 330
 331     def make_progress_tester(self):
 332         progression = []
 333         def record_func(written, expected):
 334             progression.append((written, expected))
 335         return progression, record_func
 336
 337     def test_progress_reporting(self):
 338         for expect_count in (None, 8):
 339             progression, reporter = self.make_progress_tester()
 340             cwriter = arv_put.ArvPutCollectionWriter(
 341                 reporter=reporter, bytes_expected=expect_count)
 342             with self.make_test_file() as testfile:
 343                 cwriter.write_file(testfile.name, 'test')
 344             cwriter.finish_current_stream()
 345             self.assertIn((4, expect_count), progression)
 346
 347     def test_resume_progress(self):
 348         cwriter = arv_put.ArvPutCollectionWriter(self.cache, bytes_expected=4)
 349         with self.make_test_file() as testfile:
 350             # Set up a writer with some flushed bytes.
 351             cwriter.write_file(testfile.name, 'test')
 352             cwriter.finish_current_stream()
 353             cwriter.cache_state()
 354             new_writer = arv_put.ArvPutCollectionWriter.from_cache(self.cache)
 355             self.assertEqual(new_writer.bytes_written, 4)
 356
 357
 358 class ArvadosExpectedBytesTest(ArvadosBaseTestCase):
 359     TEST_SIZE = os.path.getsize(__file__)
 360
 361     def test_expected_bytes_for_file(self):
 362         self.assertEqual(self.TEST_SIZE,
 363                           arv_put.expected_bytes_for([__file__]))
 364
 365     def test_expected_bytes_for_tree(self):
 366         tree = self.make_tmpdir()
 367         shutil.copyfile(__file__, os.path.join(tree, 'one'))
 368         shutil.copyfile(__file__, os.path.join(tree, 'two'))
 369         self.assertEqual(self.TEST_SIZE * 2,
 370                           arv_put.expected_bytes_for([tree]))
 371         self.assertEqual(self.TEST_SIZE * 3,
 372                           arv_put.expected_bytes_for([tree, __file__]))
 373
 374     def test_expected_bytes_for_device(self):
 375         self.assertIsNone(arv_put.expected_bytes_for(['/dev/null']))
 376         self.assertIsNone(arv_put.expected_bytes_for([__file__, '/dev/null']))
 377
 378
 379 class ArvadosPutReportTest(ArvadosBaseTestCase):
 380     def test_machine_progress(self):
 381         for count, total in [(0, 1), (0, None), (1, None), (235, 9283)]:
 382             expect = ": {} written {} total\n".format(
 383                 count, -1 if (total is None) else total)
 384             self.assertTrue(
 385                 arv_put.machine_progress(count, total).endswith(expect))
 386
 387     def test_known_human_progress(self):
 388         for count, total in [(0, 1), (2, 4), (45, 60)]:
 389             expect = '{:.1%}'.format(float(count) / total)
 390             actual = arv_put.human_progress(count, total)
 391             self.assertTrue(actual.startswith('\r'))
 392             self.assertIn(expect, actual)
 393
 394     def test_unknown_human_progress(self):
 395         for count in [1, 20, 300, 4000, 50000]:
 396             self.assertTrue(re.search(r'\b{}\b'.format(count),
 397                                       arv_put.human_progress(count, None)))
 398
 399
 400 class ArvadosPutTest(run_test_server.TestCaseWithServers, ArvadosBaseTestCase):
 401     MAIN_SERVER = {}
 402     Z_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
 403
 404     def call_main_with_args(self, args):
 405         self.main_stdout = StringIO()
 406         self.main_stderr = StringIO()
 407         return arv_put.main(args, self.main_stdout, self.main_stderr)
 408
 409     def call_main_on_test_file(self, args=[]):
 410         with self.make_test_file() as testfile:
 411             path = testfile.name
 412             self.call_main_with_args(['--stream', '--no-progress'] + args + [path])
 413         self.assertTrue(
 414             os.path.exists(os.path.join(os.environ['KEEP_LOCAL_STORE'],
 415                                         '098f6bcd4621d373cade4e832627b4f6')),
 416             "did not find file stream in Keep store")
 417
 418     def setUp(self):
 419         super(ArvadosPutTest, self).setUp()
 420         run_test_server.authorize_with('active')
 421         arv_put.api_client = None
 422
 423     def tearDown(self):
 424         for outbuf in ['main_stdout', 'main_stderr']:
 425             if hasattr(self, outbuf):
 426                 getattr(self, outbuf).close()
 427                 delattr(self, outbuf)
 428         super(ArvadosPutTest, self).tearDown()
 429
 430     def test_simple_file_put(self):
 431         self.call_main_on_test_file()
 432
 433     def test_put_with_unwriteable_cache_dir(self):
 434         orig_cachedir = arv_put.ResumeCache.CACHE_DIR
 435         cachedir = self.make_tmpdir()
 436         os.chmod(cachedir, 0o0)
 437         arv_put.ResumeCache.CACHE_DIR = cachedir
 438         try:
 439             self.call_main_on_test_file()
 440         finally:
 441             arv_put.ResumeCache.CACHE_DIR = orig_cachedir
 442             os.chmod(cachedir, 0o700)
 443
 444     def test_put_with_unwritable_cache_subdir(self):
 445         orig_cachedir = arv_put.ResumeCache.CACHE_DIR
 446         cachedir = self.make_tmpdir()
 447         os.chmod(cachedir, 0o0)
 448         arv_put.ResumeCache.CACHE_DIR = os.path.join(cachedir, 'cachedir')
 449         try:
 450             self.call_main_on_test_file()
 451         finally:
 452             arv_put.ResumeCache.CACHE_DIR = orig_cachedir
 453             os.chmod(cachedir, 0o700)
 454
 455     def test_put_block_replication(self):
 456         with mock.patch('arvados.collection.KeepClient.local_store_put') as put_mock, \
 457              mock.patch('arvados.commands.put.ResumeCache.load') as cache_mock:
 458             cache_mock.side_effect = ValueError
 459             put_mock.return_value = 'acbd18db4cc2f85cedef654fccc4a4d8+3'
 460             self.call_main_on_test_file(['--replication', '1'])
 461             self.call_main_on_test_file(['--replication', '4'])
 462             self.call_main_on_test_file(['--replication', '5'])
 463             self.assertEqual(
 464                 [x[-1].get('copies') for x in put_mock.call_args_list],
 465                 [1, 4, 5])
 466
 467     def test_normalize(self):
 468         testfile1 = self.make_test_file()
 469         testfile2 = self.make_test_file()
 470         test_paths = [testfile1.name, testfile2.name]
 471         # Reverse-sort the paths, so normalization must change their order.
 472         test_paths.sort(reverse=True)
 473         self.call_main_with_args(['--stream', '--no-progress', '--normalize'] +
 474                                  test_paths)
 475         manifest = self.main_stdout.getvalue()
 476         # Assert the second file we specified appears first in the manifest.
 477         file_indices = [manifest.find(':' + os.path.basename(path))
 478                         for path in test_paths]
 479         self.assertGreater(*file_indices)
 480
 481     def test_error_name_without_collection(self):
 482         self.assertRaises(SystemExit, self.call_main_with_args,
 483                           ['--name', 'test without Collection',
 484                            '--stream', '/dev/null'])
 485
 486     def test_error_when_project_not_found(self):
 487         self.assertRaises(SystemExit,
 488                           self.call_main_with_args,
 489                           ['--project-uuid', self.Z_UUID])
 490
 491     def test_error_bad_project_uuid(self):
 492         self.assertRaises(SystemExit,
 493                           self.call_main_with_args,
 494                           ['--project-uuid', self.Z_UUID, '--stream'])
 495
 496     def test_api_error_handling(self):
 497         collections_mock = mock.Mock(name='arv.collections()')
 498         coll_create_mock = collections_mock().create().execute
 499         coll_create_mock.side_effect = arvados.errors.ApiError(
 500             fake_httplib2_response(403), '{}')
 501         arv_put.api_client = arvados.api('v1')
 502         arv_put.api_client.collections = collections_mock
 503         with self.assertRaises(SystemExit) as exc_test:
 504             self.call_main_with_args(['/dev/null'])
 505         self.assertLess(0, exc_test.exception.args[0])
 506         self.assertLess(0, coll_create_mock.call_count)
 507         self.assertEqual("", self.main_stdout.getvalue())
 508
 509
 510 class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
 511                             ArvadosBaseTestCase):
 512     def _getKeepServerConfig():
 513         for config_file, mandatory in [
 514                 ['application.yml', False], ['application.default.yml', True]]:
 515             path = os.path.join(run_test_server.SERVICES_SRC_DIR,
 516                                 "api", "config", config_file)
 517             if not mandatory and not os.path.exists(path):
 518                 continue
 519             with open(path) as f:
 520                 rails_config = yaml.load(f.read())
 521                 for config_section in ['test', 'common']:
 522                     try:
 523                         key = rails_config[config_section]["blob_signing_key"]
 524                     except (KeyError, TypeError):
 525                         pass
 526                     else:
 527                         return {'blob_signing_key': key,
 528                                 'enforce_permissions': True}
 529         return {'blog_signing_key': None, 'enforce_permissions': False}
 530
 531     MAIN_SERVER = {}
 532     KEEP_SERVER = _getKeepServerConfig()
 533     PROJECT_UUID = run_test_server.fixture('groups')['aproject']['uuid']
 534
 535     @classmethod
 536     def setUpClass(cls):
 537         super(ArvPutIntegrationTest, cls).setUpClass()
 538         cls.ENVIRON = os.environ.copy()
 539         cls.ENVIRON['PYTHONPATH'] = ':'.join(sys.path)
 540
 541     def setUp(self):
 542         super(ArvPutIntegrationTest, self).setUp()
 543         arv_put.api_client = None
 544
 545     def authorize_with(self, token_name):
 546         run_test_server.authorize_with(token_name)
 547         for v in ["ARVADOS_API_HOST",
 548                   "ARVADOS_API_HOST_INSECURE",
 549                   "ARVADOS_API_TOKEN"]:
 550             self.ENVIRON[v] = arvados.config.settings()[v]
 551         arv_put.api_client = arvados.api('v1')
 552
 553     def current_user(self):
 554         return arv_put.api_client.users().current().execute()
 555
 556     def test_check_real_project_found(self):
 557         self.authorize_with('active')
 558         self.assertTrue(arv_put.desired_project_uuid(arv_put.api_client, self.PROJECT_UUID, 0),
 559                         "did not correctly find test fixture project")
 560
 561     def test_check_error_finding_nonexistent_uuid(self):
 562         BAD_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
 563         self.authorize_with('active')
 564         try:
 565             result = arv_put.desired_project_uuid(arv_put.api_client, BAD_UUID,
 566                                                   0)
 567         except ValueError as error:
 568             self.assertIn(BAD_UUID, error.message)
 569         else:
 570             self.assertFalse(result, "incorrectly found nonexistent project")
 571
 572     def test_check_error_finding_nonexistent_project(self):
 573         BAD_UUID = 'zzzzz-tpzed-zzzzzzzzzzzzzzz'
 574         self.authorize_with('active')
 575         with self.assertRaises(apiclient.errors.HttpError):
 576             result = arv_put.desired_project_uuid(arv_put.api_client, BAD_UUID,
 577                                                   0)
 578
 579     def test_short_put_from_stdin(self):
 580         # Have to run this as an integration test since arv-put can't
 581         # read from the tests' stdin.
 582         # arv-put usually can't stat(os.path.realpath('/dev/stdin')) in this
 583         # case, because the /proc entry is already gone by the time it tries.
 584         pipe = subprocess.Popen(
 585             [sys.executable, arv_put.__file__, '--stream'],
 586             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
 587             stderr=subprocess.STDOUT, env=self.ENVIRON)
 588         pipe.stdin.write('stdin test\n')
 589         pipe.stdin.close()
 590         deadline = time.time() + 5
 591         while (pipe.poll() is None) and (time.time() < deadline):
 592             time.sleep(.1)
 593         returncode = pipe.poll()
 594         if returncode is None:
 595             pipe.terminate()
 596             self.fail("arv-put did not PUT from stdin within 5 seconds")
 597         elif returncode != 0:
 598             sys.stdout.write(pipe.stdout.read())
 599             self.fail("arv-put returned exit code {}".format(returncode))
 600         self.assertIn('4a9c8b735dce4b5fa3acf221a0b13628+11', pipe.stdout.read())
 601
 602     def test_ArvPutSignedManifest(self):
 603         # ArvPutSignedManifest runs "arv-put foo" and then attempts to get
 604         # the newly created manifest from the API server, testing to confirm
 605         # that the block locators in the returned manifest are signed.
 606         self.authorize_with('active')
 607
 608         # Before doing anything, demonstrate that the collection
 609         # we're about to create is not present in our test fixture.
 610         manifest_uuid = "00b4e9f40ac4dd432ef89749f1c01e74+47"
 611         with self.assertRaises(apiclient.errors.HttpError):
 612             notfound = arv_put.api_client.collections().get(
 613                 uuid=manifest_uuid).execute()
 614
 615         datadir = self.make_tmpdir()
 616         with open(os.path.join(datadir, "foo"), "w") as f:
 617             f.write("The quick brown fox jumped over the lazy dog")
 618         p = subprocess.Popen([sys.executable, arv_put.__file__, datadir],
 619                              stdout=subprocess.PIPE, env=self.ENVIRON)
 620         (arvout, arverr) = p.communicate()
 621         self.assertEqual(arverr, None)
 622         self.assertEqual(p.returncode, 0)
 623
 624         # The manifest text stored in the API server under the same
 625         # manifest UUID must use signed locators.
 626         c = arv_put.api_client.collections().get(uuid=manifest_uuid).execute()
 627         self.assertRegexpMatches(
 628             c['manifest_text'],
 629             r'^\. 08a008a01d498c404b0c30852b39d3b8\+44\+A[0-9a-f]+@[0-9a-f]+ 0:44:foo\n')
 630
 631         os.remove(os.path.join(datadir, "foo"))
 632         os.rmdir(datadir)
 633
 634     def run_and_find_collection(self, text, extra_args=[]):
 635         self.authorize_with('active')
 636         pipe = subprocess.Popen(
 637             [sys.executable, arv_put.__file__] + extra_args,
 638             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
 639             stderr=subprocess.PIPE, env=self.ENVIRON)
 640         stdout, stderr = pipe.communicate(text)
 641         search_key = ('portable_data_hash'
 642                       if '--portable-data-hash' in extra_args else 'uuid')
 643         collection_list = arvados.api('v1').collections().list(
 644             filters=[[search_key, '=', stdout.strip()]]).execute().get('items', [])
 645         self.assertEqual(1, len(collection_list))
 646         return collection_list[0]
 647
 648     def test_put_collection_with_high_redundancy(self):
 649         # Write empty data: we're not testing CollectionWriter, just
 650         # making sure collections.create tells the API server what our
 651         # desired replication level is.
 652         collection = self.run_and_find_collection("", ['--replication', '4'])
 653         self.assertEqual(4, collection['replication_desired'])
 654
 655     def test_put_collection_with_default_redundancy(self):
 656         collection = self.run_and_find_collection("")
 657         self.assertEqual(None, collection['replication_desired'])
 658
 659     def test_put_collection_with_unnamed_project_link(self):
 660         link = self.run_and_find_collection(
 661             "Test unnamed collection",
 662             ['--portable-data-hash', '--project-uuid', self.PROJECT_UUID])
 663         username = pwd.getpwuid(os.getuid()).pw_name
 664         self.assertRegexpMatches(
 665             link['name'],
 666             r'^Saved at .* by {}@'.format(re.escape(username)))
 667
 668     def test_put_collection_with_name_and_no_project(self):
 669         link_name = 'Test Collection Link in home project'
 670         collection = self.run_and_find_collection(
 671             "Test named collection in home project",
 672             ['--portable-data-hash', '--name', link_name])
 673         self.assertEqual(link_name, collection['name'])
 674         my_user_uuid = self.current_user()['uuid']
 675         self.assertEqual(my_user_uuid, collection['owner_uuid'])
 676
 677     def test_put_collection_with_named_project_link(self):
 678         link_name = 'Test auto Collection Link'
 679         collection = self.run_and_find_collection("Test named collection",
 680                                       ['--portable-data-hash',
 681                                        '--name', link_name,
 682                                        '--project-uuid', self.PROJECT_UUID])
 683         self.assertEqual(link_name, collection['name'])
 684
 685
 686 if __name__ == '__main__':
 687     unittest.main()