2 # -*- coding: utf-8 -*-
16 from cStringIO import StringIO
19 import arvados.commands.put as arv_put
21 from arvados_testutil import ArvadosBaseTestCase
22 import run_test_server
24 class ArvadosPutResumeCacheTest(ArvadosBaseTestCase):
28 ['/dev/null', '--filename', 'empty'],
30 ['/tmp', '--max-manifest-depth', '0'],
31 ['/tmp', '--max-manifest-depth', '1']
35 super(ArvadosPutResumeCacheTest, self).tearDown()
37 self.last_cache.destroy()
38 except AttributeError:
41 def cache_path_from_arglist(self, arglist):
42 return arv_put.ResumeCache.make_path(arv_put.parse_arguments(arglist))
44 def test_cache_names_stable(self):
45 for argset in self.CACHE_ARGSET:
46 self.assertEquals(self.cache_path_from_arglist(argset),
47 self.cache_path_from_arglist(argset),
48 "cache name changed for {}".format(argset))
50 def test_cache_names_unique(self):
52 for argset in self.CACHE_ARGSET:
53 path = self.cache_path_from_arglist(argset)
54 self.assertNotIn(path, results)
57 def test_cache_names_simple(self):
58 # The goal here is to make sure the filename doesn't use characters
59 # reserved by the filesystem. Feel free to adjust this regexp as
60 # long as it still does that.
61 bad_chars = re.compile(r'[^-\.\w]')
62 for argset in self.CACHE_ARGSET:
63 path = self.cache_path_from_arglist(argset)
64 self.assertFalse(bad_chars.search(os.path.basename(path)),
65 "path too exotic: {}".format(path))
67 def test_cache_names_ignore_argument_order(self):
69 self.cache_path_from_arglist(['a', 'b', 'c']),
70 self.cache_path_from_arglist(['c', 'a', 'b']))
72 self.cache_path_from_arglist(['-', '--filename', 'stdin']),
73 self.cache_path_from_arglist(['--filename', 'stdin', '-']))
75 def test_cache_names_differ_for_similar_paths(self):
76 # This test needs names at / that don't exist on the real filesystem.
78 self.cache_path_from_arglist(['/_arvputtest1', '/_arvputtest2']),
79 self.cache_path_from_arglist(['/_arvputtest1/_arvputtest2']))
81 def test_cache_names_ignore_irrelevant_arguments(self):
82 # Workaround: parse_arguments bails on --filename with a directory.
83 path1 = self.cache_path_from_arglist(['/tmp'])
84 args = arv_put.parse_arguments(['/tmp'])
86 path2 = arv_put.ResumeCache.make_path(args)
87 self.assertEquals(path1, path2,
88 "cache path considered --filename for directory")
90 self.cache_path_from_arglist(['-']),
91 self.cache_path_from_arglist(['-', '--max-manifest-depth', '1']),
92 "cache path considered --max-manifest-depth for file")
94 def test_cache_names_treat_negative_manifest_depths_identically(self):
95 base_args = ['/tmp', '--max-manifest-depth']
97 self.cache_path_from_arglist(base_args + ['-1']),
98 self.cache_path_from_arglist(base_args + ['-2']))
100 def test_cache_names_treat_stdin_consistently(self):
102 self.cache_path_from_arglist(['-', '--filename', 'test']),
103 self.cache_path_from_arglist(['/dev/stdin', '--filename', 'test']))
105 def test_cache_names_identical_for_synonymous_names(self):
107 self.cache_path_from_arglist(['.']),
108 self.cache_path_from_arglist([os.path.realpath('.')]))
109 testdir = self.make_tmpdir()
110 looplink = os.path.join(testdir, 'loop')
111 os.symlink(testdir, looplink)
113 self.cache_path_from_arglist([testdir]),
114 self.cache_path_from_arglist([looplink]))
116 def test_cache_names_different_by_api_host(self):
117 config = arvados.config.settings()
118 orig_host = config.get('ARVADOS_API_HOST')
120 name1 = self.cache_path_from_arglist(['.'])
121 config['ARVADOS_API_HOST'] = 'x' + (orig_host or 'localhost')
122 self.assertNotEqual(name1, self.cache_path_from_arglist(['.']))
124 if orig_host is None:
125 del config['ARVADOS_API_HOST']
127 config['ARVADOS_API_HOST'] = orig_host
129 def test_basic_cache_storage(self):
130 thing = ['test', 'list']
131 with tempfile.NamedTemporaryFile() as cachefile:
132 self.last_cache = arv_put.ResumeCache(cachefile.name)
133 self.last_cache.save(thing)
134 self.assertEquals(thing, self.last_cache.load())
136 def test_empty_cache(self):
137 with tempfile.NamedTemporaryFile() as cachefile:
138 cache = arv_put.ResumeCache(cachefile.name)
139 self.assertRaises(ValueError, cache.load)
141 def test_cache_persistent(self):
142 thing = ['test', 'list']
143 path = os.path.join(self.make_tmpdir(), 'cache')
144 cache = arv_put.ResumeCache(path)
147 self.last_cache = arv_put.ResumeCache(path)
148 self.assertEquals(thing, self.last_cache.load())
150 def test_multiple_cache_writes(self):
151 thing = ['short', 'list']
152 with tempfile.NamedTemporaryFile() as cachefile:
153 self.last_cache = arv_put.ResumeCache(cachefile.name)
154 # Start writing an object longer than the one we test, to make
155 # sure the cache file gets truncated.
156 self.last_cache.save(['long', 'long', 'list'])
157 self.last_cache.save(thing)
158 self.assertEquals(thing, self.last_cache.load())
160 def test_cache_is_locked(self):
161 with tempfile.NamedTemporaryFile() as cachefile:
162 cache = arv_put.ResumeCache(cachefile.name)
163 self.assertRaises(arv_put.ResumeCacheConflict,
164 arv_put.ResumeCache, cachefile.name)
166 def test_cache_stays_locked(self):
167 with tempfile.NamedTemporaryFile() as cachefile:
168 self.last_cache = arv_put.ResumeCache(cachefile.name)
169 path = cachefile.name
170 self.last_cache.save('test')
171 self.assertRaises(arv_put.ResumeCacheConflict,
172 arv_put.ResumeCache, path)
174 def test_destroy_cache(self):
175 cachefile = tempfile.NamedTemporaryFile(delete=False)
177 cache = arv_put.ResumeCache(cachefile.name)
181 arv_put.ResumeCache(cachefile.name)
182 except arv_put.ResumeCacheConflict:
183 self.fail("could not load cache after destroying it")
184 self.assertRaises(ValueError, cache.load)
186 if os.path.exists(cachefile.name):
187 os.unlink(cachefile.name)
189 def test_restart_cache(self):
190 path = os.path.join(self.make_tmpdir(), 'cache')
191 cache = arv_put.ResumeCache(path)
194 self.assertRaises(ValueError, cache.load)
195 self.assertRaises(arv_put.ResumeCacheConflict,
196 arv_put.ResumeCache, path)
199 class ArvadosPutCollectionWriterTest(run_test_server.TestCaseWithServers,
200 ArvadosBaseTestCase):
202 super(ArvadosPutCollectionWriterTest, self).setUp()
203 run_test_server.authorize_with('active')
204 with tempfile.NamedTemporaryFile(delete=False) as cachefile:
205 self.cache = arv_put.ResumeCache(cachefile.name)
206 self.cache_filename = cachefile.name
209 super(ArvadosPutCollectionWriterTest, self).tearDown()
210 if os.path.exists(self.cache_filename):
214 def test_writer_caches(self):
215 cwriter = arv_put.ArvPutCollectionWriter(self.cache)
216 cwriter.write_file('/dev/null')
217 cwriter.cache_state()
218 self.assertTrue(self.cache.load())
219 self.assertEquals(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text())
221 def test_writer_works_without_cache(self):
222 cwriter = arv_put.ArvPutCollectionWriter()
223 cwriter.write_file('/dev/null')
224 self.assertEquals(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text())
226 def test_writer_resumes_from_cache(self):
227 cwriter = arv_put.ArvPutCollectionWriter(self.cache)
228 with self.make_test_file() as testfile:
229 cwriter.write_file(testfile.name, 'test')
230 cwriter.cache_state()
231 new_writer = arv_put.ArvPutCollectionWriter.from_cache(
234 ". 098f6bcd4621d373cade4e832627b4f6+4 0:4:test\n",
235 new_writer.manifest_text())
237 def test_new_writer_from_stale_cache(self):
238 cwriter = arv_put.ArvPutCollectionWriter(self.cache)
239 with self.make_test_file() as testfile:
240 cwriter.write_file(testfile.name, 'test')
241 new_writer = arv_put.ArvPutCollectionWriter.from_cache(self.cache)
242 new_writer.write_file('/dev/null')
243 self.assertEquals(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", new_writer.manifest_text())
245 def test_new_writer_from_empty_cache(self):
246 cwriter = arv_put.ArvPutCollectionWriter.from_cache(self.cache)
247 cwriter.write_file('/dev/null')
248 self.assertEquals(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text())
250 def test_writer_resumable_after_arbitrary_bytes(self):
251 cwriter = arv_put.ArvPutCollectionWriter(self.cache)
252 # These bytes are intentionally not valid UTF-8.
253 with self.make_test_file('\x00\x07\xe2') as testfile:
254 cwriter.write_file(testfile.name, 'test')
255 cwriter.cache_state()
256 new_writer = arv_put.ArvPutCollectionWriter.from_cache(
258 self.assertEquals(cwriter.manifest_text(), new_writer.manifest_text())
260 def make_progress_tester(self):
262 def record_func(written, expected):
263 progression.append((written, expected))
264 return progression, record_func
266 def test_progress_reporting(self):
267 for expect_count in (None, 8):
268 progression, reporter = self.make_progress_tester()
269 cwriter = arv_put.ArvPutCollectionWriter(
270 reporter=reporter, bytes_expected=expect_count)
271 with self.make_test_file() as testfile:
272 cwriter.write_file(testfile.name, 'test')
273 cwriter.finish_current_stream()
274 self.assertIn((4, expect_count), progression)
276 def test_resume_progress(self):
277 cwriter = arv_put.ArvPutCollectionWriter(self.cache, bytes_expected=4)
278 with self.make_test_file() as testfile:
279 # Set up a writer with some flushed bytes.
280 cwriter.write_file(testfile.name, 'test')
281 cwriter.finish_current_stream()
282 cwriter.cache_state()
283 new_writer = arv_put.ArvPutCollectionWriter.from_cache(self.cache)
284 self.assertEqual(new_writer.bytes_written, 4)
287 class ArvadosExpectedBytesTest(ArvadosBaseTestCase):
288 TEST_SIZE = os.path.getsize(__file__)
290 def test_expected_bytes_for_file(self):
291 self.assertEquals(self.TEST_SIZE,
292 arv_put.expected_bytes_for([__file__]))
294 def test_expected_bytes_for_tree(self):
295 tree = self.make_tmpdir()
296 shutil.copyfile(__file__, os.path.join(tree, 'one'))
297 shutil.copyfile(__file__, os.path.join(tree, 'two'))
298 self.assertEquals(self.TEST_SIZE * 2,
299 arv_put.expected_bytes_for([tree]))
300 self.assertEquals(self.TEST_SIZE * 3,
301 arv_put.expected_bytes_for([tree, __file__]))
303 def test_expected_bytes_for_device(self):
304 self.assertIsNone(arv_put.expected_bytes_for(['/dev/null']))
305 self.assertIsNone(arv_put.expected_bytes_for([__file__, '/dev/null']))
308 class ArvadosPutReportTest(ArvadosBaseTestCase):
309 def test_machine_progress(self):
310 for count, total in [(0, 1), (0, None), (1, None), (235, 9283)]:
311 expect = ": {} written {} total\n".format(
312 count, -1 if (total is None) else total)
314 arv_put.machine_progress(count, total).endswith(expect))
316 def test_known_human_progress(self):
317 for count, total in [(0, 1), (2, 4), (45, 60)]:
318 expect = '{:.1%}'.format(float(count) / total)
319 actual = arv_put.human_progress(count, total)
320 self.assertTrue(actual.startswith('\r'))
321 self.assertIn(expect, actual)
323 def test_unknown_human_progress(self):
324 for count in [1, 20, 300, 4000, 50000]:
325 self.assertTrue(re.search(r'\b{}\b'.format(count),
326 arv_put.human_progress(count, None)))
329 class ArvadosPutProjectLinkTest(ArvadosBaseTestCase):
330 Z_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
333 self.stderr = StringIO()
334 super(ArvadosPutProjectLinkTest, self).setUp()
338 super(ArvadosPutProjectLinkTest, self).tearDown()
340 def prep_link_from_arguments(self, args, uuid_found=True):
342 link = arv_put.prep_project_link(arv_put.parse_arguments(args),
344 lambda uuid: uuid_found)
349 def check_link(self, link, project_uuid, link_name=None):
350 self.assertEqual(project_uuid, link.get('tail_uuid'))
351 self.assertEqual(project_uuid, link.get('owner_uuid'))
352 self.assertEqual('name', link.get('link_class'))
353 if link_name is None:
354 self.assertNotIn('name', link)
356 self.assertEqual(link_name, link.get('name'))
357 self.assertNotIn('head_uuid', link)
359 def check_stderr_empty(self):
360 self.assertEqual('', self.stderr.getvalue())
362 def test_project_link_with_name(self):
363 link = self.prep_link_from_arguments(['--project-uuid', self.Z_UUID,
364 '--name', 'test link AAA'])
365 self.check_link(link, self.Z_UUID, 'test link AAA')
366 self.check_stderr_empty()
368 def test_project_link_without_name(self):
369 username = pwd.getpwuid(os.getuid()).pw_name
370 link = self.prep_link_from_arguments(['--project-uuid', self.Z_UUID])
371 self.assertIsNotNone(link.get('name', None))
372 self.assertRegexpMatches(
374 r'^Saved at .* by {}@'.format(re.escape(username)))
375 self.check_link(link, self.Z_UUID, link.get('name', None))
376 for line in self.stderr:
377 if "No --name specified" in line:
380 self.fail("no warning emitted about the lack of collection name")
382 @unittest.skip("prep_project_link needs an API lookup for this case")
383 def test_collection_without_project_defaults_to_home(self):
384 link = self.prep_link_from_arguments(['--name', 'test link BBB'])
385 self.check_link(link, self.Z_UUID)
386 self.check_stderr_empty()
388 def test_no_link_or_warning_with_no_collection(self):
389 self.assertIsNone(self.prep_link_from_arguments(['--raw']))
390 self.check_stderr_empty()
392 def test_error_when_project_not_found(self):
393 self.assertRaises(ValueError,
394 self.prep_link_from_arguments,
395 ['--project-uuid', self.Z_UUID], False)
397 def test_link_without_collection_is_error(self):
398 self.assertRaises(ValueError,
399 self.prep_link_from_arguments,
400 ['--project-uuid', self.Z_UUID, '--stream'])
403 class ArvadosPutTest(run_test_server.TestCaseWithServers, ArvadosBaseTestCase):
406 def call_main_with_args(self, args):
407 self.main_stdout = StringIO()
408 self.main_stderr = StringIO()
409 return arv_put.main(args, self.main_stdout, self.main_stderr)
411 def call_main_on_test_file(self):
412 with self.make_test_file() as testfile:
414 self.call_main_with_args(['--stream', '--no-progress', path])
416 os.path.exists(os.path.join(os.environ['KEEP_LOCAL_STORE'],
417 '098f6bcd4621d373cade4e832627b4f6')),
418 "did not find file stream in Keep store")
421 super(ArvadosPutTest, self).setUp()
422 run_test_server.authorize_with('active')
423 arv_put.api_client = None
426 for outbuf in ['main_stdout', 'main_stderr']:
427 if hasattr(self, outbuf):
428 getattr(self, outbuf).close()
429 delattr(self, outbuf)
430 super(ArvadosPutTest, self).tearDown()
432 def test_simple_file_put(self):
433 self.call_main_on_test_file()
435 def test_put_with_unwriteable_cache_dir(self):
436 orig_cachedir = arv_put.ResumeCache.CACHE_DIR
437 cachedir = self.make_tmpdir()
438 os.chmod(cachedir, 0o0)
439 arv_put.ResumeCache.CACHE_DIR = cachedir
441 self.call_main_on_test_file()
443 arv_put.ResumeCache.CACHE_DIR = orig_cachedir
444 os.chmod(cachedir, 0o700)
446 def test_put_with_unwritable_cache_subdir(self):
447 orig_cachedir = arv_put.ResumeCache.CACHE_DIR
448 cachedir = self.make_tmpdir()
449 os.chmod(cachedir, 0o0)
450 arv_put.ResumeCache.CACHE_DIR = os.path.join(cachedir, 'cachedir')
452 self.call_main_on_test_file()
454 arv_put.ResumeCache.CACHE_DIR = orig_cachedir
455 os.chmod(cachedir, 0o700)
457 def test_link_without_collection_aborts(self):
458 self.assertRaises(SystemExit, self.call_main_with_args,
459 ['--name', 'test without Collection',
460 '--stream', '/dev/null'])
462 class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
463 ArvadosBaseTestCase):
464 def _getKeepServerConfig():
465 for config_file in ['application.yml', 'application.default.yml']:
466 with open(os.path.join(run_test_server.SERVICES_SRC_DIR,
467 "api", "config", config_file)) as f:
468 rails_config = yaml.load(f.read())
469 for config_section in ['test', 'common']:
471 key = rails_config[config_section]["blob_signing_key"]
472 except (KeyError, TypeError):
475 return {'blob_signing_key': key,
476 'enforce_permissions': True}
477 return {'blog_signing_key': None, 'enforce_permissions': False}
480 KEEP_SERVER = _getKeepServerConfig()
481 PROJECT_UUID = run_test_server.fixture('groups')['aproject']['uuid']
485 super(ArvPutIntegrationTest, cls).setUpClass()
486 cls.ENVIRON = os.environ.copy()
487 cls.ENVIRON['PYTHONPATH'] = ':'.join(sys.path)
490 super(ArvPutIntegrationTest, self).setUp()
491 arv_put.api_client = None
493 def authorize_with(self, token_name):
494 run_test_server.authorize_with(token_name)
495 for v in ["ARVADOS_API_HOST",
496 "ARVADOS_API_HOST_INSECURE",
497 "ARVADOS_API_TOKEN"]:
498 self.ENVIRON[v] = arvados.config.settings()[v]
499 arv_put.api_client = arvados.api('v1', cache=False)
501 def current_user(self):
502 return arv_put.api_client.users().current().execute()
504 def test_check_real_project_found(self):
505 self.authorize_with('active')
506 self.assertTrue(arv_put.check_project_exists(self.PROJECT_UUID),
507 "did not correctly find test fixture project")
509 def test_check_error_finding_nonexistent_project(self):
510 BAD_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
511 self.authorize_with('active')
513 result = arv_put.check_project_exists(BAD_UUID)
514 except ValueError as error:
515 self.assertIn(BAD_UUID, error.message)
517 self.assertFalse(result, "incorrectly found nonexistent project")
519 def test_short_put_from_stdin(self):
520 # Have to run this as an integration test since arv-put can't
521 # read from the tests' stdin.
522 # arv-put usually can't stat(os.path.realpath('/dev/stdin')) in this
523 # case, because the /proc entry is already gone by the time it tries.
524 pipe = subprocess.Popen(
525 [sys.executable, arv_put.__file__, '--stream'],
526 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
527 stderr=subprocess.STDOUT, env=self.ENVIRON)
528 pipe.stdin.write('stdin test\n')
530 deadline = time.time() + 5
531 while (pipe.poll() is None) and (time.time() < deadline):
533 returncode = pipe.poll()
534 if returncode is None:
536 self.fail("arv-put did not PUT from stdin within 5 seconds")
537 elif returncode != 0:
538 sys.stdout.write(pipe.stdout.read())
539 self.fail("arv-put returned exit code {}".format(returncode))
540 self.assertIn('4a9c8b735dce4b5fa3acf221a0b13628+11', pipe.stdout.read())
542 def test_ArvPutSignedManifest(self):
543 # ArvPutSignedManifest runs "arv-put foo" and then attempts to get
544 # the newly created manifest from the API server, testing to confirm
545 # that the block locators in the returned manifest are signed.
546 self.authorize_with('active')
548 # Before doing anything, demonstrate that the collection
549 # we're about to create is not present in our test fixture.
550 manifest_uuid = "00b4e9f40ac4dd432ef89749f1c01e74+47"
551 with self.assertRaises(apiclient.errors.HttpError):
552 notfound = arv_put.api_client.collections().get(
553 uuid=manifest_uuid).execute()
555 datadir = self.make_tmpdir()
556 with open(os.path.join(datadir, "foo"), "w") as f:
557 f.write("The quick brown fox jumped over the lazy dog")
558 p = subprocess.Popen([sys.executable, arv_put.__file__, datadir],
559 stdout=subprocess.PIPE, env=self.ENVIRON)
560 (arvout, arverr) = p.communicate()
561 self.assertEqual(p.returncode, 0)
562 self.assertEqual(arverr, None)
563 self.assertEqual(arvout.strip(), manifest_uuid)
565 # The manifest text stored in the API server under the same
566 # manifest UUID must use signed locators.
567 c = arv_put.api_client.collections().get(uuid=manifest_uuid).execute()
568 self.assertRegexpMatches(
570 r'^\. 08a008a01d498c404b0c30852b39d3b8\+44\+A[0-9a-f]+@[0-9a-f]+ 0:44:foo\n')
572 def run_and_find_link(self, text, extra_args=[]):
573 self.authorize_with('active')
574 pipe = subprocess.Popen(
575 [sys.executable, arv_put.__file__] + extra_args,
576 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
577 stderr=subprocess.PIPE, env=self.ENVIRON)
578 stdout, stderr = pipe.communicate(text)
579 link_list = arv_put.api_client.links().list(
580 filters=[['head_uuid', '=', stdout.strip()],
581 ['link_class', '=', 'name']]).execute().get('items', [])
582 self.assertEqual(1, len(link_list))
585 def test_put_collection_with_unnamed_project_link(self):
586 link = self.run_and_find_link("Test unnamed collection",
587 ['--project-uuid', self.PROJECT_UUID])
588 username = pwd.getpwuid(os.getuid()).pw_name
589 self.assertRegexpMatches(
591 r'^Saved at .* by {}@'.format(re.escape(username)))
593 def test_put_collection_with_name_and_no_project(self):
594 link_name = 'Test Collection Link in home project'
595 link = self.run_and_find_link("Test named collection in home project",
596 ['--name', link_name])
597 self.assertEqual(link_name, link['name'])
598 my_user_uuid = self.current_user()['uuid']
599 self.assertEqual(my_user_uuid, link['tail_uuid'])
600 self.assertEqual(my_user_uuid, link['owner_uuid'])
602 def test_put_collection_with_named_project_link(self):
603 link_name = 'Test auto Collection Link'
604 link = self.run_and_find_link("Test named collection",
605 ['--name', link_name,
606 '--project-uuid', self.PROJECT_UUID])
607 self.assertEqual(link_name, link['name'])
610 if __name__ == '__main__':