1 require "arvados/collection"
2 require "minitest/autorun"
5 class CollectionTest < Minitest::Test
8 TWO_BY_TWO_BLOCKS = SDKFixtures.random_blocks(2, 9)
9 TWO_BY_TWO_MANIFEST_A =
10 [". #{TWO_BY_TWO_BLOCKS.first} 0:5:f1 5:4:f2\n",
11 "./s1 #{TWO_BY_TWO_BLOCKS.last} 0:5:f1 5:4:f3\n"]
12 TWO_BY_TWO_MANIFEST_S = TWO_BY_TWO_MANIFEST_A.join("")
16 def test_empty_construction
17 coll = Arv::Collection.new
18 assert_equal("", coll.manifest_text)
21 def test_successful_construction
22 [:SIMPLEST_MANIFEST, :MULTIBLOCK_FILE_MANIFEST, :MULTILEVEL_MANIFEST].
23 each do |manifest_name|
24 manifest_text = SDKFixtures.const_get(manifest_name)
25 coll = Arv::Collection.new(manifest_text)
26 assert_equal(manifest_text, coll.manifest_text,
27 "did not get same manifest back out from #{manifest_name}")
31 def test_non_manifest_construction_error
32 ["word", ". abc def", ". #{random_block} 0:", ". / !"].each do |m_text|
33 assert_raises(ArgumentError,
34 "built collection from manifest #{m_text.inspect}") do
35 Arv::Collection.new(m_text)
40 def test_file_directory_conflict_construction_error
41 assert_raises(ArgumentError) do
42 Arv::Collection.new(NAME_CONFLICT_MANIFEST)
46 def test_no_implicit_normalization
47 coll = Arv::Collection.new(NONNORMALIZED_MANIFEST)
48 assert_equal(NONNORMALIZED_MANIFEST, coll.manifest_text)
53 def test_non_posix_path_handling
54 m_text = "./.. #{random_block(9)} 0:5:. 5:4:..\n"
55 coll = Arv::Collection.new(m_text.dup)
57 assert_equal(m_text, coll.manifest_text)
60 def test_escaping_through_normalization
61 coll = Arv::Collection.new(MANY_ESCAPES_MANIFEST)
63 # The result should simply duplicate the file spec.
64 # The source file spec has an unescaped backslash in it.
65 # It's OK for the Collection class to properly escape that.
66 expect_text = MANY_ESCAPES_MANIFEST.sub(/ \d+:\d+:\S+/) do |file_spec|
67 file_spec.gsub(/([^\\])(\\[^\\\d])/, '\1\\\\\2')
69 assert_equal(expect_text, coll.manifest_text)
72 def test_concatenation_with_locator_overlap(over_index=0)
73 blocks = random_blocks(4, 2)
74 blocks_s = blocks.join(" ")
75 coll = Arv::Collection.new(". %s 0:8:file\n. %s 0:4:file\n" %
76 [blocks_s, blocks[over_index, 2].join(" ")])
78 assert_equal(". #{blocks_s} 0:8:file #{over_index * 2}:4:file\n",
82 def test_concatenation_with_middle_locator_overlap
83 test_concatenation_with_locator_overlap(1)
86 def test_concatenation_with_end_locator_overlap
87 test_concatenation_with_locator_overlap(2)
90 def test_concatenation_with_partial_locator_overlap
91 blocks = random_blocks(3, 3)
92 coll = Arv::Collection
93 .new(". %s 0:6:overlap\n. %s 0:6:overlap\n" %
94 [blocks[0, 2].join(" "), blocks[1, 2].join(" ")])
96 assert_equal(". #{blocks.join(' ')} 0:6:overlap 3:6:overlap\n",
102 coll = Arv::Collection.new(". #{block} 0:0:f2 0:0:f1\n")
104 assert_equal(". #{block} 0:0:f1 0:0:f2\n", coll.manifest_text)
107 def test_normalization_file_spans_two_whole_blocks(file_specs="0:10:f1",
109 blocks = random_blocks(num_blocks, 5)
110 m_text = ". #{blocks.join(' ')} #{file_specs}\n"
111 coll = Arv::Collection.new(m_text.dup)
113 assert_equal(m_text, coll.manifest_text)
116 def test_normalization_file_fits_beginning_block
117 test_normalization_file_spans_two_whole_blocks("0:7:f1")
120 def test_normalization_file_fits_end_block
121 test_normalization_file_spans_two_whole_blocks("3:7:f1")
124 def test_normalization_file_spans_middle
125 test_normalization_file_spans_two_whole_blocks("3:5:f1")
128 def test_normalization_file_spans_three_whole_blocks
129 test_normalization_file_spans_two_whole_blocks("0:15:f1", 3)
132 def test_normalization_file_skips_bytes
133 test_normalization_file_spans_two_whole_blocks("0:3:f1 5:5:f1")
136 def test_normalization_file_inserts_bytes
137 test_normalization_file_spans_two_whole_blocks("0:3:f1 5:3:f1 3:2:f1")
140 def test_normalization_file_duplicates_bytes
141 test_normalization_file_spans_two_whole_blocks("2:3:f1 2:3:f1", 1)
144 def test_normalization_dedups_locators
145 blocks = random_blocks(2, 5)
146 coll = Arv::Collection.new(". %s %s 1:8:f1 11:8:f1\n" %
147 [blocks.join(" "), blocks.reverse.join(" ")])
149 assert_equal(". #{blocks.join(' ')} 1:8:f1 6:4:f1 0:4:f1\n",
155 def test_simple_file_copy
156 coll = Arv::Collection.new(SIMPLEST_MANIFEST)
157 coll.cp_r("./simple.txt", "./new")
158 assert_equal(SIMPLEST_MANIFEST.sub(" 0:9:", " 0:9:new 0:9:"),
162 def test_copy_file_into_other_stream(target="./s1/f2", basename="f2")
163 coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
164 coll.cp_r("./f2", target)
165 expected = "%s./s1 %s 0:5:f1 14:4:%s 5:4:f3\n" %
166 [TWO_BY_TWO_MANIFEST_A.first,
167 TWO_BY_TWO_BLOCKS.reverse.join(" "), basename]
168 assert_equal(expected, coll.manifest_text)
171 def test_implicit_copy_file_into_other_stream
172 test_copy_file_into_other_stream("./s1")
175 def test_copy_file_into_other_stream_with_new_name
176 test_copy_file_into_other_stream("./s1/f2a", "f2a")
179 def test_copy_file_over_in_other_stream(target="./s1/f1")
180 coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
181 coll.cp_r("./f1", target)
182 expected = "%s./s1 %s 0:5:f1 14:4:f3\n" %
183 [TWO_BY_TWO_MANIFEST_A.first, TWO_BY_TWO_BLOCKS.join(" ")]
184 assert_equal(expected, coll.manifest_text)
187 def test_implicit_copy_file_over_in_other_stream
188 test_copy_file_over_in_other_stream("./s1")
191 def test_simple_stream_copy
192 coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
193 coll.cp_r("./s1", "./sNew")
194 new_line = TWO_BY_TWO_MANIFEST_A.last.sub("./s1 ", "./sNew ")
195 assert_equal(TWO_BY_TWO_MANIFEST_S + new_line, coll.manifest_text)
198 def test_copy_stream_into_other_stream(target="./dir2/subdir",
200 coll = Arv::Collection.new(MULTILEVEL_MANIFEST)
201 coll.cp_r("./dir1/subdir", target)
202 new_line = MULTILEVEL_MANIFEST.lines[4].sub("./dir1/subdir ",
203 "./dir2/#{basename} ")
204 assert_equal(MULTILEVEL_MANIFEST + new_line, coll.manifest_text)
207 def test_implicit_copy_stream_into_other_stream
208 test_copy_stream_into_other_stream("./dir2")
211 def test_copy_stream_into_other_stream_with_new_name
212 test_copy_stream_into_other_stream("./dir2/newsub", "newsub")
215 def test_copy_stream_over_empty_stream
216 coll = Arv::Collection.new(MULTILEVEL_MANIFEST)
217 (1..3).each do |file_num|
218 coll.rm("./dir0/subdir/file#{file_num}")
220 coll.cp_r("./dir1/subdir", "./dir0")
221 expected = MULTILEVEL_MANIFEST.lines
222 expected[2] = expected[4].sub("./dir1/", "./dir0/")
223 assert_equal(expected.join(""), coll.manifest_text)
226 def test_copy_stream_over_file_raises_ENOTDIR
227 coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
228 assert_raises(Errno::ENOTDIR) do
229 coll.cp_r("./s1", "./f2")
233 def test_copy_stream_over_nonempty_stream_merges_and_overwrites
234 blocks = random_blocks(3, 9)
236 ["./subdir #{blocks[0]} 0:1:s1 1:2:zero\n",
237 "./zdir #{blocks[1]} 0:9:zfile\n",
238 "./zdir/subdir #{blocks[2]} 0:1:s2 1:2:zero\n"]
239 coll = Arv::Collection.new(manifest_a.join(""))
240 coll.cp_r("./subdir", "./zdir")
241 manifest_a[2] = "./zdir/subdir %s %s 0:1:s1 9:1:s2 1:2:zero\n" %
242 [blocks[0], blocks[2]]
243 assert_equal(manifest_a.join(""), coll.manifest_text)
246 def test_copy_stream_into_substream(source="./dir1",
247 target="./dir1/subdir/dir1")
248 coll = Arv::Collection.new(MULTILEVEL_MANIFEST)
249 coll.cp_r(source, target)
250 expected = MULTILEVEL_MANIFEST.lines.flat_map do |line|
251 [line, line.gsub(/^#{Regexp.escape(source)}([\/ ])/, "#{target}\\1")].uniq
253 assert_equal(expected.sort.join(""), coll.manifest_text)
257 test_copy_stream_into_substream(".", "./root")
260 def test_adding_to_root_after_copy
261 coll = Arv::Collection.new(SIMPLEST_MANIFEST)
262 coll.cp_r(".", "./root")
263 src_coll = Arv::Collection.new(COLON_FILENAME_MANIFEST)
264 coll.cp_r("./file:test.txt", ".", src_coll)
265 got_lines = coll.manifest_text.lines
266 assert_equal(2, got_lines.size)
267 assert_match(/^\. \S{33,} \S{33,} 0:9:file:test\.txt 9:9:simple\.txt\n/,
269 assert_equal(SIMPLEST_MANIFEST.sub(". ", "./root "), got_lines.last)
272 def test_copy_chaining
273 coll = Arv::Collection.new(SIMPLEST_MANIFEST)
274 coll.cp_r("./simple.txt", "./a").cp_r("./a", "./b")
275 assert_equal(SIMPLEST_MANIFEST.sub(" 0:9:", " 0:9:a 0:9:b 0:9:"),
279 def prep_two_collections_for_copy(src_stream, dst_stream)
280 blocks = random_blocks(2, 8)
281 src_text = "#{src_stream} #{blocks.first} 0:8:f1\n"
282 dst_text = "#{dst_stream} #{blocks.last} 0:8:f2\n"
283 return [blocks, src_text, dst_text,
284 Arv::Collection.new(src_text.dup),
285 Arv::Collection.new(dst_text.dup)]
288 def test_copy_file_from_other_collection(src_stream=".", dst_stream="./s1")
289 blocks, src_text, dst_text, src_coll, dst_coll =
290 prep_two_collections_for_copy(src_stream, dst_stream)
291 dst_coll.cp_r("#{src_stream}/f1", dst_stream, src_coll)
292 assert_equal("#{dst_stream} #{blocks.join(' ')} 0:8:f1 8:8:f2\n",
293 dst_coll.manifest_text)
294 assert_equal(src_text, src_coll.manifest_text)
297 def test_copy_file_from_other_collection_to_root
298 test_copy_file_from_other_collection("./s1", ".")
301 def test_copy_stream_from_other_collection
302 blocks, src_text, dst_text, src_coll, dst_coll =
303 prep_two_collections_for_copy("./s2", "./s1")
304 dst_coll.cp_r("./s2", "./s1", src_coll)
305 assert_equal(dst_text + src_text.sub("./s2 ", "./s1/s2 "),
306 dst_coll.manifest_text)
307 assert_equal(src_text, src_coll.manifest_text)
310 def test_copy_stream_from_other_collection_to_root
311 blocks, src_text, dst_text, src_coll, dst_coll =
312 prep_two_collections_for_copy("./s1", ".")
313 dst_coll.cp_r("./s1", ".", src_coll)
314 assert_equal(dst_text + src_text, dst_coll.manifest_text)
315 assert_equal(src_text, src_coll.manifest_text)
318 def test_copy_stream_contents
319 coll = Arv::Collection.new(MULTILEVEL_MANIFEST)
320 coll.cp_r("./dir0/subdir/", "./dir1/subdir")
321 expect_lines = MULTILEVEL_MANIFEST.lines
322 expect_lines[4] = expect_lines[2].sub("./dir0/", "./dir1/")
323 assert_equal(expect_lines.join(""), coll.manifest_text)
326 def test_copy_stream_contents_into_root
327 coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
328 coll.cp_r("./s1/", ".")
329 assert_equal(". %s 0:5:f1 14:4:f2 5:4:f3\n%s" %
330 [TWO_BY_TWO_BLOCKS.reverse.join(" "),
331 TWO_BY_TWO_MANIFEST_A.last],
335 def test_copy_root_contents_into_stream
336 # This is especially fun, because we're copying a parent into its child.
337 # Make sure that happens depth-first.
338 coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
339 coll.cp_r("./", "./s1")
340 assert_equal("%s./s1 %s 0:5:f1 5:4:f2 14:4:f3\n%s" %
341 [TWO_BY_TWO_MANIFEST_A.first, TWO_BY_TWO_BLOCKS.join(" "),
342 TWO_BY_TWO_MANIFEST_A.last.sub("./s1 ", "./s1/s1 ")],
346 def test_copy_stream_contents_across_collections
347 block = random_block(8)
348 src_coll = Arv::Collection.new("./s1 #{block} 0:8:f1\n")
349 dst_coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
350 dst_coll.cp_r("./s1/", "./s1", src_coll)
351 assert_equal("%s./s1 %s %s 0:8:f1 13:4:f3\n" %
352 [TWO_BY_TWO_MANIFEST_A.first, block, TWO_BY_TWO_BLOCKS.last],
353 dst_coll.manifest_text)
356 def test_copy_root_contents_across_collections
357 block = random_block(8)
358 src_coll = Arv::Collection.new(". #{block} 0:8:f1\n")
359 dst_coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
360 dst_coll.cp_r("./", ".", src_coll)
361 assert_equal(". %s %s 0:8:f1 13:4:f2\n%s" %
362 [block, TWO_BY_TWO_BLOCKS.first, TWO_BY_TWO_MANIFEST_A.last],
363 dst_coll.manifest_text)
366 def test_copy_empty_source_path_raises_ArgumentError(src="", dst="./s1")
367 coll = Arv::Collection.new(SIMPLEST_MANIFEST)
368 assert_raises(ArgumentError) do
373 def test_copy_empty_destination_path_raises_ArgumentError
374 test_copy_empty_source_path_raises_ArgumentError(".", "")
379 def test_simple_file_rename
380 coll = Arv::Collection.new(SIMPLEST_MANIFEST)
381 coll.rename("./simple.txt", "./new")
382 assert_equal(SIMPLEST_MANIFEST.sub(":simple.txt", ":new"),
386 def test_rename_file_into_other_stream(target="./s1/f2", basename="f2")
387 coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
388 coll.rename("./f2", target)
389 expected = ". %s 0:5:f1\n./s1 %s 0:5:f1 14:4:%s 5:4:f3\n" %
390 [TWO_BY_TWO_BLOCKS.first,
391 TWO_BY_TWO_BLOCKS.reverse.join(" "), basename]
392 assert_equal(expected, coll.manifest_text)
395 def test_implicit_rename_file_into_other_stream
396 test_rename_file_into_other_stream("./s1")
399 def test_rename_file_into_other_stream_with_new_name
400 test_rename_file_into_other_stream("./s1/f2a", "f2a")
403 def test_rename_file_over_in_other_stream(target="./s1/f1")
404 coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
405 coll.rename("./f1", target)
406 expected = ". %s 5:4:f2\n./s1 %s 0:5:f1 14:4:f3\n" %
407 [TWO_BY_TWO_BLOCKS.first, TWO_BY_TWO_BLOCKS.join(" ")]
408 assert_equal(expected, coll.manifest_text)
411 def test_implicit_rename_file_over_in_other_stream
412 test_rename_file_over_in_other_stream("./s1")
415 def test_simple_stream_rename
416 coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
417 coll.rename("./s1", "./newS")
418 assert_equal(TWO_BY_TWO_MANIFEST_S.sub("\n./s1 ", "\n./newS "),
422 def test_rename_stream_into_other_stream(target="./dir2/subdir",
424 coll = Arv::Collection.new(MULTILEVEL_MANIFEST)
425 coll.rename("./dir1/subdir", target)
426 expected = MULTILEVEL_MANIFEST.lines
427 replaced_line = expected.delete_at(4)
428 expected << replaced_line.sub("./dir1/subdir ", "./dir2/#{basename} ")
429 assert_equal(expected.join(""), coll.manifest_text)
432 def test_implicit_rename_stream_into_other_stream
433 test_rename_stream_into_other_stream("./dir2")
436 def test_rename_stream_into_other_stream_with_new_name
437 test_rename_stream_into_other_stream("./dir2/newsub", "newsub")
440 def test_rename_stream_over_empty_stream
441 coll = Arv::Collection.new(MULTILEVEL_MANIFEST)
442 (1..3).each do |file_num|
443 coll.rm("./dir0/subdir/file#{file_num}")
445 coll.rename("./dir1/subdir", "./dir0")
446 expected = MULTILEVEL_MANIFEST.lines
447 expected[2] = expected.delete_at(4).sub("./dir1/", "./dir0/")
448 assert_equal(expected.sort.join(""), coll.manifest_text)
451 def test_rename_stream_over_file_raises_ENOTDIR
452 coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
453 assert_raises(Errno::ENOTDIR) do
454 coll.rename("./s1", "./f2")
458 def test_rename_stream_over_nonempty_stream_raises_ENOTEMPTY
459 coll = Arv::Collection.new(MULTILEVEL_MANIFEST)
460 assert_raises(Errno::ENOTEMPTY) do
461 coll.rename("./dir1/subdir", "./dir0")
465 def test_rename_stream_into_substream(source="./dir1",
466 target="./dir1/subdir/dir1")
467 coll = Arv::Collection.new(MULTILEVEL_MANIFEST)
468 coll.rename(source, target)
469 assert_equal(MULTILEVEL_MANIFEST.gsub(/^#{Regexp.escape(source)}([\/ ])/m,
475 test_rename_stream_into_substream(".", "./root")
478 def test_adding_to_root_after_rename
479 coll = Arv::Collection.new(SIMPLEST_MANIFEST)
480 coll.rename(".", "./root")
481 src_coll = Arv::Collection.new(SIMPLEST_MANIFEST)
482 coll.cp_r("./simple.txt", ".", src_coll)
483 assert_equal(SIMPLEST_MANIFEST + SIMPLEST_MANIFEST.sub(". ", "./root "),
487 def test_rename_chaining
488 coll = Arv::Collection.new(SIMPLEST_MANIFEST)
489 coll.rename("./simple.txt", "./x").rename("./x", "./simple.txt")
490 assert_equal(SIMPLEST_MANIFEST, coll.manifest_text)
495 def test_simple_remove
496 coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S.dup)
498 assert_equal(TWO_BY_TWO_MANIFEST_S.sub(" 5:4:f2", ""), coll.manifest_text)
501 def empty_stream_and_assert(expect_index=0)
502 coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
504 assert_equal(TWO_BY_TWO_MANIFEST_A[expect_index], coll.manifest_text)
507 def test_remove_all_files_in_substream
508 empty_stream_and_assert do |coll|
514 def test_remove_all_files_in_root_stream
515 empty_stream_and_assert(1) do |coll|
521 def test_chaining_removes
522 empty_stream_and_assert do |coll|
523 coll.rm("./s1/f1").rm("./s1/f3")
527 def test_remove_last_file
528 coll = Arv::Collection.new(SIMPLEST_MANIFEST)
529 coll.rm("./simple.txt")
530 assert_equal("", coll.manifest_text)
533 def test_remove_nonexistent_file_raises_ENOENT(path="./NoSuchFile",
535 coll = Arv::Collection.new(SIMPLEST_MANIFEST)
536 assert_raises(Errno::ENOENT) do
537 coll.send(method, path)
541 def test_remove_from_nonexistent_stream_raises_ENOENT
542 test_remove_nonexistent_file_raises_ENOENT("./NoSuchStream/simple.txt")
545 def test_remove_stream_raises_EISDIR(path="./s1")
546 coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
547 assert_raises(Errno::EISDIR) do
552 def test_remove_root_raises_EISDIR
553 test_remove_stream_raises_EISDIR(".")
556 def test_remove_empty_string_raises_ArgumentError
557 coll = Arv::Collection.new(SIMPLEST_MANIFEST)
558 assert_raises(ArgumentError) do
565 def test_recursive_remove
566 empty_stream_and_assert do |coll|
571 def test_recursive_remove_on_files
572 empty_stream_and_assert do |coll|
578 def test_recursive_remove_root
579 coll = Arv::Collection.new(MULTILEVEL_MANIFEST)
581 assert_equal("", coll.manifest_text)
584 def test_rm_r_nonexistent_file_raises_ENOENT(path="./NoSuchFile")
585 test_remove_nonexistent_file_raises_ENOENT("./NoSuchFile", :rm_r)
588 def test_rm_r_from_nonexistent_stream_raises_ENOENT
589 test_remove_nonexistent_file_raises_ENOENT("./NoSuchStream/file", :rm_r)
592 def test_rm_r_empty_string_raises_ArgumentError
593 coll = Arv::Collection.new(SIMPLEST_MANIFEST)
594 assert_raises(ArgumentError) do
601 def test_new_collection_unmodified(*args)
602 coll = Arv::Collection.new(*args)
603 yield coll if block_given?
604 refute(coll.modified?)
607 def test_collection_unmodified_after_instantiation
608 test_new_collection_unmodified(SIMPLEST_MANIFEST)
611 def test_collection_unmodified_after_mark
612 test_new_collection_unmodified(SIMPLEST_MANIFEST) do |coll|
613 coll.cp_r("./simple.txt", "./copy")
618 def check_collection_modified
619 coll = Arv::Collection.new(SIMPLEST_MANIFEST)
621 assert(coll.modified?)
624 def test_collection_modified_after_copy
625 check_collection_modified do |coll|
626 coll.cp_r("./simple.txt", "./copy")
630 def test_collection_modified_after_remove
631 check_collection_modified do |coll|
632 coll.rm("./simple.txt")
636 def test_collection_modified_after_rename
637 check_collection_modified do |coll|
638 coll.rename("./simple.txt", "./newname")