1 require "minitest/autorun"
5 def random_block(size=nil)
6 sprintf("%032x+%d", rand(16 ** 32), size || rand(64 * 1024 * 1024))
9 class ManifestTest < Minitest::Test
10 SIMPLEST_MANIFEST = ". #{random_block(9)} 0:9:simple.txt\n"
11 MULTIBLOCK_FILE_MANIFEST =
12 [". #{random_block(8)} 0:4:repfile 4:4:uniqfile",
13 "./s1 #{random_block(6)} 0:3:repfile 3:3:uniqfile",
14 ". #{random_block(8)} 0:7:uniqfile2 7:1:repfile\n"].join("\n")
16 [". #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n",
17 "./dir1 #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n",
18 "./dir1/subdir #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n",
19 "./dir2 #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n"].join("")
21 def check_stream(stream, exp_name, exp_blocks, exp_files)
22 assert_equal(exp_name, stream.first)
23 assert_equal(exp_blocks, stream[1].map(&:to_s))
24 assert_equal(exp_files, stream.last)
27 def test_simple_each_line_array
28 manifest = Keep::Manifest.new(SIMPLEST_MANIFEST)
29 stream_name, block_s, file = SIMPLEST_MANIFEST.strip.split
30 stream_a = manifest.each_line.to_a
31 assert_equal(1, stream_a.size, "wrong number of streams")
32 check_stream(stream_a.first, stream_name, [block_s], [file])
35 def test_simple_each_line_block
36 manifest = Keep::Manifest.new(SIMPLEST_MANIFEST)
38 manifest.each_line do |stream, blocks, files|
41 assert_equal([[SIMPLEST_MANIFEST.split.last]], result,
42 "wrong result from each_line block")
45 def test_multilevel_each_line
46 manifest = Keep::Manifest.new(MULTILEVEL_MANIFEST)
48 manifest.each_line do |stream, blocks, files|
49 refute(seen.include?(stream),
50 "each_line already yielded stream #{stream}")
52 assert_equal(3, files.size, "wrong file count for stream #{stream}")
54 assert_equal(4, seen.size, "wrong number of streams")
57 def test_empty_each_line
58 assert_empty(Keep::Manifest.new("").each_line.to_a)
61 def test_empty_each_file_spec
62 assert_empty(Keep::Manifest.new("").each_file_spec.to_a)
66 assert_empty(Keep::Manifest.new("").files)
69 def test_empty_files_count
70 assert_equal(0, Keep::Manifest.new("").files_count)
73 def test_empty_has_file?
74 refute(Keep::Manifest.new("").has_file?(""))
77 def test_empty_line_within_manifest
78 block_s = random_block
79 manifest = Keep::Manifest.
80 new([". #{block_s} 0:1:file1 1:2:file2\n",
82 ". #{block_s} 3:3:file3 6:4:file4\n"].join(""))
83 streams = manifest.each_line.to_a
84 assert_equal(2, streams.size)
85 check_stream(streams[0], ".", [block_s], ["0:1:file1", "1:2:file2"])
86 check_stream(streams[1], ".", [block_s], ["3:3:file3", "6:4:file4"])
89 def test_backslash_escape_parsing
90 m_text = "./dir\\040name #{random_block} 0:0:file\\\\name\\011\\here.txt\n"
91 manifest = Keep::Manifest.new(m_text)
92 streams = manifest.each_line.to_a
93 assert_equal(1, streams.size, "wrong number of streams with whitespace")
94 assert_equal("./dir name", streams.first.first,
95 "wrong stream name with whitespace")
96 assert_equal(["0:0:file\\name\t\\here.txt"], streams.first.last,
97 "wrong filename(s) with whitespace")
100 def test_simple_files
101 manifest = Keep::Manifest.new(SIMPLEST_MANIFEST)
102 assert_equal([[".", "simple.txt", 9]], manifest.files)
105 def test_multilevel_files
106 manifest = Keep::Manifest.new(MULTILEVEL_MANIFEST)
107 seen = Hash.new { |this, key| this[key] = [] }
108 manifest.files.each do |stream, basename, size|
109 refute(seen[stream].include?(basename),
110 "each_file repeated #{stream}/#{basename}")
111 seen[stream] << basename
112 assert_equal(3, size, "wrong size for #{stream}/#{basename}")
114 seen.each_pair do |stream, basenames|
115 assert_equal(%w(file1 file2 file3), basenames.sort,
116 "wrong file list for #{stream}")
120 def test_files_with_colons_in_names
121 manifest = Keep::Manifest.new(". #{random_block(9)} 0:9:file:test.txt\n")
122 assert_equal([[".", "file:test.txt", 9]], manifest.files)
125 def test_files_with_escape_sequence_in_filename
126 manifest = Keep::Manifest.new(". #{random_block(9)} 0:9:a\\040\\141.txt\n")
127 assert_equal([[".", "a a.txt", 9]], manifest.files)
130 def test_files_spanning_multiple_blocks
131 manifest = Keep::Manifest.new(MULTIBLOCK_FILE_MANIFEST)
132 assert_equal([[".", "repfile", 5],
133 [".", "uniqfile", 4],
134 [".", "uniqfile2", 7],
135 ["./s1", "repfile", 3],
136 ["./s1", "uniqfile", 3]],
140 def test_minimum_file_count_simple
141 manifest = Keep::Manifest.new(SIMPLEST_MANIFEST)
142 assert(manifest.minimum_file_count?(1), "real minimum file count false")
143 refute(manifest.minimum_file_count?(2), "fake minimum file count true")
146 def test_minimum_file_count_multiblock
147 manifest = Keep::Manifest.new(MULTIBLOCK_FILE_MANIFEST)
148 assert(manifest.minimum_file_count?(2), "low minimum file count false")
149 assert(manifest.minimum_file_count?(5), "real minimum file count false")
150 refute(manifest.minimum_file_count?(6), "fake minimum file count true")
153 def test_exact_file_count_simple
154 manifest = Keep::Manifest.new(SIMPLEST_MANIFEST)
155 assert(manifest.exact_file_count?(1), "exact file count false")
156 refute(manifest.exact_file_count?(0), "-1 file count true")
157 refute(manifest.exact_file_count?(2), "+1 file count true")
160 def test_exact_file_count_multiblock
161 manifest = Keep::Manifest.new(MULTIBLOCK_FILE_MANIFEST)
162 assert(manifest.exact_file_count?(5), "exact file count false")
163 refute(manifest.exact_file_count?(4), "-1 file count true")
164 refute(manifest.exact_file_count?(6), "+1 file count true")
168 manifest = Keep::Manifest.new(MULTIBLOCK_FILE_MANIFEST)
169 assert(manifest.has_file?("./repfile"), "one-arg repfile not found")
170 assert(manifest.has_file?(".", "repfile"), "two-arg repfile not found")
171 assert(manifest.has_file?("./s1/repfile"), "one-arg s1/repfile not found")
172 assert(manifest.has_file?("./s1", "repfile"), "two-arg s1/repfile not found")
173 refute(manifest.has_file?("./s1/uniqfile2"), "one-arg missing file found")
174 refute(manifest.has_file?("./s1", "uniqfile2"), "two-arg missing file found")
175 refute(manifest.has_file?("./s2/repfile"), "one-arg missing stream found")
176 refute(manifest.has_file?("./s2", "repfile"), "two-arg missing stream found")
179 def test_has_file_with_spaces
180 manifest = Keep::Manifest.new(". #{random_block(3)} 0:3:a\\040b\\040c\n")
181 assert(manifest.has_file?("./a b c"), "one-arg 'a b c' not found")
182 assert(manifest.has_file?(".", "a b c"), "two-arg 'a b c' not found")
183 refute(manifest.has_file?("a\\040b\\040c"), "one-arg unescaped found")
184 refute(manifest.has_file?(".", "a\\040b\\040c"), "two-arg unescaped found")
187 def test_parse_all_fixtures
188 fixtures('collections').each do |name, collection|
189 parse_collection_manifest name, collection
193 def test_raise_on_bogus_fixture
194 assert_raises ArgumentError do
195 parse_collection_manifest('bogus collection',
196 {'manifest_text' => ". zzz 0:\n"})
200 def parse_collection_manifest name, collection
201 manifest = Keep::Manifest.new(collection['manifest_text'])
202 manifest.each_file_spec do |stream_name, start_pos, file_size, file_name|
203 assert_kind_of String, stream_name
204 assert_kind_of Integer, start_pos
205 assert_kind_of Integer, file_size
206 assert_kind_of String, file_name
207 assert !stream_name.empty?, "empty stream_name in #{name} fixture"
208 assert !file_name.empty?, "empty file_name in #{name} fixture"
214 return @@fixtures if @@fixtures
215 path = File.expand_path("../../../../services/api/test/fixtures/#{name}.yml",
218 trim_index = file.index('# Test Helper trims the rest of the file')
219 file = file[0, trim_index] if trim_index
220 @@fixtures = YAML.load(file)