sdk/ruby/test/test_keep_manifest.rb

   1 require "minitest/autorun"
   2 require "arvados/keep"
   3
   4 def random_block(size=nil)
   5   sprintf("%032x+%d", rand(16 ** 32), size || rand(64 * 1024 * 1024))
   6 end
   7
   8 class ManifestTest < Minitest::Test
   9   SIMPLEST_MANIFEST = ". #{random_block(9)} 0:9:simple.txt\n"
  10   MULTIBLOCK_FILE_MANIFEST =
  11     [". #{random_block(8)} 0:4:repfile 4:4:uniqfile",
  12      "./s1 #{random_block(6)} 0:3:repfile 3:3:uniqfile",
  13      ". #{random_block(8)} 0:7:uniqfile2 7:1:repfile\n"].join("\n")
  14   MULTILEVEL_MANIFEST =
  15     [". #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n",
  16      "./dir1 #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n",
  17      "./dir1/subdir #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n",
  18      "./dir2 #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n"].join("")
  19
  20   def test_simple_each_line_array
  21     manifest = Keep::Manifest.new(SIMPLEST_MANIFEST)
  22     stream_name, block_s, file = SIMPLEST_MANIFEST.strip.split
  23     stream_a = manifest.each_line.to_a
  24     assert_equal(1, stream_a.size, "wrong number of streams")
  25     assert_equal(stream_name, stream_a[0][0])
  26     assert_equal([block_s], stream_a[0][1].map(&:to_s))
  27     assert_equal([file], stream_a[0][2])
  28   end
  29
  30   def test_simple_each_line_block
  31     manifest = Keep::Manifest.new(SIMPLEST_MANIFEST)
  32     result = []
  33     manifest.each_line do |stream, blocks, files|
  34       result << files
  35     end
  36     assert_equal([[SIMPLEST_MANIFEST.split.last]], result,
  37                  "wrong result from each_line block")
  38   end
  39
  40   def test_multilevel_each_line
  41     manifest = Keep::Manifest.new(MULTILEVEL_MANIFEST)
  42     seen = []
  43     manifest.each_line do |stream, blocks, files|
  44       refute(seen.include?(stream),
  45              "each_line already yielded stream #{stream}")
  46       seen << stream
  47       assert_equal(3, files.size, "wrong file count for stream #{stream}")
  48     end
  49     assert_equal(4, seen.size, "wrong number of streams")
  50   end
  51
  52   def test_empty_each_line
  53     assert_empty(Keep::Manifest.new("").each_line.to_a)
  54   end
  55
  56   def test_backslash_escape_parsing
  57     m_text = "./dir\\040name #{random_block} 0:0:file\\\\name\\011\\here.txt\n"
  58     manifest = Keep::Manifest.new(m_text)
  59     streams = manifest.each_line.to_a
  60     assert_equal(1, streams.size, "wrong number of streams with whitespace")
  61     assert_equal("./dir name", streams.first.first,
  62                  "wrong stream name with whitespace")
  63     assert_equal(["0:0:file\\name\t\\here.txt"], streams.first.last,
  64                  "wrong filename(s) with whitespace")
  65   end
  66
  67   def test_simple_files
  68     manifest = Keep::Manifest.new(SIMPLEST_MANIFEST)
  69     assert_equal([[".", "simple.txt", 9]], manifest.files)
  70   end
  71
  72   def test_multilevel_files
  73     manifest = Keep::Manifest.new(MULTILEVEL_MANIFEST)
  74     seen = Hash.new { |this, key| this[key] = [] }
  75     manifest.files.each do |stream, basename, size|
  76       refute(seen[stream].include?(basename),
  77              "each_file repeated #{stream}/#{basename}")
  78       seen[stream] << basename
  79       assert_equal(3, size, "wrong size for #{stream}/#{basename}")
  80     end
  81     seen.each_pair do |stream, basenames|
  82       assert_equal(%w(file1 file2 file3), basenames.sort,
  83                    "wrong file list for #{stream}")
  84     end
  85   end
  86
  87   def test_files_with_colons_in_names
  88     manifest = Keep::Manifest.new(". #{random_block(9)} 0:9:file:test.txt\n")
  89     assert_equal([[".", "file:test.txt", 9]], manifest.files)
  90   end
  91
  92   def test_files_spanning_multiple_blocks
  93     manifest = Keep::Manifest.new(MULTIBLOCK_FILE_MANIFEST)
  94     assert_equal([[".", "repfile", 5],
  95                   [".", "uniqfile", 4],
  96                   [".", "uniqfile2", 7],
  97                   ["./s1", "repfile", 3],
  98                   ["./s1", "uniqfile", 3]],
  99                  manifest.files.sort)
 100   end
 101
 102   def test_minimum_file_count_simple
 103     manifest = Keep::Manifest.new(SIMPLEST_MANIFEST)
 104     assert(manifest.minimum_file_count?(1), "real minimum file count false")
 105     refute(manifest.minimum_file_count?(2), "fake minimum file count true")
 106   end
 107
 108   def test_minimum_file_count_multiblock
 109     manifest = Keep::Manifest.new(MULTIBLOCK_FILE_MANIFEST)
 110     assert(manifest.minimum_file_count?(2), "low minimum file count false")
 111     assert(manifest.minimum_file_count?(5), "real minimum file count false")
 112     refute(manifest.minimum_file_count?(6), "fake minimum file count true")
 113   end
 114
 115   def test_exact_file_count_simple
 116     manifest = Keep::Manifest.new(SIMPLEST_MANIFEST)
 117     assert(manifest.exact_file_count?(1), "exact file count false")
 118     refute(manifest.exact_file_count?(0), "-1 file count true")
 119     refute(manifest.exact_file_count?(2), "+1 file count true")
 120   end
 121
 122   def test_exact_file_count_multiblock
 123     manifest = Keep::Manifest.new(MULTIBLOCK_FILE_MANIFEST)
 124     assert(manifest.exact_file_count?(5), "exact file count false")
 125     refute(manifest.exact_file_count?(4), "-1 file count true")
 126     refute(manifest.exact_file_count?(6), "+1 file count true")
 127   end
 128
 129   def test_has_file
 130     manifest = Keep::Manifest.new(MULTIBLOCK_FILE_MANIFEST)
 131     assert(manifest.has_file?("./repfile"), "one-arg repfile not found")
 132     assert(manifest.has_file?(".", "repfile"), "two-arg repfile not found")
 133     assert(manifest.has_file?("./s1/repfile"), "one-arg s1/repfile not found")
 134     assert(manifest.has_file?("./s1", "repfile"), "two-arg s1/repfile not found")
 135     refute(manifest.has_file?("./s1/uniqfile2"), "one-arg missing file found")
 136     refute(manifest.has_file?("./s1", "uniqfile2"), "two-arg missing file found")
 137     refute(manifest.has_file?("./s2/repfile"), "one-arg missing stream found")
 138     refute(manifest.has_file?("./s2", "repfile"), "two-arg missing stream found")
 139   end
 140 end