sdk/cwl/tests/test_pathmapper.py

   1 import functools
   2 import mock
   3 import sys
   4 import unittest
   5 import json
   6 import logging
   7 import os
   8
   9 import arvados
  10 import arvados.keep
  11 import arvados.collection
  12 import arvados_cwl
  13
  14 from cwltool.pathmapper import MapperEnt
  15
  16 from arvados_cwl.pathmapper import ArvPathMapper
  17
  18 def upload_mock(files, api, dry_run=False, num_retries=0, project=None, fnPattern="$(file %s/%s)", name=None):
  19     pdh = "99999999999999999999999999999991+99"
  20     for c in files:
  21         c.fn = os.path.basename(c.fn)
  22         c.keepref = "%s/%s" % (pdh, c.fn)
  23         c.fn = fnPattern % (pdh, c.fn)
  24
  25 class TestPathmap(unittest.TestCase):
  26     def test_keepref(self):
  27         """Test direct keep references."""
  28
  29         arvrunner = arvados_cwl.ArvCwlRunner(mock.MagicMock())
  30
  31         p = ArvPathMapper(arvrunner, [{
  32             "class": "File",
  33             "location": "keep:99999999999999999999999999999991+99/hw.py"
  34         }], "", "/test/%s", "/test/%s/%s")
  35
  36         self.assertEqual({'keep:99999999999999999999999999999991+99/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
  37                          p._pathmap)
  38
  39     @mock.patch("arvados.commands.run.uploadfiles")
  40     def test_upload(self, upl):
  41         """Test pathmapper uploading files."""
  42
  43         arvrunner = arvados_cwl.ArvCwlRunner(mock.MagicMock())
  44
  45         upl.side_effect = upload_mock
  46
  47         p = ArvPathMapper(arvrunner, [{
  48             "class": "File",
  49             "location": "tests/hw.py"
  50         }], "", "/test/%s", "/test/%s/%s")
  51
  52         self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
  53                          p._pathmap)
  54
  55     @mock.patch("arvados.commands.run.uploadfiles")
  56     def test_prev_uploaded(self, upl):
  57         """Test pathmapper handling previously uploaded files."""
  58
  59         arvrunner = arvados_cwl.ArvCwlRunner(mock.MagicMock())
  60         arvrunner.add_uploaded('tests/hw.py', MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='', type='File'))
  61
  62         upl.side_effect = upload_mock
  63
  64         p = ArvPathMapper(arvrunner, [{
  65             "class": "File",
  66             "location": "tests/hw.py"
  67         }], "", "/test/%s", "/test/%s/%s")
  68
  69         self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
  70                          p._pathmap)
  71
  72     @mock.patch("arvados.commands.run.uploadfiles")
  73     @mock.patch("arvados.commands.run.statfile")
  74     def test_statfile(self, statfile, upl):
  75         """Test pathmapper handling ArvFile references."""
  76         arvrunner = arvados_cwl.ArvCwlRunner(mock.MagicMock())
  77
  78         # An ArvFile object returned from arvados.commands.run.statfile means the file is located on a
  79         # keep mount, so we can construct a direct reference directly without upload.
  80         def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)"):
  81             st = arvados.commands.run.ArvFile("", fnPattern % ("99999999999999999999999999999991+99", "hw.py"))
  82             st.keepref = "99999999999999999999999999999991+99/hw.py"
  83             return st
  84
  85         upl.side_effect = upload_mock
  86         statfile.side_effect = statfile_mock
  87
  88         p = ArvPathMapper(arvrunner, [{
  89             "class": "File",
  90             "location": "tests/hw.py"
  91         }], "", "/test/%s", "/test/%s/%s")
  92
  93         self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
  94                          p._pathmap)