sdk/cwl/tests/test_pathmapper.py

   1 import functools
   2 import mock
   3 import sys
   4 import unittest
   5 import json
   6 import logging
   7 import os
   8
   9 import arvados
  10 import arvados.keep
  11 import arvados.collection
  12 import arvados_cwl
  13
  14 from cwltool.pathmapper import MapperEnt
  15 from .mock_discovery import get_rootDesc
  16
  17 from arvados_cwl.pathmapper import ArvPathMapper
  18
  19 def upload_mock(files, api, dry_run=False, num_retries=0, project=None, fnPattern="$(file %s/%s)", name=None):
  20     pdh = "99999999999999999999999999999991+99"
  21     for c in files:
  22         c.fn = fnPattern % (pdh, os.path.basename(c.fn))
  23
  24 class TestPathmap(unittest.TestCase):
  25     def setUp(self):
  26         self.api = mock.MagicMock()
  27         self.api._rootDesc = get_rootDesc()
  28
  29     def test_keepref(self):
  30         """Test direct keep references."""
  31
  32         arvrunner = arvados_cwl.ArvCwlRunner(self.api)
  33
  34         p = ArvPathMapper(arvrunner, [{
  35             "class": "File",
  36             "location": "keep:99999999999999999999999999999991+99/hw.py"
  37         }], "", "/test/%s", "/test/%s/%s")
  38
  39         self.assertEqual({'keep:99999999999999999999999999999991+99/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
  40                          p._pathmap)
  41
  42     @mock.patch("arvados.commands.run.uploadfiles")
  43     def test_upload(self, upl):
  44         """Test pathmapper uploading files."""
  45
  46         arvrunner = arvados_cwl.ArvCwlRunner(self.api)
  47
  48         upl.side_effect = upload_mock
  49
  50         p = ArvPathMapper(arvrunner, [{
  51             "class": "File",
  52             "location": "tests/hw.py"
  53         }], "", "/test/%s", "/test/%s/%s")
  54
  55         self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
  56                          p._pathmap)
  57
  58     @mock.patch("arvados.commands.run.uploadfiles")
  59     def test_prev_uploaded(self, upl):
  60         """Test pathmapper handling previously uploaded files."""
  61
  62         arvrunner = arvados_cwl.ArvCwlRunner(self.api)
  63         arvrunner.add_uploaded('tests/hw.py', MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='', type='File'))
  64
  65         upl.side_effect = upload_mock
  66
  67         p = ArvPathMapper(arvrunner, [{
  68             "class": "File",
  69             "location": "tests/hw.py"
  70         }], "", "/test/%s", "/test/%s/%s")
  71
  72         self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
  73                          p._pathmap)
  74
  75     @mock.patch("arvados.commands.run.uploadfiles")
  76     @mock.patch("arvados.commands.run.statfile")
  77     def test_statfile(self, statfile, upl):
  78         """Test pathmapper handling ArvFile references."""
  79         arvrunner = arvados_cwl.ArvCwlRunner(self.api)
  80
  81         # An ArvFile object returned from arvados.commands.run.statfile means the file is located on a
  82         # keep mount, so we can construct a direct reference directly without upload.
  83         def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)"):
  84             st = arvados.commands.run.ArvFile("", fnPattern % ("99999999999999999999999999999991+99", "hw.py"))
  85             return st
  86
  87         upl.side_effect = upload_mock
  88         statfile.side_effect = statfile_mock
  89
  90         p = ArvPathMapper(arvrunner, [{
  91             "class": "File",
  92             "location": "tests/hw.py"
  93         }], "", "/test/%s", "/test/%s/%s")
  94
  95         self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
  96                          p._pathmap)