sdk/cwl/tests/test_pathmapper.py

   1 import functools
   2 import mock
   3 import sys
   4 import unittest
   5 import json
   6 import logging
   7 import os
   8
   9 import arvados
  10 import arvados.keep
  11 import arvados.collection
  12 import arvados_cwl
  13
  14 from cwltool.pathmapper import MapperEnt
  15 from .mock_discovery import get_rootDesc
  16
  17 from arvados_cwl.pathmapper import ArvPathMapper
  18
  19 def upload_mock(files, api, dry_run=False, num_retries=0, project=None, fnPattern="$(file %s/%s)", name=None, collection=None):
  20     pdh = "99999999999999999999999999999991+99"
  21     for c in files:
  22         c.keepref = "%s/%s" % (pdh, os.path.basename(c.fn))
  23         c.fn = fnPattern % (pdh, os.path.basename(c.fn))
  24
  25 class TestPathmap(unittest.TestCase):
  26     def setUp(self):
  27         self.api = mock.MagicMock()
  28         self.api._rootDesc = get_rootDesc()
  29
  30     def test_keepref(self):
  31         """Test direct keep references."""
  32
  33         arvrunner = arvados_cwl.ArvCwlRunner(self.api)
  34
  35         p = ArvPathMapper(arvrunner, [{
  36             "class": "File",
  37             "location": "keep:99999999999999999999999999999991+99/hw.py"
  38         }], "", "/test/%s", "/test/%s/%s")
  39
  40         self.assertEqual({'keep:99999999999999999999999999999991+99/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
  41                          p._pathmap)
  42
  43     @mock.patch("arvados.commands.run.uploadfiles")
  44     @mock.patch("arvados.commands.run.statfile")
  45     def test_upload(self, statfile, upl):
  46         """Test pathmapper uploading files."""
  47
  48         arvrunner = arvados_cwl.ArvCwlRunner(self.api)
  49
  50         def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)", raiseOSError=False):
  51             st = arvados.commands.run.UploadFile("", "tests/hw.py")
  52             return st
  53
  54         upl.side_effect = upload_mock
  55         statfile.side_effect = statfile_mock
  56
  57         p = ArvPathMapper(arvrunner, [{
  58             "class": "File",
  59             "location": "file:tests/hw.py"
  60         }], "", "/test/%s", "/test/%s/%s")
  61
  62         self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
  63                          p._pathmap)
  64
  65     @mock.patch("arvados.commands.run.uploadfiles")
  66     def test_prev_uploaded(self, upl):
  67         """Test pathmapper handling previously uploaded files."""
  68
  69         arvrunner = arvados_cwl.ArvCwlRunner(self.api)
  70         arvrunner.add_uploaded('file:tests/hw.py', MapperEnt(resolved='keep:99999999999999999999999999999992+99/hw.py', target='', type='File', staged=True))
  71
  72         upl.side_effect = upload_mock
  73
  74         p = ArvPathMapper(arvrunner, [{
  75             "class": "File",
  76             "location": "file:tests/hw.py"
  77         }], "", "/test/%s", "/test/%s/%s")
  78
  79         self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999992+99/hw.py', target='/test/99999999999999999999999999999992+99/hw.py', type='File', staged=True)},
  80                          p._pathmap)
  81
  82     @mock.patch("arvados.commands.run.uploadfiles")
  83     @mock.patch("arvados.commands.run.statfile")
  84     def test_statfile(self, statfile, upl):
  85         """Test pathmapper handling ArvFile references."""
  86         arvrunner = arvados_cwl.ArvCwlRunner(self.api)
  87
  88         # An ArvFile object returned from arvados.commands.run.statfile means the file is located on a
  89         # keep mount, so we can construct a direct reference directly without upload.
  90         def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)", raiseOSError=False):
  91             st = arvados.commands.run.ArvFile("", fnPattern % ("99999999999999999999999999999991+99", "hw.py"))
  92             return st
  93
  94         upl.side_effect = upload_mock
  95         statfile.side_effect = statfile_mock
  96
  97         p = ArvPathMapper(arvrunner, [{
  98             "class": "File",
  99             "location": "file:tests/hw.py"
 100         }], "", "/test/%s", "/test/%s/%s")
 101
 102         self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
 103                          p._pathmap)
 104
 105     @mock.patch("os.stat")
 106     def test_missing_file(self, stat):
 107         """Test pathmapper handling missing references."""
 108         arvrunner = arvados_cwl.ArvCwlRunner(self.api)
 109
 110         stat.side_effect = OSError(2, "No such file or directory")
 111
 112         with self.assertRaises(OSError):
 113             p = ArvPathMapper(arvrunner, [{
 114                 "class": "File",
 115                 "location": "file:tests/hw.py"
 116             }], "", "/test/%s", "/test/%s/%s")