sdk/cwl/tests/test_pathmapper.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import functools
   6 import mock
   7 import sys
   8 import unittest
   9 import json
  10 import logging
  11 import os
  12
  13 import arvados
  14 import arvados.keep
  15 import arvados.collection
  16 import arvados_cwl
  17
  18 from cwltool.pathmapper import MapperEnt
  19 from .mock_discovery import get_rootDesc
  20
  21 from arvados_cwl.pathmapper import ArvPathMapper
  22
  23 def upload_mock(files, api, dry_run=False, num_retries=0, project=None, fnPattern="$(file %s/%s)", name=None, collection=None):
  24     pdh = "99999999999999999999999999999991+99"
  25     for c in files:
  26         c.keepref = "%s/%s" % (pdh, os.path.basename(c.fn))
  27         c.fn = fnPattern % (pdh, os.path.basename(c.fn))
  28
  29 class TestPathmap(unittest.TestCase):
  30     def setUp(self):
  31         self.api = mock.MagicMock()
  32         self.api._rootDesc = get_rootDesc()
  33
  34     def test_keepref(self):
  35         """Test direct keep references."""
  36
  37         arvrunner = arvados_cwl.ArvCwlRunner(self.api)
  38
  39         p = ArvPathMapper(arvrunner, [{
  40             "class": "File",
  41             "location": "keep:99999999999999999999999999999991+99/hw.py"
  42         }], "", "/test/%s", "/test/%s/%s")
  43
  44         self.assertEqual({'keep:99999999999999999999999999999991+99/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
  45                          p._pathmap)
  46
  47     @mock.patch("arvados.commands.run.uploadfiles")
  48     @mock.patch("arvados.commands.run.statfile")
  49     def test_upload(self, statfile, upl):
  50         """Test pathmapper uploading files."""
  51
  52         arvrunner = arvados_cwl.ArvCwlRunner(self.api)
  53
  54         def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)", raiseOSError=False):
  55             st = arvados.commands.run.UploadFile("", "tests/hw.py")
  56             return st
  57
  58         upl.side_effect = upload_mock
  59         statfile.side_effect = statfile_mock
  60
  61         p = ArvPathMapper(arvrunner, [{
  62             "class": "File",
  63             "location": "file:tests/hw.py"
  64         }], "", "/test/%s", "/test/%s/%s")
  65
  66         self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
  67                          p._pathmap)
  68
  69     @mock.patch("arvados.commands.run.uploadfiles")
  70     def test_prev_uploaded(self, upl):
  71         """Test pathmapper handling previously uploaded files."""
  72
  73         arvrunner = arvados_cwl.ArvCwlRunner(self.api)
  74         arvrunner.add_uploaded('file:tests/hw.py', MapperEnt(resolved='keep:99999999999999999999999999999992+99/hw.py', target='', type='File', staged=True))
  75
  76         upl.side_effect = upload_mock
  77
  78         p = ArvPathMapper(arvrunner, [{
  79             "class": "File",
  80             "location": "file:tests/hw.py"
  81         }], "", "/test/%s", "/test/%s/%s")
  82
  83         self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999992+99/hw.py', target='/test/99999999999999999999999999999992+99/hw.py', type='File', staged=True)},
  84                          p._pathmap)
  85
  86     @mock.patch("arvados.commands.run.uploadfiles")
  87     @mock.patch("arvados.commands.run.statfile")
  88     def test_statfile(self, statfile, upl):
  89         """Test pathmapper handling ArvFile references."""
  90         arvrunner = arvados_cwl.ArvCwlRunner(self.api)
  91
  92         # An ArvFile object returned from arvados.commands.run.statfile means the file is located on a
  93         # keep mount, so we can construct a direct reference directly without upload.
  94         def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)", raiseOSError=False):
  95             st = arvados.commands.run.ArvFile("", fnPattern % ("99999999999999999999999999999991+99", "hw.py"))
  96             return st
  97
  98         upl.side_effect = upload_mock
  99         statfile.side_effect = statfile_mock
 100
 101         p = ArvPathMapper(arvrunner, [{
 102             "class": "File",
 103             "location": "file:tests/hw.py"
 104         }], "", "/test/%s", "/test/%s/%s")
 105
 106         self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
 107                          p._pathmap)
 108
 109     @mock.patch("os.stat")
 110     def test_missing_file(self, stat):
 111         """Test pathmapper handling missing references."""
 112         arvrunner = arvados_cwl.ArvCwlRunner(self.api)
 113
 114         stat.side_effect = OSError(2, "No such file or directory")
 115
 116         with self.assertRaises(OSError):
 117             p = ArvPathMapper(arvrunner, [{
 118                 "class": "File",
 119                 "location": "file:tests/hw.py"
 120             }], "", "/test/%s", "/test/%s/%s")