From b3a1f8ce34c4069b315b18f6d1c8fb7e534540f2 Mon Sep 17 00:00:00 2001 From: Brett Smith Date: Fri, 12 Jul 2024 14:37:12 -0400 Subject: [PATCH] 21935: Move arvados.http_to_keep under arvados._internal This functionality could be useful to provide to people writing their own tools. However, because of the way it was extracted out of arvados-cwl-runner, the API wasn't really designed to be public, and has already had one breaking change since it was introduced. Make the current module internal so the API doesn't ossify. We can always develop a public API for it in the future. Arvados-DCO-1.1-Signed-off-by: Brett Smith --- sdk/cwl/arvados_cwl/pathmapper.py | 6 ++--- .../arvados/{ => _internal}/http_to_keep.py | 0 sdk/python/arvados/commands/arv_copy.py | 14 ++++++------ .../{test_http.py => test_http_to_keep.py} | 22 +++++++++---------- 4 files changed, 20 insertions(+), 22 deletions(-) rename sdk/python/arvados/{ => _internal}/http_to_keep.py (100%) rename sdk/python/tests/{test_http.py => test_http_to_keep.py} (94%) diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py index ac6df543ad..dbb5b53afc 100644 --- a/sdk/cwl/arvados_cwl/pathmapper.py +++ b/sdk/cwl/arvados_cwl/pathmapper.py @@ -12,15 +12,13 @@ import arvados_cwl.util import arvados.commands.run import arvados.collection -from schema_salad.sourceline import SourceLine - from arvados.errors import ApiError +from arvados._internal.http_to_keep import http_to_keep from cwltool.pathmapper import PathMapper, MapperEnt from cwltool.utils import adjustFileObjs, adjustDirObjs from cwltool.stdfsaccess import abspath from cwltool.workflow import WorkflowException - -from arvados.http_to_keep import http_to_keep +from schema_salad.sourceline import SourceLine logger = logging.getLogger('arvados.cwl-runner') diff --git a/sdk/python/arvados/http_to_keep.py b/sdk/python/arvados/_internal/http_to_keep.py similarity index 100% rename from sdk/python/arvados/http_to_keep.py rename to sdk/python/arvados/_internal/http_to_keep.py diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py index 59f2710360..2c0ed45bd0 100755 --- a/sdk/python/arvados/commands/arv_copy.py +++ b/sdk/python/arvados/commands/arv_copy.py @@ -40,8 +40,8 @@ import arvados.keep import arvados.util import arvados.commands._util as arv_cmd import arvados.commands.keepdocker -import arvados.http_to_keep +from arvados._internal import http_to_keep from arvados._version import __version__ COMMIT_HASH_RE = re.compile(r'^[0-9a-f]{1,40}$') @@ -873,15 +873,15 @@ def copy_from_http(url, src, dst, args): varying_url_params = args.varying_url_params prefer_cached_downloads = args.prefer_cached_downloads - cached = arvados.http_to_keep.check_cached_url(src, project_uuid, url, {}, - varying_url_params=varying_url_params, - prefer_cached_downloads=prefer_cached_downloads) + cached = http_to_keep.check_cached_url(src, project_uuid, url, {}, + varying_url_params=varying_url_params, + prefer_cached_downloads=prefer_cached_downloads) if cached[2] is not None: return copy_collection(cached[2], src, dst, args) - cached = arvados.http_to_keep.http_to_keep(dst, project_uuid, url, - varying_url_params=varying_url_params, - prefer_cached_downloads=prefer_cached_downloads) + cached = http_to_keep.http_to_keep(dst, project_uuid, url, + varying_url_params=varying_url_params, + prefer_cached_downloads=prefer_cached_downloads) if cached is not None: return {"uuid": cached[2]} diff --git a/sdk/python/tests/test_http.py b/sdk/python/tests/test_http_to_keep.py similarity index 94% rename from sdk/python/tests/test_http.py rename to sdk/python/tests/test_http_to_keep.py index 476e2c88e9..b8d4679029 100644 --- a/sdk/python/tests/test_http.py +++ b/sdk/python/tests/test_http_to_keep.py @@ -19,7 +19,7 @@ import arvados.collection import arvados.keep import pycurl -from arvados.http_to_keep import http_to_keep +from arvados._internal import http_to_keep # Turns out there was already "FakeCurl" that serves the same purpose, but # I wrote this before I knew that. Whoops. @@ -92,7 +92,7 @@ class TestHttpToKeep(unittest.TestCase): utcnow = mock.MagicMock() utcnow.return_value = datetime.datetime(2018, 5, 15) - r = http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow) + r = http_to_keep.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow) self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt", 'zzzzz-4zz18-zzzzzzzzzzzzzz3', 'http://example.com/file1.txt', datetime.datetime(2018, 5, 15, 0, 0))) @@ -143,7 +143,7 @@ class TestHttpToKeep(unittest.TestCase): utcnow = mock.MagicMock() utcnow.return_value = datetime.datetime(2018, 5, 16) - r = http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow) + r = http_to_keep.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow) self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt", 'zzzzz-4zz18-zzzzzzzzzzzzzz3', 'http://example.com/file1.txt', datetime.datetime(2018, 5, 16, 0, 0))) @@ -184,7 +184,7 @@ class TestHttpToKeep(unittest.TestCase): utcnow = mock.MagicMock() utcnow.return_value = datetime.datetime(2018, 5, 16) - r = http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow) + r = http_to_keep.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow) self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt", 'zzzzz-4zz18-zzzzzzzzzzzzzz3', 'http://example.com/file1.txt', datetime.datetime(2018, 5, 16, 0, 0))) @@ -224,7 +224,7 @@ class TestHttpToKeep(unittest.TestCase): utcnow = mock.MagicMock() utcnow.return_value = datetime.datetime(2018, 5, 17) - r = http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow) + r = http_to_keep.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow) self.assertEqual(r, ("99999999999999999999999999999997+99", "file1.txt", 'zzzzz-4zz18-zzzzzzzzzzzzzz4', 'http://example.com/file1.txt', datetime.datetime(2018, 5, 17, 0, 0))) @@ -281,7 +281,7 @@ class TestHttpToKeep(unittest.TestCase): utcnow = mock.MagicMock() utcnow.return_value = datetime.datetime(2018, 5, 17) - r = http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow) + r = http_to_keep.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow) self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt", 'zzzzz-4zz18-zzzzzzzzzzzzzz3', 'http://example.com/file1.txt', datetime.datetime(2018, 5, 17, 0, 0))) @@ -320,7 +320,7 @@ class TestHttpToKeep(unittest.TestCase): utcnow = mock.MagicMock() utcnow.return_value = datetime.datetime(2018, 5, 15) - r = http_to_keep(api, None, "http://example.com/download?fn=/file1.txt", utcnow=utcnow) + r = http_to_keep.http_to_keep(api, None, "http://example.com/download?fn=/file1.txt", utcnow=utcnow) self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt", 'zzzzz-4zz18-zzzzzzzzzzzzzz3', 'http://example.com/download?fn=/file1.txt', @@ -377,7 +377,7 @@ class TestHttpToKeep(unittest.TestCase): utcnow = mock.MagicMock() utcnow.return_value = datetime.datetime(2018, 5, 17) - r = http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow) + r = http_to_keep.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow) self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt", 'zzzzz-4zz18-zzzzzzzzzzzzzz3', 'http://example.com/file1.txt', datetime.datetime(2018, 5, 17, 0, 0))) @@ -428,7 +428,7 @@ class TestHttpToKeep(unittest.TestCase): utcnow = mock.MagicMock() utcnow.return_value = datetime.datetime(2018, 5, 17) - r = http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow, prefer_cached_downloads=True) + r = http_to_keep.http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow, prefer_cached_downloads=True) self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt", 'zzzzz-4zz18-zzzzzzzzzzzzzz3', 'http://example.com/file1.txt', datetime.datetime(2018, 5, 17, 0, 0))) @@ -475,8 +475,8 @@ class TestHttpToKeep(unittest.TestCase): utcnow = mock.MagicMock() utcnow.return_value = datetime.datetime(2018, 5, 17) - r = http_to_keep(api, None, "http://example.com/file1.txt?KeyId=123&Signature=456&Expires=789", - utcnow=utcnow, varying_url_params="KeyId,Signature,Expires") + r = http_to_keep.http_to_keep(api, None, "http://example.com/file1.txt?KeyId=123&Signature=456&Expires=789", + utcnow=utcnow, varying_url_params="KeyId,Signature,Expires") self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt", 'zzzzz-4zz18-zzzzzzzzzzzzzz3', 'http://example.com/file1.txt', datetime.datetime(2018, 5, 17, 0, 0))) -- 2.30.2