From 9b64ec17351e3c034b4f96470b12a1c00ff6813f Mon Sep 17 00:00:00 2001 From: Brett Smith Date: Fri, 29 Sep 2023 11:28:50 -0400 Subject: [PATCH] Merge branch '19818-api-pydoc' Closes #19818. Arvados-DCO-1.1-Signed-off-by: Brett Smith --- sdk/cwl/arvados_cwl/__init__.py | 2 - sdk/python/arvados/api.py | 91 +++++++++++++------ sdk/python/arvados/commands/arv_copy.py | 4 +- .../crunchstat_summary/summarizer.py | 7 +- 4 files changed, 68 insertions(+), 36 deletions(-) diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py index 8108934aae..7968fb1e2b 100644 --- a/sdk/cwl/arvados_cwl/__init__.py +++ b/sdk/cwl/arvados_cwl/__init__.py @@ -32,7 +32,6 @@ import arvados.logging from arvados.keep import KeepClient from arvados.errors import ApiError import arvados.commands._util as arv_cmd -from arvados.api import OrderedJsonModel from .perf import Perf from ._version import __version__ @@ -338,7 +337,6 @@ def main(args=sys.argv[1:], if api_client is None: api_client = arvados.safeapi.ThreadSafeApiCache( api_params={ - 'model': OrderedJsonModel(), 'num_retries': arvargs.retries, 'timeout': arvargs.http_timeout, }, diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py index c51be82b20..c62c1a74a9 100644 --- a/sdk/python/arvados/api.py +++ b/sdk/python/arvados/api.py @@ -43,13 +43,12 @@ _logger = logging.getLogger('arvados.api') _googleapiclient_log_lock = threading.Lock() MAX_IDLE_CONNECTION_DURATION = 30 - -# These constants supported our own retry logic that we've since removed in -# favor of using googleapiclient's num_retries. They're kept here purely for -# API compatibility, but set to 0 to indicate no retries happen. -RETRY_DELAY_INITIAL = 0 -RETRY_DELAY_BACKOFF = 0 -RETRY_COUNT = 0 +""" +Number of seconds that API client HTTP connections should be allowed to idle +in keepalive state before they are forced closed. Client code can adjust this +constant, and it will be used for all Arvados API clients constructed after +that point. +""" # An unused HTTP 5xx status code to request a retry internally. # See _intercept_http_request. This should not be user-visible. @@ -58,26 +57,6 @@ _RETRY_4XX_STATUS = 545 if sys.version_info >= (3,): httplib2.SSLHandshakeError = None -class OrderedJsonModel(apiclient.model.JsonModel): - """Model class for JSON that preserves the contents' order. - - API clients that care about preserving the order of fields in API - server responses can use this model to do so, like this: - - from arvados.api import OrderedJsonModel - client = arvados.api('v1', ..., model=OrderedJsonModel()) - """ - - def deserialize(self, content): - # This is a very slightly modified version of the parent class' - # implementation. Copyright (c) 2010 Google. - content = content.decode('utf-8') - body = json.loads(content, object_pairs_hook=collections.OrderedDict) - if self._data_wrapper and isinstance(body, dict) and 'data' in body: - body = body['data'] - return body - - _orig_retry_request = apiclient.http._retry_request def _retry_request(http, num_retries, *args, **kwargs): try: @@ -174,6 +153,18 @@ def _new_http_error(cls, *args, **kwargs): apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error) def http_cache(data_type): + """Set up an HTTP file cache + + This function constructs and returns an `arvados.cache.SafeHTTPCache` + backed by the filesystem under `~/.cache/arvados/`, or `None` if the + directory cannot be set up. The return value can be passed to + `httplib2.Http` as the `cache` argument. + + Arguments: + + * data_type: str --- The name of the subdirectory under `~/.cache/arvados` + where data is cached. + """ try: homedir = pathlib.Path.home() except RuntimeError: @@ -492,3 +483,49 @@ def api_from_config(version=None, apiconfig=None, **kwargs): docstring for more information about their meaning. """ return api(**api_kwargs_from_config(version, apiconfig, **kwargs)) + +class OrderedJsonModel(apiclient.model.JsonModel): + """Model class for JSON that preserves the contents' order + + .. WARNING:: Deprecated + This model is redundant now that Python dictionaries preserve insertion + ordering. Code that passes this model to API constructors can remove it. + + In Python versions before 3.6, API clients that cared about preserving the + order of fields in API server responses could use this model to do so. + Typical usage looked like: + + from arvados.api import OrderedJsonModel + client = arvados.api('v1', ..., model=OrderedJsonModel()) + """ + @util._deprecated(preferred="the default model and rely on Python's built-in dictionary ordering") + def __init__(self, data_wrapper=False): + return super().__init__(data_wrapper) + + +RETRY_DELAY_INITIAL = 0 +""" +.. WARNING:: Deprecated + This constant was used by retry code in previous versions of the Arvados SDK. + Changing the value has no effect anymore. + Prefer passing `num_retries` to an API client constructor instead. + Refer to the constructor docstrings for details. +""" + +RETRY_DELAY_BACKOFF = 0 +""" +.. WARNING:: Deprecated + This constant was used by retry code in previous versions of the Arvados SDK. + Changing the value has no effect anymore. + Prefer passing `num_retries` to an API client constructor instead. + Refer to the constructor docstrings for details. +""" + +RETRY_COUNT = 0 +""" +.. WARNING:: Deprecated + This constant was used by retry code in previous versions of the Arvados SDK. + Changing the value has no effect anymore. + Prefer passing `num_retries` to an API client constructor instead. + Refer to the constructor docstrings for details. +""" diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py index ef3936e267..7326840896 100755 --- a/sdk/python/arvados/commands/arv_copy.py +++ b/sdk/python/arvados/commands/arv_copy.py @@ -49,7 +49,6 @@ import arvados.commands.keepdocker import arvados.http_to_keep import ruamel.yaml as yaml -from arvados.api import OrderedJsonModel from arvados._version import __version__ COMMIT_HASH_RE = re.compile(r'^[0-9a-f]{1,40}$') @@ -208,7 +207,7 @@ def set_src_owner_uuid(resource, uuid, args): def api_for_instance(instance_name, num_retries): if not instance_name: # Use environment - return arvados.api('v1', model=OrderedJsonModel()) + return arvados.api('v1') if '/' in instance_name: config_file = instance_name @@ -232,7 +231,6 @@ def api_for_instance(instance_name, num_retries): host=cfg['ARVADOS_API_HOST'], token=cfg['ARVADOS_API_TOKEN'], insecure=api_is_insecure, - model=OrderedJsonModel(), num_retries=num_retries, ) else: diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py index a876257abc..2bd329719b 100644 --- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py +++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py @@ -15,7 +15,6 @@ import sys import threading import _strptime -from arvados.api import OrderedJsonModel from crunchstat_summary import logger # Recommend memory constraints that are this multiple of an integral @@ -518,7 +517,7 @@ def NewSummarizer(process_or_uuid, **kwargs): else: uuid = process_or_uuid process = None - arv = arvados.api('v1', model=OrderedJsonModel()) + arv = arvados.api('v1') if '-dz642-' in uuid: if process is None: @@ -639,7 +638,7 @@ class MultiSummarizer(object): class JobTreeSummarizer(MultiSummarizer): """Summarizes a job and all children listed in its components field.""" def __init__(self, job, label=None, **kwargs): - arv = arvados.api('v1', model=OrderedJsonModel()) + arv = arvados.api('v1') label = label or job.get('name', job['uuid']) children = collections.OrderedDict() children[job['uuid']] = JobSummarizer(job, label=label, **kwargs) @@ -683,7 +682,7 @@ class PipelineSummarizer(MultiSummarizer): class ContainerRequestTreeSummarizer(MultiSummarizer): def __init__(self, root, skip_child_jobs=False, **kwargs): - arv = arvados.api('v1', model=OrderedJsonModel()) + arv = arvados.api('v1') label = kwargs.pop('label', None) or root.get('name') or root['uuid'] root['name'] = label -- 2.30.2