Merge branch '9964-output-glob-acr' refs #9964

[arvados.git] / sdk / python / discovery2pydoc.py

diff --git a/sdk/python/discovery2pydoc.py b/sdk/python/discovery2pydoc.py
index ad7606145a199b95738f0045cdf08fedbb3f1fe3..70a51371ace81e0850640d5e3394eb0d092a9fc3 100755 (executable)
--- a/sdk/python/discovery2pydoc.py
+++ b/sdk/python/discovery2pydoc.py
@@ -49,8 +49,8 @@ _ALIASED_METHODS = frozenset([
 ])
 _DEPRECATED_NOTICE = '''
 
 ])
 _DEPRECATED_NOTICE = '''
 

-!!! deprecated
-    This resource is deprecated in the Arvados API.
+.. WARNING:: Deprecated
+   This resource is deprecated in the Arvados API.

 '''
 _DEPRECATED_RESOURCES = frozenset([
     'Humans',
 '''
 _DEPRECATED_RESOURCES = frozenset([
     'Humans',


@@ -77,33 +77,87 @@ If you work with this raw object, the keys of the dictionary are documented
 below, along with their types. The `items` key maps to a list of matching
 `{cls_name}` objects.
 '''
 below, along with their types. The `items` key maps to a list of matching
 `{cls_name}` objects.
 '''

-_MODULE_PYDOC = '''Arvados API client documentation skeleton
-
-This module documents the methods and return types provided by the Arvados API
-client. Start with `ArvadosAPIClient`, which documents the methods available
-from the API client objects constructed by `arvados.api`. The implementation is
-generated dynamically at runtime when the client object is built.
+_MODULE_PYDOC = '''Arvados API client reference documentation
+
+This module provides reference documentation for the interface of the
+Arvados API client, including method signatures and type information for
+returned objects. However, the functions in `arvados.api` will return
+different classes at runtime that are generated dynamically from the Arvados
+API discovery document. The classes in this module do not have any
+implementation, and you should not instantiate them in your code.
+
+If you're just starting out, `ArvadosAPIClient` documents the methods
+available from the client object. From there, you can follow the trail into
+resource methods, request objects, and finally the data dictionaries returned
+by the API server.

 '''
 _SCHEMA_PYDOC = '''
 
 '''
 _SCHEMA_PYDOC = '''
 

-This is the dictionary object that represents a single {cls_name} in Arvados.
+This is the dictionary object that represents a single {cls_name} in Arvados
+and is returned by most `{cls_name}s` methods.

 The keys of the dictionary are documented below, along with their types.
 Not every key may appear in every dictionary returned by an API call.
 The keys of the dictionary are documented below, along with their types.
 Not every key may appear in every dictionary returned by an API call.

-Refer to the API documentation for details about how to retrieve specific keys
-if you need them.
+When a method doesn't return all the data, you can use its `select` parameter
+to list the specific keys you need. Refer to the API documentation for details.
+'''
+
+_MODULE_PRELUDE = '''
+import googleapiclient.discovery
+import googleapiclient.http
+import httplib2
+import sys
+from typing import Any, Dict, Generic, List, Optional, TypeVar
+if sys.version_info < (3, 8):
+    from typing_extensions import TypedDict
+else:
+    from typing import TypedDict
+
+# ST represents an API response type
+ST = TypeVar('ST', bound=TypedDict)
+'''
+_REQUEST_CLASS = '''
+class ArvadosAPIRequest(googleapiclient.http.HttpRequest, Generic[ST]):
+    """Generic API request object
+
+    When you call an API method in the Arvados Python SDK, it returns a
+    request object. You usually call `execute()` on this object to submit the
+    request to your Arvados API server and retrieve the response. `execute()`
+    will return the type of object annotated in the subscript of
+    `ArvadosAPIRequest`.
+    """
+
+    def execute(self, http: Optional[httplib2.Http]=None, num_retries: int=0) -> ST:
+        """Execute this request and return the response
+
+        Arguments:
+
+        * http: httplib2.Http | None --- The HTTP client object to use to
+          execute the request. If not specified, uses the HTTP client object
+          created with the API client object.
+
+        * num_retries: int --- The maximum number of times to retry this
+          request if the server returns a retryable failure. The API client
+          object also has a maximum number of retries specified when it is
+          instantiated (see `arvados.api.api_client`). This request is run
+          with the larger of that number and this argument. Default 0.
+        """
+

 '''
 
 '''
 

-_TYPE_MAP = {
+# Annotation represents a valid Python type annotation. Future development
+# could expand this to include other valid types like `type`.
+Annotation = str
+_TYPE_MAP: Mapping[str, Annotation] = {

     # Map the API's JavaScript-based type names to Python annotations.
     # Some of these may disappear after Arvados issue #19795 is fixed.
     # Map the API's JavaScript-based type names to Python annotations.
     # Some of these may disappear after Arvados issue #19795 is fixed.

-    'Array': 'list',
-    'array': 'list',
+    'Array': 'List',
+    'array': 'List',

     'boolean': 'bool',
     # datetime fields are strings in ISO 8601 format.
     'datetime': 'str',
     'boolean': 'bool',
     # datetime fields are strings in ISO 8601 format.
     'datetime': 'str',

-    'Hash': 'dict[str, Any]',
+    'Hash': 'Dict[str, Any]',

     'integer': 'int',
     'integer': 'int',

-    'object': 'dict[str, Any]',
+    'object': 'Dict[str, Any]',

     'string': 'str',
     'text': 'str',
 }
     'string': 'str',
     'text': 'str',
 }


@@ -172,20 +226,30 @@ class Parameter(inspect.Parameter):
         if default_value is None:
             default_doc = ''
         else:
         if default_value is None:
             default_doc = ''
         else:

-            default_doc = f" Default {default_value!r}."
-        # If there is no description, use a zero-width space to help Markdown
-        # parsers retain the definition list structure.
-        description = self._spec['description'] or '\u200b'
+            default_doc = f"Default {default_value!r}."
+        description = self._spec['description']
+        doc_parts = [f'{self.api_name}: {self.annotation}']
+        if description or default_doc:
+            doc_parts.append('---')
+            if description:
+                doc_parts.append(description)
+            if default_doc:
+                doc_parts.append(default_doc)

         return f'''
         return f'''

-{self.api_name}: {self.annotation}
-: {description}{default_doc}
+* {' '.join(doc_parts)}

 '''
 
 
 class Method:
 '''
 
 
 class Method:

-    def __init__(self, name: str, spec: Mapping[str, Any]) -> None:
+    def __init__(
+            self,
+            name: str,
+            spec: Mapping[str, Any],
+            annotate: Callable[[Annotation], Annotation]=str,
+    ) -> None:

         self.name = name
         self._spec = spec
         self.name = name
         self._spec = spec

+        self._annotate = annotate

         self._required_params = []
         self._optional_params = []
         for param_name, param_spec in spec['parameters'].items():
         self._required_params = []
         self._optional_params = []
         for param_name, param_spec in spec['parameters'].items():


@@ -207,7 +271,8 @@ class Method:
         try:
             returns = get_type_annotation(self._spec['response']['$ref'])
         except KeyError:
         try:
             returns = get_type_annotation(self._spec['response']['$ref'])
         except KeyError:

-            returns = 'dict[str, Any]'
+            returns = 'Dict[str, Any]'
+        returns = self._annotate(returns)

         return inspect.Signature(parameters, return_annotation=returns)
 
     def doc(self, doc_slice: slice=slice(None)) -> str:
         return inspect.Signature(parameters, return_annotation=returns)
 
     def doc(self, doc_slice: slice=slice(None)) -> str:


@@ -271,7 +336,7 @@ def document_resource(name: str, spec: Mapping[str, Any]) -> str:
     if class_name in _DEPRECATED_RESOURCES:
         docstring += _DEPRECATED_NOTICE
     methods = [
     if class_name in _DEPRECATED_RESOURCES:
         docstring += _DEPRECATED_NOTICE
     methods = [

-        Method(key, meth_spec)
+        Method(key, meth_spec, 'ArvadosAPIRequest[{}]'.format)

         for key, meth_spec in spec['methods'].items()
         if key not in _ALIASED_METHODS
     ]
         for key, meth_spec in spec['methods'].items()
         if key not in _ALIASED_METHODS
     ]


@@ -308,10 +373,11 @@ If not provided, retrieved dynamically from Arvados client configuration.
         parts = urllib.parse.urlsplit(args.discovery_url)
         if not (parts.scheme or parts.netloc):
             args.discovery_url = pathlib.Path(args.discovery_url).resolve().as_uri()
         parts = urllib.parse.urlsplit(args.discovery_url)
         if not (parts.scheme or parts.netloc):
             args.discovery_url = pathlib.Path(args.discovery_url).resolve().as_uri()

+    # Our output is Python source, so it should be UTF-8 regardless of locale.

     if args.output_file == STDSTREAM_PATH:
     if args.output_file == STDSTREAM_PATH:

-        args.out_file = sys.stdout
+        args.out_file = open(sys.stdout.fileno(), 'w', encoding='utf-8', closefd=False)

     else:
     else:

-        args.out_file = args.output_file.open('w')
+        args.out_file = args.output_file.open('w', encoding='utf-8')

     return args
 
 def main(arglist: Optional[Sequence[str]]=None) -> int:
     return args
 
 def main(arglist: Optional[Sequence[str]]=None) -> int:


@@ -327,8 +393,8 @@ def main(arglist: Optional[Sequence[str]]=None) -> int:
         discovery_document = json.load(discovery_file)
     print(
         to_docstring(_MODULE_PYDOC, indent=0),
         discovery_document = json.load(discovery_file)
     print(
         to_docstring(_MODULE_PYDOC, indent=0),

-        '''from typing import Any, TypedDict''',
-        sep='\n\n', end='\n\n', file=args.out_file,
+        _MODULE_PRELUDE,
+        sep='\n', file=args.out_file,

     )
 
     schemas = sorted(discovery_document['schemas'].items())
     )
 
     schemas = sorted(discovery_document['schemas'].items())


@@ -339,7 +405,11 @@ def main(arglist: Optional[Sequence[str]]=None) -> int:
     for name, resource_spec in resources:
         print(document_resource(name, resource_spec), file=args.out_file)
 
     for name, resource_spec in resources:
         print(document_resource(name, resource_spec), file=args.out_file)
 

-    print('''class ArvadosAPIClient:''', file=args.out_file)
+    print(
+        _REQUEST_CLASS,
+        '''class ArvadosAPIClient(googleapiclient.discovery.Resource):''',
+        sep='\n', file=args.out_file,
+    )

     for name, _ in resources:
         class_name = classify_name(name)
         docstring = f"Return an instance of `{class_name}` to call methods via this client"
     for name, _ in resources:
         class_name = classify_name(name)
         docstring = f"Return an instance of `{class_name}` to call methods via this client"


@@ -354,6 +424,7 @@ def main(arglist: Optional[Sequence[str]]=None) -> int:
         }
         print(Method(name, method_spec).doc(), file=args.out_file)
 
         }
         print(Method(name, method_spec).doc(), file=args.out_file)
 

+    args.out_file.close()

     return os.EX_OK
 
 if __name__ == '__main__':
     return os.EX_OK
 
 if __name__ == '__main__':