+# Retrieve all of the source collection manifest texts
+src_collection_uuid_list = [
+ 'zzzzz-4zz18-111111111111111',
+ 'zzzzz-4zz18-222222222222222',
+ ...,
+]
+manifest_texts = [
+ arvados.collection.Collection(uuid).manifest_text()
+ for uuid in src_collection_uuid_list
+]
+
+# Initialize a new collection object from the concatenated manifest text
+new_collection = arvados.collection.Collection(''.join(manifest_texts), ...)
+
+# Record the new collection in Arvados
+new_collection.save_new(
+ name='Collection merged by Python SDK',
+ owner_uuid='zzzzz-j7d0g-12345abcde67890',
+)
+{% endcodeblock %}
+
+h3(#sharing-link). Create a collection sharing link
+
+You can create a sharing link for a collection by creating a new API token that is only allowed to read that collection; then constructing a link to your Keep web server that includes the collection UUID and the new token.
+
+{% codeblock as python %}
+import urllib.parse
+
+# The UUID of the collection you want to share
+collection_uuid = 'zzzzz-4zz18-12345abcde67890'
+
+sharing_token_scopes = [
+ 'GET /arvados/v1/keep_services/accessible',
+ f'GET /arvados/v1/collections/{collection_uuid}',
+ f'GET /arvados/v1/collections/{collection_uuid}/',
+]
+sharing_token = arv_client.api_client_authorizations().create(
+ body={
+ 'api_client_authorization': {
+ 'scopes': sharing_token_scopes,
+ },
+ },
+).execute()
+
+sharing_url_parts = (
+ # The scheme your Keep web server uses. Change this to 'http' if necessary.
+ 'https',
+ # The hostname, and optionally port, your Keep web server uses
+ 'collections.zzzzz.example.com',
+ # You shouldn't need to change any other items
+ f'/c={collection_uuid}/t={sharing_token["api_token"]}/_/',
+ None,
+ None,
+)
+sharing_url = urllib.parse.urlunsplit(sharing_url_parts)
+print(sharing_url)
+{% endcodeblock %}
+
+h2(#working-with-containers). Working with containers
+
+If you haven't already, start by reading the "Computing with Crunch":{{ site.baseurl }}/api/execution.html guide. It provides a high-level overview of how users submit work to Arvados as container requests; how Arvados dispatches that work to containers; and how Arvados records the association and results back on the original container request record.
+
+If you have experience running CWL workflows on Workbench 2, it runs through this same API. When you start that workflow run, Workbench 2 creates a small container request to run a "CWL runner" tool with the specific inputs you gave it. Once Crunch dispatches a container for it, the CWL runner creates additional container requests to run each step of the workflow, and oversees the process until the workflow runs to completion. The UUID of this container is recorded in the @container_uuid@ field of the container request you submitted.
+
+The UUID of the CWL runner container is recorded in the @requesting_container_uuid@ field of each container request it creates. You can list container requests with a filter on this field to inspect each step of the workflow individually, as shown below.
+
+The next few sections include two examples: a high-level example that describes how to work with any container request, and a more specific example that provides more detail about how to work with CWL workflow runs.
+
+h3(#get-input-of-a-cwl-workflow). Get input of a container or CWL workflow run
+
+A container request's most varied inputs are recorded in the @mounts@ field, which can include data from Keep, specific collections, Git checkouts, and static files. You might also be interested in the @environment@, @command@, @container_image@, and @secret_mounts@ fields. Refer to the "container requests API documentation":{{ site.baseurl }}/api/methods/container_requests.html for details.
+
+{% codeblock as python %}
+container_request = arv_client.container_requests().get(
+ uuid='zzzzz-xvhdp-12345abcde67890',
+).execute()
+# From here, you can process any of the container request's input fields.
+# Below is an example of listing all the mounts.
+import pprint
+for mount_name, mount_source in container_request['mounts'].items():
+ mount_summary = []
+ # These are the fields that define different types of mounts.
+ # Try to collect them all. Just skip any that aren't set.
+ for key in ['kind', 'uuid', 'portable_data_hash', 'commit', 'path']:
+ try:
+ mount_summary.append(mount_source[key])
+ except KeyError:
+ pass
+ print(f"{mount_name}: {' '.join(mount_summary)}")
+ if mount_source.get('kind') == 'json':
+ pprint.pprint(mount_source.get('content'))
+{% endcodeblock %}
+
+When you run a CWL workflow, the CWL inputs are stored in a JSON mount named @/var/lib/cwl/cwl.input.json@.
+
+{% codeblock as python %}
+container_request = arv_client.container_requests().get(
+ uuid='zzzzz-xvhdp-12345abcde67890',
+).execute()
+cwl_input = container_request['mounts']['/var/lib/cwl/cwl.input.json']['content']
+... # Work with the cwl_input dictionary
+{% endcodeblock %}
+
+h3(#get-output-of-a-cwl-workflow). Get output of a container or CWL workflow run
+
+A container's output files are saved in a collection. The UUID of that collection is recorded in the @output_uuid@ of the container request, which you can load as you like.
+
+{% codeblock as python %}
+import arvados.collection
+container_request = arv_client.container_requests().get(
+ uuid='zzzzz-xvhdp-12345abcde67890',
+).execute()
+container_output = arvados.collection.Collection(
+ container_request.get('output_uuid'),
+)
+... # Work with the container_output collection object
+{% endcodeblock %}
+
+When you run a CWL workflow, the output collection includes a file named @cwl.output.json@ that provides additional information about other files in the output.
+
+{% codeblock as python %}
+import arvados.collection
+import json
+cwl_container_request = arv_client.container_requests().get(
+ uuid='zzzzz-xvhdp-12345abcde67890',
+).execute()
+cwl_output_collection = arvados.collection.Collection(
+ cwl_container_request['output_uuid'],
+)
+with cwl_output_collection.open('cwl.output.json') as cwl_output_file:
+ cwl_output = json.load(cwl_output_file)
+... # Work with the cwl_output dictionary
+{% endcodeblock %}
+
+h3(#get-log-of-a-child-request). Get logs of a container or CWL workflow run
+
+A container's log files are saved in a collection. The UUID of that collection is recorded in the @log_uuid@ of the container request, which you can load as you like.
+
+{% codeblock as python %}
+import arvados.collection
+container_request = arv_client.container_requests().get(
+ uuid='zzzzz-xvhdp-12345abcde67890',
+).execute()
+log_collection = arvados.collection.Collection(
+ container_request['log_uuid'],
+)
+# From here, you can process the container's log collection any way you like.
+# Below is an example that writes the container's stderr to this process' stderr.
+import shutil
+import sys
+with log_collection.open('stderr.txt') as containter_stderr:
+ shutil.copyfileobj(container_stderr, sys.stderr)
+{% endcodeblock %}
+
+h3(#get-state-of-a-cwl-workflow). Get status of a container or CWL workflow run
+
+Workbench shows users a single status badge for container requests. This status is synthesized from different fields on the container request and associated container. This code shows how to do analogous reporting using the Python SDK.
+
+{% codeblock as python %}
+container_request = arv_client.container_requests().get(
+ uuid='zzzzz-xvhdp-12345abcde67890',
+).execute()
+if container_request['container_uuid'] is None:
+ status = container_request['state']
+else:
+ container = arv_client.containers().get(
+ uuid=container_request['container_uuid'],
+ ).execute()
+ container_state = container['state']
+ if container_state == 'Queued' or container_state == 'Locked':
+ status = "On hold" if container['priority'] == 0 else "Queued"
+ elif container_state == 'Running':
+ if container['runtime_status'].get('error'):
+ status = "Failing"
+ elif container['runtime_status'].get('warning'):
+ status = "Warning"
+ else:
+ status = container_state
+ elif container_state == 'Cancelled':
+ status = container_state
+ elif container_state == 'Complete':
+ status = "Completed" if container['exit_code'] == 0 else "Failed"
+... # Report status as desired
+{% endcodeblock %}
+
+h3(#list-failed-child-requests). List child requests of a container or CWL workflow run
+
+When a running container creates a container request to do additional work, the UUID of the source container is recorded in the @requesting_container_uuid@ field of the new container request. You can list container requests with this filter to find requests created by a specific container.
+
+{% codeblock as python %}
+import arvados.util
+for child_container_requests in arvados.util.keyset_list_all(
+ # Do *not* call the method here, just pass it.
+ arv_client.container_requests().list,
+ filters=[
+ # Note this is a container UUID, *not* a container request UUID
+ ['requesting_container_uuid', '=', 'zzzzz-dz642-12345abcde67890'],
+ # You may add other filters for your listing.
+ # For example, you could filter by 'name' to find specific kinds
+ # of steps of a CWL workflow.
+ ...,
+ ],
+):
+ ... # Work with each child container request
+{% endcodeblock %}
+
+If your input only provides the UUID for a container request rather than a container, you can get that container request, then follow the @container_uuid@ field if it is set. (It might not be if the container request has not been dispatched yet.)
+
+{% codeblock as python %}
+import arvados.util
+parent_container_request = arv_client.container_requests().get(
+ uuid='zzzzz-xvhdp-12345abcde67890',
+).execute()
+parent_container_uuid = parent_container_request['container_uuid']
+if parent_container_uuid is None:
+ # No container has run for this request yet, so there cannot be child requests.
+ child_container_requests = ()
+else:
+ child_container_requests = arvados.util.keyset_list_all(
+ # Do *not* call the method here, just pass it.
+ arv_client.container_requests().list,
+ filters=[
+ ['requesting_container_uuid', '=', parent_container_uuid],
+ # You may add other filters for your listing.
+ # For example, you could filter by 'name' to find specific kinds
+ # of steps of a CWL workflow.
+ ...,
+ ],
+ )
+for child_container_request in child_container_requests:
+ ... # Work with each child container request
+{% endcodeblock %}
+
+With each child container request, you could repeat any of the recipes listed earlier in this section: examine their status, inputs, outputs, logs, and so on.
+
+h2(#working-with-container-request-queue). Working with the container request queue
+
+h3(#list-completed-container-requests). List completed container requests
+
+Completed container requests have their @state@ field set to @"Final"@. You can list container requests with this filter to find completed requests.
+
+{% codeblock as python %}
+import arvados.util
+import datetime
+time_filter = datetime.datetime.utcnow()
+time_filter -= datetime.timedelta(days=7)
+
+for container_request in arvados.util.keyset_list_all(
+ # Do *not* call the method here, just pass it.
+ arv_client.container_requests().list,
+ filters=[
+ # This is the filter you need to find completed container requests.
+ ['state', '=', 'Final'],
+ # There could be many completed container requests, so you should
+ # provide additional filters. This example limits the listing to
+ # container requests from the past week.
+ ['created_at', '>=', f'{time_filter.isoformat()}Z'],
+ ...,
+ ],
+):
+ # Work with each container_request as desired.
+ # This example provides a basic status table with the container request
+ # UUID, time the request was created, and time the container finished
+ # (both in UTC).
+ print(
+ container_request['uuid'],
+ container_request['created_at'],
+ container_request['modified_at'],
+ )
+{% endcodeblock %}
+
+h3(#cancel-a-container-request). Cancel a container request
+
+To cancel a container request, update it to set its @priority@ field to 0. See the "containers API reference":{{ site.baseurl }}/api/methods/containers.html for details.
+
+{% codeblock as python %}
+cancelled_container_request = arv_client.container_requests().update(
+ uuid='zzzzz-xvhdp-12345abcde67890',
+ body={
+ 'container_request': {
+ 'priority': 0,
+ },
+ },
+).execute()
+{% endcodeblock %}
+
+p(#cancel-all-container-requests). If you want to cancel many container requests, you can list container requests with the @state@ field set to @"Committed"@, a @priority@ greater than zero, and any other filters you like. Then update each container request in turn.
+
+{% codeblock as python %}
+import arvados.util
+for container_request in arvados.util.keyset_list_all(
+ # Do *not* call the method here, just pass it.
+ arv_client.container_requests().list,
+ filters=[
+ # These are the filters you need to find cancellable container requests.
+ ['state', '=', 'Committed'],
+ ['priority', '>', 0],
+ # You can add other filters as desired.
+ # For example, you might filter on `requesting_container_uuid` to
+ # cancel only steps of one specific workflow.
+ ...,
+ ],
+):
+ cancelled_container_request = arv_client.container_requests().update(
+ uuid=container_request['uuid'],
+ body={
+ 'container_request': {
+ 'priority': 0,
+ },
+ },
+ ).execute()
+{% endcodeblock %}
+
+h2(#working-with-vocabularies). Working with vocabularies
+
+h3(#querying-the-vocabulary-definition). Query the vocabulary definition
+
+The @arvados.vocabulary@ module provides facilities to interact with the "active metadata vocabulary":{{ site.baseurl }}/admin/metadata-vocabulary.html in the system. The "@Vocabulary@ class":{{ site.baseurl }}/sdk/python/arvados/vocabulary.html#arvados.vocabulary.Vocabulary provides a mapping-like view of a cluster's configured vocabulary.
+
+{% codeblock as python %}
+import arvados.vocabulary
+vocabulary = arvados.vocabulary.load_vocabulary(arv_client)
+
+# You can use the vocabulary object to access specific keys and values by
+# case-insensitive mapping, like this:
+# vocabulary_value = vocabulary[key_alias][value_alias]
+# You can also access the `key_aliases` and `value_aliases` mapping
+# attributes directly to view the entire vocabulary. The example below
+# writes a plaintext table of the vocabulary.
+for vocabulary_key in set(vocabulary.key_aliases.values()):
+ print(
+ vocabulary_key.identifier,
+ vocabulary_key.preferred_label,
+ ', '.join(vocabulary_key.aliases[1:]),
+ sep='\t',
+ )
+ for vocabulary_value in set(vocabulary_key.value_aliases.values()):
+ print(
+ f'↳ {vocabulary_value.identifier}',
+ vocabulary_value.preferred_label,
+ ', '.join(vocabulary_value.aliases[1:]),
+ sep='\t',
+ )
+{% endcodeblock %}
+
+h3(#translating-between-vocabulary-identifiers-and-labels). Translate between vocabulary identifiers and labels
+
+Client software might need to present properties to the user in a human-readable form or take input from the user without requiring them to remember identifiers. The "@Vocabulary.convert_to_labels@":{{ site.baseurl }}/sdk/python/arvados/vocabulary.html#arvados.vocabulary.Vocabulary.convert_to_labels and "@Vocabulary.convert_to_identifiers@":{{ site.baseurl }}/sdk/python/arvados/vocabulary.html#arvados.vocabulary.Vocabulary.convert_to_identifiers methods help with these tasks, respectively.
+
+{% codeblock as python %}
+import arvados.vocabulary
+vocabulary = arvados.vocabulary.load_vocabulary(arv_client)
+
+# The argument should be a mapping of vocabulary keys and values using any
+# defined aliases, like this:
+# {'Creature': 'Human', 'Priority': 'Normal'}
+# The return value will be an analogous mapping where all the aliases have
+# been translated to identifiers, like this:
+# {'IDTAGANIMALS': 'IDVALANIMALS2', 'IDTAGIMPORTANCES': 'IDTAGIMPORTANCES1'}
+properties_by_identifier = vocabulary.convert_to_identifiers({...})
+
+# You can use this to set metadata properties on objects that support them.
+project = arv_client.groups().update(
+ uuid='zzzzz-j7d0g-12345abcde67890',
+ body={
+ 'group': {
+ 'properties': properties_by_identifier,
+ },
+ },
+).execute()
+
+# You can report properties to the user by their preferred name.
+print(f"{project['name']} ({project['group_class']} {project['uuid']}) updated with properties:")
+for key, value in vocabulary.convert_to_labels(project['properties']).items():
+ print(f"↳ {key}: {value}")