16347: Merge branch 'main'

[arvados.git] / doc / sdk / python / cookbook.html.textile.liquid

diff --git a/doc/sdk/python/cookbook.html.textile.liquid b/doc/sdk/python/cookbook.html.textile.liquid
index fa059f7028cd25db55e693d26439b5ce91697ecf..eda6563d7a1de99602eecada47eab021807a1782 100644 (file)
--- a/doc/sdk/python/cookbook.html.textile.liquid
+++ b/doc/sdk/python/cookbook.html.textile.liquid
@@ -12,22 +12,25 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 h2. Cancel a container request
 
 
 h2. Cancel a container request
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 arvados.api().container_requests().update(uuid=container_request_uuid, body={"priority": 0}).execute()
 arvados.api().container_requests().update(uuid=container_request_uuid, body={"priority": 0}).execute()

-</code></pre>
+{% endcodeblock %}

 
 h2. Cancel all container requests
 
 
 h2. Cancel all container requests
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 api = arvados.api()
 result = api.container_requests().list(filters=[["state", "=", "Committed"], ["priority", ">", "0"]]).execute()["items"]
 for container_request in result:
     api.container_requests().update(uuid=container_request["uuid"], body={"priority": 0}).execute()
 api = arvados.api()
 result = api.container_requests().list(filters=[["state", "=", "Committed"], ["priority", ">", "0"]]).execute()["items"]
 for container_request in result:
     api.container_requests().update(uuid=container_request["uuid"], body={"priority": 0}).execute()

-</code></pre>
+{% endcodeblock %}

 
 h2. List completed container requests
 
 
 h2. List completed container requests
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 api = arvados.api()
 result = api.container_requests().list(filters=[["name", "like", name], ["state", "=", "Final"]]).execute()["items"]
 container_uuids = [cr["container_uuid"] for cr in result]
 api = arvados.api()
 result = api.container_requests().list(filters=[["name", "like", name], ["state", "=", "Final"]]).execute()["items"]
 container_uuids = [cr["container_uuid"] for cr in result]


@@ -37,35 +40,69 @@ container_dict = {c["uuid"]: c for c in containers}
 for container_request in result:
     container = container_dict[container_request["container_uuid"]]
     print("%s, %s, %s" % (container_request["uuid"], container_request["name"], "Success" if container["exit_code"] == 0 else "Failed"))
 for container_request in result:
     container = container_dict[container_request["container_uuid"]]
     print("%s, %s, %s" % (container_request["uuid"], container_request["name"], "Success" if container["exit_code"] == 0 else "Failed"))

-</code></pre>
+{% endcodeblock %}

 
 h2. Get input of a CWL workflow
 
 
 h2. Get input of a CWL workflow
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 api = arvados.api()
 api = arvados.api()

-container_request = arvados.api().container_requests().get(uuid=container_request_uuid).execute()
+container_request_uuid="zzzzz-xvhdp-zzzzzzzzzzzzzzz"
+container_request = api.container_requests().get(uuid=container_request_uuid).execute()

 print(container_request["mounts"]["/var/lib/cwl/cwl.input.json"])
 print(container_request["mounts"]["/var/lib/cwl/cwl.input.json"])

-</code></pre>
+{% endcodeblock %}

 
 h2. Get output of a CWL workflow
 
 
 h2. Get output of a CWL workflow
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 import arvados.collection
 api = arvados.api()
 import arvados.collection
 api = arvados.api()

-container_request = arvados.api().container_requests().get(uuid=container_request_uuid).execute()
+container_request_uuid="zzzzz-xvhdp-zzzzzzzzzzzzzzz"
+container_request = api.container_requests().get(uuid=container_request_uuid).execute()

 collection = arvados.collection.CollectionReader(container_request["output_uuid"])
 print(collection.open("cwl.output.json").read())
 collection = arvados.collection.CollectionReader(container_request["output_uuid"])
 print(collection.open("cwl.output.json").read())

-</code></pre>
+{% endcodeblock %}
+
+h2. Get state of a CWL workflow
+
+{% codeblock as python %}
+import arvados
+def get_cr_state(cr_uuid):
+    api = arvados.api()
+    cr = api.container_requests().get(uuid=cr_uuid).execute()
+    if cr['container_uuid'] is None:
+        return cr['state']
+    c = api.containers().get(uuid=cr['container_uuid']).execute()
+    if cr['state'] == 'Final' and c['state'] != 'Complete':
+        return 'Cancelled'
+    elif c['state'] in ['Locked', 'Queued']:
+        if c['priority'] == 0:
+            return 'On hold'
+        else:
+            return 'Queued'
+    elif c['state'] == 'Complete' and c['exit_code'] != 0:
+        return 'Failed'
+    elif c['state'] == 'Running':
+        if c['runtime_status'].get('error', None):
+            return 'Failing'
+        elif c['runtime_status'].get('warning', None):
+            return 'Warning'
+    return c['state']
+container_request_uuid = 'zzzzz-xvhdp-zzzzzzzzzzzzzzz'
+print(get_cr_state(container_request_uuid))
+{% endcodeblock %}

 
 h2. List input of child requests
 
 
 h2. List input of child requests
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 api = arvados.api()
 api = arvados.api()

-parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
-namefilter = "bwa%"
-container_request = arvados.api().container_requests().get(uuid=parent_request_uuid).execute()
+parent_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
+namefilter = "bwa%"  # the "like" filter uses SQL pattern match syntax
+container_request = api.container_requests().get(uuid=parent_request_uuid).execute()

 parent_container_uuid = container_request["container_uuid"]
 parent_container_uuid = container_request["container_uuid"]

-child_requests = arvados.api().container_requests().list(filters=[
+child_requests = api.container_requests().list(filters=[

     ["requesting_container_uuid", "=", parent_container_uuid],
     ["name", "like", namefilter]]).execute()
 for c in child_requests["items"]:
     ["requesting_container_uuid", "=", parent_container_uuid],
     ["name", "like", namefilter]]).execute()
 for c in child_requests["items"]:


@@ -73,81 +110,191 @@ for c in child_requests["items"]:
     for m in c["mounts"].values():
         if "portable_data_hash" in m:
             print("  %s" % m["portable_data_hash"])
     for m in c["mounts"].values():
         if "portable_data_hash" in m:
             print("  %s" % m["portable_data_hash"])

-</code></pre>
+{% endcodeblock %}

 
 h2. List output of child requests
 
 
 h2. List output of child requests
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 api = arvados.api()
 api = arvados.api()

-parent_request_uuid = "qr1hi-xvhdp-61y7cjymnhyp7yu"
-namefilter = "bwa%"
-container_request = arvados.api().container_requests().get(uuid=parent_request_uuid).execute()
+parent_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
+namefilter = "bwa%"  # the "like" filter uses SQL pattern match syntax
+container_request = api.container_requests().get(uuid=parent_request_uuid).execute()

 parent_container_uuid = container_request["container_uuid"]
 parent_container_uuid = container_request["container_uuid"]

-child_requests = arvados.api().container_requests().list(filters=[
+child_requests = api.container_requests().list(filters=[

     ["requesting_container_uuid", "=", parent_container_uuid],
     ["name", "like", namefilter]]).execute()
 output_uuids = [c["output_uuid"] for c in child_requests["items"]]
     ["requesting_container_uuid", "=", parent_container_uuid],
     ["name", "like", namefilter]]).execute()
 output_uuids = [c["output_uuid"] for c in child_requests["items"]]

-collections = arvados.api().collections().list(filters=[["uuid", "in", output_uuids]]).execute()
+collections = api.collections().list(filters=[["uuid", "in", output_uuids]]).execute()

 uuid_to_pdh = {c["uuid"]: c["portable_data_hash"] for c in collections["items"]}
 for c in child_requests["items"]:
     print("%s -> %s" % (c["name"], uuid_to_pdh[c["output_uuid"]]))
 uuid_to_pdh = {c["uuid"]: c["portable_data_hash"] for c in collections["items"]}
 for c in child_requests["items"]:
     print("%s -> %s" % (c["name"], uuid_to_pdh[c["output_uuid"]]))

-</code></pre>
+{% endcodeblock %}

 
 h2. List failed child requests
 
 
 h2. List failed child requests
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 api = arvados.api()
 api = arvados.api()

-parent_request_uuid = "qr1hi-xvhdp-3d8f1zatx1xio36"
-container_request = arvados.api().container_requests().get(uuid=parent_request_uuid).execute()
+parent_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
+container_request = api.container_requests().get(uuid=parent_request_uuid).execute()

 parent_container_uuid = container_request["container_uuid"]
 parent_container_uuid = container_request["container_uuid"]

-child_requests = arvados.api().container_requests().list(filters=[
+child_requests = api.container_requests().list(filters=[

     ["requesting_container_uuid", "=", parent_container_uuid]], limit=1000).execute()
 child_containers = {c["container_uuid"]: c for c in child_requests["items"]}
     ["requesting_container_uuid", "=", parent_container_uuid]], limit=1000).execute()
 child_containers = {c["container_uuid"]: c for c in child_requests["items"]}

-cancelled_child_containers = arvados.api().containers().list(filters=[
+cancelled_child_containers = api.containers().list(filters=[

     ["exit_code", "!=", "0"],
     ["exit_code", "!=", "0"],

-    ["uuid", "in", child_containers.keys()]], limit=1000).execute()
+    ["uuid", "in", list(child_containers.keys())]], limit=1000).execute()

 for c in cancelled_child_containers["items"]:
     print("%s (%s)" % (child_containers[c["uuid"]]["name"], child_containers[c["uuid"]]["uuid"]))
 for c in cancelled_child_containers["items"]:
     print("%s (%s)" % (child_containers[c["uuid"]]["name"], child_containers[c["uuid"]]["uuid"]))

-</code></pre>
+{% endcodeblock %}

 
 h2. Get log of a child request
 
 
 h2. Get log of a child request
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 import arvados.collection
 api = arvados.api()
 import arvados.collection
 api = arvados.api()

-container_request_uuid = "qr1hi-xvhdp-3d8f1zatx1xio36"
-container_request = arvados.api().container_requests().get(uuid=container_request_uuid).execute()
+container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
+container_request = api.container_requests().get(uuid=container_request_uuid).execute()

 collection = arvados.collection.CollectionReader(container_request["log_uuid"])
 for c in collection:
 collection = arvados.collection.CollectionReader(container_request["log_uuid"])
 for c in collection:

-    print(collection.open(c).read())
-</code></pre>
+    if isinstance(collection.find(c), arvados.arvfile.ArvadosFile):
+        print(collection.open(c).read())
+{% endcodeblock %}

 
 

-h2. Create a collection sharing link
+h2(#sharing_link). Create a collection sharing link

 
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 api = arvados.api()
 download="https://your.download.server"
 api = arvados.api()
 download="https://your.download.server"

-collection_uuid="your-4zz18-collectionuuid"
+collection_uuid="zzzzz-4zz18-zzzzzzzzzzzzzzz"

 token = api.api_client_authorizations().create(body={"api_client_authorization":{"scopes": [
     "GET /arvados/v1/collections/%s" % collection_uuid,
     "GET /arvados/v1/collections/%s/" % collection_uuid,
     "GET /arvados/v1/keep_services/accessible"]}}).execute()
 print("%s/c=%s/t=%s/_/" % (download, collection_uuid, token["api_token"]))
 token = api.api_client_authorizations().create(body={"api_client_authorization":{"scopes": [
     "GET /arvados/v1/collections/%s" % collection_uuid,
     "GET /arvados/v1/collections/%s/" % collection_uuid,
     "GET /arvados/v1/keep_services/accessible"]}}).execute()
 print("%s/c=%s/t=%s/_/" % (download, collection_uuid, token["api_token"]))

-</code></pre>
+{% endcodeblock %}

 
 h2. Combine two or more collections
 
 Note, if two collections have files of the same name, the contents will be concatenated in the resulting manifest.
 
 
 h2. Combine two or more collections
 
 Note, if two collections have files of the same name, the contents will be concatenated in the resulting manifest.
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 import arvados.collection
 api = arvados.api()
 import arvados.collection
 api = arvados.api()

-project_uuid = "qr1hi-tpzed-zzzzzzzzzzzzzzz"
-collection_uuids = ["qr1hi-4zz18-aaaaaaaaaaaaaaa", "qr1hi-4zz18-bbbbbbbbbbbbbbb"]
+project_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
+collection_uuids = ["zzzzz-4zz18-aaaaaaaaaaaaaaa", "zzzzz-4zz18-bbbbbbbbbbbbbbb"]

 combined_manifest = ""
 for u in collection_uuids:
     c = api.collections().get(uuid=u).execute()
     combined_manifest += c["manifest_text"]
 newcol = arvados.collection.Collection(combined_manifest)
 newcol.save_new(name="My combined collection", owner_uuid=project_uuid)
 combined_manifest = ""
 for u in collection_uuids:
     c = api.collections().get(uuid=u).execute()
     combined_manifest += c["manifest_text"]
 newcol = arvados.collection.Collection(combined_manifest)
 newcol.save_new(name="My combined collection", owner_uuid=project_uuid)

-</code></pre>
+{% endcodeblock %}
+
+h2. Upload a file into a new collection
+
+{% codeblock as python %}
+import arvados
+import arvados.collection
+
+project_uuid = "zzzzz-j7d0g-zzzzzzzzzzzzzzz"
+collection_name = "My collection"
+filename = "file1.txt"
+
+api = arvados.api()
+c = arvados.collection.Collection()
+with open(filename, "rb") as reader:
+    with c.open(filename, "wb") as writer:
+        content = reader.read(128*1024)
+        while content:
+            writer.write(content)
+            content = reader.read(128*1024)
+c.save_new(name=collection_name, owner_uuid=project_uuid)
+print("Saved %s to %s" % (collection_name, c.manifest_locator()))
+{% endcodeblock %}
+
+h2. Download a file from a collection
+
+{% codeblock as python %}
+import arvados
+import arvados.collection
+
+collection_uuid = "zzzzz-4zz18-zzzzzzzzzzzzzzz"
+filename = "file1.txt"
+
+api = arvados.api()
+c = arvados.collection.CollectionReader(collection_uuid)
+with c.open(filename, "rb") as reader:
+    with open(filename, "wb") as writer:
+        content = reader.read(128*1024)
+        while content:
+            writer.write(content)
+            content = reader.read(128*1024)
+print("Finished downloading %s" % filename)
+{% endcodeblock %}
+
+h2. Copy files from a collection to a new collection
+
+{% codeblock as python %}
+import arvados.collection
+
+source_collection = "zzzzz-4zz18-zzzzzzzzzzzzzzz"
+target_project = "zzzzz-j7d0g-zzzzzzzzzzzzzzz"
+target_name = "Files copied from source_collection"
+files_to_copy = ["folder1/sample1/sample1_R1.fastq",
+                 "folder1/sample2/sample2_R1.fastq"]
+
+source = arvados.collection.CollectionReader(source_collection)
+target = arvados.collection.Collection()
+
+for f in files_to_copy:
+    target.copy(f, "", source_collection=source)
+
+target.save_new(name=target_name, owner_uuid=target_project)
+print("Created collection %s" % target.manifest_locator())
+{% endcodeblock %}
+
+h2. Copy files from a collection to another collection
+
+{% codeblock as python %}
+import arvados.collection
+
+source_collection = "zzzzz-4zz18-zzzzzzzzzzzzzzz"
+target_collection = "zzzzz-4zz18-aaaaaaaaaaaaaaa"
+files_to_copy = ["folder1/sample1/sample1_R1.fastq",
+                 "folder1/sample2/sample2_R1.fastq"]
+
+source = arvados.collection.CollectionReader(source_collection)
+target = arvados.collection.Collection(target_collection)
+
+for f in files_to_copy:
+    target.copy(f, "", source_collection=source)
+
+target.save()
+{% endcodeblock %}
+
+h2. Delete a file from an existing collection
+
+{% codeblock as python %}
+import arvados
+
+c = arvados.collection.Collection("zzzzz-4zz18-zzzzzzzzzzzzzzz")
+c.remove("file2.txt")
+c.save()
+{% endcodeblock %}
+
+h2. Listing records with paging
+
+Use the @arvados.util.keyset_list_all@ helper method to iterate over all the records matching an optional filter.  This method handles paging internally and returns results incrementally using a Python iterator.  The first parameter of the method takes a @list@ method of an Arvados resource (@collections@, @container_requests@, etc).
+
+{% codeblock as python %}
+import arvados.util
+
+api = arvados.api()
+for c in arvados.util.keyset_list_all(api.collections().list, filters=[["name", "like", "%sample123%"]]):
+    print("got collection " + c["uuid"])
+{% endcodeblock %}