16306: Merge branch 'master'

[arvados.git] / doc / sdk / python / cookbook.html.textile.liquid

diff --git a/doc/sdk/python/cookbook.html.textile.liquid b/doc/sdk/python/cookbook.html.textile.liquid
index fa059f7028cd25db55e693d26439b5ce91697ecf..75c51ee5a8126c57b9b23bc95d9cffdcf7fc027c 100644 (file)
--- a/doc/sdk/python/cookbook.html.textile.liquid
+++ b/doc/sdk/python/cookbook.html.textile.liquid
@@ -12,22 +12,25 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 h2. Cancel a container request
 
 
 h2. Cancel a container request
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 arvados.api().container_requests().update(uuid=container_request_uuid, body={"priority": 0}).execute()
 arvados.api().container_requests().update(uuid=container_request_uuid, body={"priority": 0}).execute()

-</code></pre>
+{% endcodeblock %}

 
 h2. Cancel all container requests
 
 
 h2. Cancel all container requests
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 api = arvados.api()
 result = api.container_requests().list(filters=[["state", "=", "Committed"], ["priority", ">", "0"]]).execute()["items"]
 for container_request in result:
     api.container_requests().update(uuid=container_request["uuid"], body={"priority": 0}).execute()
 api = arvados.api()
 result = api.container_requests().list(filters=[["state", "=", "Committed"], ["priority", ">", "0"]]).execute()["items"]
 for container_request in result:
     api.container_requests().update(uuid=container_request["uuid"], body={"priority": 0}).execute()

-</code></pre>
+{% endcodeblock %}

 
 h2. List completed container requests
 
 
 h2. List completed container requests
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 api = arvados.api()
 result = api.container_requests().list(filters=[["name", "like", name], ["state", "=", "Final"]]).execute()["items"]
 container_uuids = [cr["container_uuid"] for cr in result]
 api = arvados.api()
 result = api.container_requests().list(filters=[["name", "like", name], ["state", "=", "Final"]]).execute()["items"]
 container_uuids = [cr["container_uuid"] for cr in result]


@@ -37,35 +40,69 @@ container_dict = {c["uuid"]: c for c in containers}
 for container_request in result:
     container = container_dict[container_request["container_uuid"]]
     print("%s, %s, %s" % (container_request["uuid"], container_request["name"], "Success" if container["exit_code"] == 0 else "Failed"))
 for container_request in result:
     container = container_dict[container_request["container_uuid"]]
     print("%s, %s, %s" % (container_request["uuid"], container_request["name"], "Success" if container["exit_code"] == 0 else "Failed"))

-</code></pre>
+{% endcodeblock %}

 
 h2. Get input of a CWL workflow
 
 
 h2. Get input of a CWL workflow
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 api = arvados.api()
 api = arvados.api()

-container_request = arvados.api().container_requests().get(uuid=container_request_uuid).execute()
+container_request_uuid="qr1hi-xvhdp-zzzzzzzzzzzzzzz"
+container_request = api.container_requests().get(uuid=container_request_uuid).execute()

 print(container_request["mounts"]["/var/lib/cwl/cwl.input.json"])
 print(container_request["mounts"]["/var/lib/cwl/cwl.input.json"])

-</code></pre>
+{% endcodeblock %}

 
 h2. Get output of a CWL workflow
 
 
 h2. Get output of a CWL workflow
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 import arvados.collection
 api = arvados.api()
 import arvados.collection
 api = arvados.api()

-container_request = arvados.api().container_requests().get(uuid=container_request_uuid).execute()
+container_request_uuid="qr1hi-xvhdp-zzzzzzzzzzzzzzz"
+container_request = api.container_requests().get(uuid=container_request_uuid).execute()

 collection = arvados.collection.CollectionReader(container_request["output_uuid"])
 print(collection.open("cwl.output.json").read())
 collection = arvados.collection.CollectionReader(container_request["output_uuid"])
 print(collection.open("cwl.output.json").read())

-</code></pre>
+{% endcodeblock %}
+
+h2. Get state of a CWL workflow
+
+{% codeblock as python %}
+import arvados
+def get_cr_state(cr_uuid):
+    api = arvados.api()
+    cr = api.container_requests().get(uuid=cr_uuid).execute()
+    if cr['container_uuid'] is None:
+        return cr['state']
+    c = api.containers().get(uuid=cr['container_uuid']).execute()
+    if cr['state'] == 'Final' and c['state'] != 'Complete':
+        return 'Cancelled'
+    elif c['state'] in ['Locked', 'Queued']:
+        if c['priority'] == 0:
+            return 'On hold'
+        else:
+            return 'Queued'
+    elif c['state'] == 'Complete' and c['exit_code'] != 0:
+        return 'Failed'
+    elif c['state'] == 'Running':
+        if c['runtime_status'].get('error', None):
+            return 'Failing'
+        elif c['runtime_status'].get('warning', None):
+            return 'Warning'
+    return c['state']
+container_request_uuid = 'qr1hi-xvhdp-zzzzzzzzzzzzzzz'
+print(get_cr_state(container_request_uuid))
+{% endcodeblock %}

 
 h2. List input of child requests
 
 
 h2. List input of child requests
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 api = arvados.api()
 parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
 api = arvados.api()
 parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"

-namefilter = "bwa%"
-container_request = arvados.api().container_requests().get(uuid=parent_request_uuid).execute()
+namefilter = "bwa%"  # the "like" filter uses SQL pattern match syntax
+container_request = api.container_requests().get(uuid=parent_request_uuid).execute()

 parent_container_uuid = container_request["container_uuid"]
 parent_container_uuid = container_request["container_uuid"]

-child_requests = arvados.api().container_requests().list(filters=[
+child_requests = api.container_requests().list(filters=[

     ["requesting_container_uuid", "=", parent_container_uuid],
     ["name", "like", namefilter]]).execute()
 for c in child_requests["items"]:
     ["requesting_container_uuid", "=", parent_container_uuid],
     ["name", "like", namefilter]]).execute()
 for c in child_requests["items"]:


@@ -73,73 +110,79 @@ for c in child_requests["items"]:
     for m in c["mounts"].values():
         if "portable_data_hash" in m:
             print("  %s" % m["portable_data_hash"])
     for m in c["mounts"].values():
         if "portable_data_hash" in m:
             print("  %s" % m["portable_data_hash"])

-</code></pre>
+{% endcodeblock %}

 
 h2. List output of child requests
 
 
 h2. List output of child requests
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 api = arvados.api()
 api = arvados.api()

-parent_request_uuid = "qr1hi-xvhdp-61y7cjymnhyp7yu"
-namefilter = "bwa%"
-container_request = arvados.api().container_requests().get(uuid=parent_request_uuid).execute()
+parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
+namefilter = "bwa%"  # the "like" filter uses SQL pattern match syntax
+container_request = api.container_requests().get(uuid=parent_request_uuid).execute()

 parent_container_uuid = container_request["container_uuid"]
 parent_container_uuid = container_request["container_uuid"]

-child_requests = arvados.api().container_requests().list(filters=[
+child_requests = api.container_requests().list(filters=[

     ["requesting_container_uuid", "=", parent_container_uuid],
     ["name", "like", namefilter]]).execute()
 output_uuids = [c["output_uuid"] for c in child_requests["items"]]
     ["requesting_container_uuid", "=", parent_container_uuid],
     ["name", "like", namefilter]]).execute()
 output_uuids = [c["output_uuid"] for c in child_requests["items"]]

-collections = arvados.api().collections().list(filters=[["uuid", "in", output_uuids]]).execute()
+collections = api.collections().list(filters=[["uuid", "in", output_uuids]]).execute()

 uuid_to_pdh = {c["uuid"]: c["portable_data_hash"] for c in collections["items"]}
 for c in child_requests["items"]:
     print("%s -> %s" % (c["name"], uuid_to_pdh[c["output_uuid"]]))
 uuid_to_pdh = {c["uuid"]: c["portable_data_hash"] for c in collections["items"]}
 for c in child_requests["items"]:
     print("%s -> %s" % (c["name"], uuid_to_pdh[c["output_uuid"]]))

-</code></pre>
+{% endcodeblock %}

 
 h2. List failed child requests
 
 
 h2. List failed child requests
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 api = arvados.api()
 api = arvados.api()

-parent_request_uuid = "qr1hi-xvhdp-3d8f1zatx1xio36"
-container_request = arvados.api().container_requests().get(uuid=parent_request_uuid).execute()
+parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
+container_request = api.container_requests().get(uuid=parent_request_uuid).execute()

 parent_container_uuid = container_request["container_uuid"]
 parent_container_uuid = container_request["container_uuid"]

-child_requests = arvados.api().container_requests().list(filters=[
+child_requests = api.container_requests().list(filters=[

     ["requesting_container_uuid", "=", parent_container_uuid]], limit=1000).execute()
 child_containers = {c["container_uuid"]: c for c in child_requests["items"]}
     ["requesting_container_uuid", "=", parent_container_uuid]], limit=1000).execute()
 child_containers = {c["container_uuid"]: c for c in child_requests["items"]}

-cancelled_child_containers = arvados.api().containers().list(filters=[
+cancelled_child_containers = api.containers().list(filters=[

     ["exit_code", "!=", "0"],
     ["exit_code", "!=", "0"],

-    ["uuid", "in", child_containers.keys()]], limit=1000).execute()
+    ["uuid", "in", list(child_containers.keys())]], limit=1000).execute()

 for c in cancelled_child_containers["items"]:
     print("%s (%s)" % (child_containers[c["uuid"]]["name"], child_containers[c["uuid"]]["uuid"]))
 for c in cancelled_child_containers["items"]:
     print("%s (%s)" % (child_containers[c["uuid"]]["name"], child_containers[c["uuid"]]["uuid"]))

-</code></pre>
+{% endcodeblock %}

 
 h2. Get log of a child request
 
 
 h2. Get log of a child request
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 import arvados.collection
 api = arvados.api()
 import arvados.collection
 api = arvados.api()

-container_request_uuid = "qr1hi-xvhdp-3d8f1zatx1xio36"
-container_request = arvados.api().container_requests().get(uuid=container_request_uuid).execute()
+container_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
+container_request = api.container_requests().get(uuid=container_request_uuid).execute()

 collection = arvados.collection.CollectionReader(container_request["log_uuid"])
 for c in collection:
 collection = arvados.collection.CollectionReader(container_request["log_uuid"])
 for c in collection:

-    print(collection.open(c).read())
-</code></pre>
+    if isinstance(collection.find(c), arvados.arvfile.ArvadosFile):
+        print(collection.open(c).read())
+{% endcodeblock %}

 
 

-h2. Create a collection sharing link
+h2(#sharing_link). Create a collection sharing link

 
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 api = arvados.api()
 download="https://your.download.server"
 api = arvados.api()
 download="https://your.download.server"

-collection_uuid="your-4zz18-collectionuuid"
+collection_uuid="qr1hi-4zz18-zzzzzzzzzzzzzzz"

 token = api.api_client_authorizations().create(body={"api_client_authorization":{"scopes": [
     "GET /arvados/v1/collections/%s" % collection_uuid,
     "GET /arvados/v1/collections/%s/" % collection_uuid,
     "GET /arvados/v1/keep_services/accessible"]}}).execute()
 print("%s/c=%s/t=%s/_/" % (download, collection_uuid, token["api_token"]))
 token = api.api_client_authorizations().create(body={"api_client_authorization":{"scopes": [
     "GET /arvados/v1/collections/%s" % collection_uuid,
     "GET /arvados/v1/collections/%s/" % collection_uuid,
     "GET /arvados/v1/keep_services/accessible"]}}).execute()
 print("%s/c=%s/t=%s/_/" % (download, collection_uuid, token["api_token"]))

-</code></pre>
+{% endcodeblock %}

 
 h2. Combine two or more collections
 
 Note, if two collections have files of the same name, the contents will be concatenated in the resulting manifest.
 
 
 h2. Combine two or more collections
 
 Note, if two collections have files of the same name, the contents will be concatenated in the resulting manifest.
 

-<pre><code>import arvados
+{% codeblock as python %}
+import arvados

 import arvados.collection
 api = arvados.api()
 project_uuid = "qr1hi-tpzed-zzzzzzzzzzzzzzz"
 import arvados.collection
 api = arvados.api()
 project_uuid = "qr1hi-tpzed-zzzzzzzzzzzzzzz"


@@ -150,4 +193,67 @@ for u in collection_uuids:
     combined_manifest += c["manifest_text"]
 newcol = arvados.collection.Collection(combined_manifest)
 newcol.save_new(name="My combined collection", owner_uuid=project_uuid)
     combined_manifest += c["manifest_text"]
 newcol = arvados.collection.Collection(combined_manifest)
 newcol.save_new(name="My combined collection", owner_uuid=project_uuid)

-</code></pre>
+{% endcodeblock %}
+
+h2. Upload a file into a new collection
+
+{% codeblock as python %}
+import arvados
+import arvados.collection
+
+project_uuid = "qr1hi-j7d0g-zzzzzzzzzzzzzzz"
+collection_name = "My collection"
+filename = "file1.txt"
+
+api = arvados.api()
+c = arvados.collection.Collection()
+with open(filename, "rb") as reader:
+    with c.open(filename, "wb") as writer:
+        content = reader.read(128*1024)
+        while content:
+            writer.write(content)
+            content = reader.read(128*1024)
+c.save_new(name=collection_name, owner_uuid=project_uuid)
+print("Saved %s to %s" % (collection_name, c.manifest_locator()))
+{% endcodeblock %}
+
+h2. Download a file from a collection
+
+{% codeblock as python %}
+import arvados
+import arvados.collection
+
+collection_uuid = "qr1hi-4zz18-zzzzzzzzzzzzzzz"
+filename = "file1.txt"
+
+api = arvados.api()
+c = arvados.collection.CollectionReader(collection_uuid)
+with c.open(filename, "rb") as reader:
+    with open(filename, "wb") as writer:
+        content = reader.read(128*1024)
+        while content:
+            writer.write(content)
+            content = reader.read(128*1024)
+print("Finished downloading %s" % filename)
+{% endcodeblock %}
+
+h2. Copy files from a collection a new collection
+
+{% codeblock as python %}
+import arvados.collection
+
+source_collection = "x1u39-4zz18-krzg64ufvehgitl"
+target_project = "x1u39-j7d0g-67q94einb8ptznm"
+target_name = "Files copied from source_collection"
+files_to_copy = ["folder1/sample1/sample1_R1.fastq",
+                 "folder1/sample2/sample2_R1.fastq"]
+
+source = arvados.collection.CollectionReader(source_collection)
+target = arvados.collection.Collection()
+
+for f in files_to_copy:
+    target.copy(f, "", source_collection=source)
+
+target.save_new(name=target_name, owner_uuid=target_project)
+print("Created collection %s" % target.manifest_locator())
+{% endcodeblock %}