16625: Merge branch 'master' into 16625-add-azure-managed-image-support
[arvados.git] / doc / sdk / python / cookbook.html.textile.liquid
1 ---
2 layout: default
3 navsection: sdk
4 navmenu: Python
5 title: Code cookbook
6 ...
7 {% comment %}
8 Copyright (C) The Arvados Authors. All rights reserved.
9
10 SPDX-License-Identifier: CC-BY-SA-3.0
11 {% endcomment %}
12
13 h2. Cancel a container request
14
15 {% codeblock as python %}
16 import arvados
17 arvados.api().container_requests().update(uuid=container_request_uuid, body={"priority": 0}).execute()
18 {% endcodeblock %}
19
20 h2. Cancel all container requests
21
22 {% codeblock as python %}
23 import arvados
24 api = arvados.api()
25 result = api.container_requests().list(filters=[["state", "=", "Committed"], ["priority", ">", "0"]]).execute()["items"]
26 for container_request in result:
27     api.container_requests().update(uuid=container_request["uuid"], body={"priority": 0}).execute()
28 {% endcodeblock %}
29
30 h2. List completed container requests
31
32 {% codeblock as python %}
33 import arvados
34 api = arvados.api()
35 result = api.container_requests().list(filters=[["name", "like", name], ["state", "=", "Final"]]).execute()["items"]
36 container_uuids = [cr["container_uuid"] for cr in result]
37 containers = api.containers().list(filters=[["uuid", "in", container_uuids]]).execute()["items"]
38 container_dict = {c["uuid"]: c for c in containers}
39
40 for container_request in result:
41     container = container_dict[container_request["container_uuid"]]
42     print("%s, %s, %s" % (container_request["uuid"], container_request["name"], "Success" if container["exit_code"] == 0 else "Failed"))
43 {% endcodeblock %}
44
45 h2. Get input of a CWL workflow
46
47 {% codeblock as python %}
48 import arvados
49 api = arvados.api()
50 container_request_uuid="qr1hi-xvhdp-zzzzzzzzzzzzzzz"
51 container_request = api.container_requests().get(uuid=container_request_uuid).execute()
52 print(container_request["mounts"]["/var/lib/cwl/cwl.input.json"])
53 {% endcodeblock %}
54
55 h2. Get output of a CWL workflow
56
57 {% codeblock as python %}
58 import arvados
59 import arvados.collection
60 api = arvados.api()
61 container_request_uuid="qr1hi-xvhdp-zzzzzzzzzzzzzzz"
62 container_request = api.container_requests().get(uuid=container_request_uuid).execute()
63 collection = arvados.collection.CollectionReader(container_request["output_uuid"])
64 print(collection.open("cwl.output.json").read())
65 {% endcodeblock %}
66
67 h2. Get state of a CWL workflow
68
69 {% codeblock as python %}
70 import arvados
71 def get_cr_state(cr_uuid):
72     api = arvados.api()
73     cr = api.container_requests().get(uuid=cr_uuid).execute()
74     if cr['container_uuid'] is None:
75         return cr['state']
76     c = api.containers().get(uuid=cr['container_uuid']).execute()
77     if cr['state'] == 'Final' and c['state'] != 'Complete':
78         return 'Cancelled'
79     elif c['state'] in ['Locked', 'Queued']:
80         if c['priority'] == 0:
81             return 'On hold'
82         else:
83             return 'Queued'
84     elif c['state'] == 'Complete' and c['exit_code'] != 0:
85         return 'Failed'
86     elif c['state'] == 'Running':
87         if c['runtime_status'].get('error', None):
88             return 'Failing'
89         elif c['runtime_status'].get('warning', None):
90             return 'Warning'
91     return c['state']
92 container_request_uuid = 'qr1hi-xvhdp-zzzzzzzzzzzzzzz'
93 print(get_cr_state(container_request_uuid))
94 {% endcodeblock %}
95
96 h2. List input of child requests
97
98 {% codeblock as python %}
99 import arvados
100 api = arvados.api()
101 parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
102 namefilter = "bwa%"  # the "like" filter uses SQL pattern match syntax
103 container_request = api.container_requests().get(uuid=parent_request_uuid).execute()
104 parent_container_uuid = container_request["container_uuid"]
105 child_requests = api.container_requests().list(filters=[
106     ["requesting_container_uuid", "=", parent_container_uuid],
107     ["name", "like", namefilter]]).execute()
108 for c in child_requests["items"]:
109     print("%s" % c["name"])
110     for m in c["mounts"].values():
111         if "portable_data_hash" in m:
112             print("  %s" % m["portable_data_hash"])
113 {% endcodeblock %}
114
115 h2. List output of child requests
116
117 {% codeblock as python %}
118 import arvados
119 api = arvados.api()
120 parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
121 namefilter = "bwa%"  # the "like" filter uses SQL pattern match syntax
122 container_request = api.container_requests().get(uuid=parent_request_uuid).execute()
123 parent_container_uuid = container_request["container_uuid"]
124 child_requests = api.container_requests().list(filters=[
125     ["requesting_container_uuid", "=", parent_container_uuid],
126     ["name", "like", namefilter]]).execute()
127 output_uuids = [c["output_uuid"] for c in child_requests["items"]]
128 collections = api.collections().list(filters=[["uuid", "in", output_uuids]]).execute()
129 uuid_to_pdh = {c["uuid"]: c["portable_data_hash"] for c in collections["items"]}
130 for c in child_requests["items"]:
131     print("%s -> %s" % (c["name"], uuid_to_pdh[c["output_uuid"]]))
132 {% endcodeblock %}
133
134 h2. List failed child requests
135
136 {% codeblock as python %}
137 import arvados
138 api = arvados.api()
139 parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
140 container_request = api.container_requests().get(uuid=parent_request_uuid).execute()
141 parent_container_uuid = container_request["container_uuid"]
142 child_requests = api.container_requests().list(filters=[
143     ["requesting_container_uuid", "=", parent_container_uuid]], limit=1000).execute()
144 child_containers = {c["container_uuid"]: c for c in child_requests["items"]}
145 cancelled_child_containers = api.containers().list(filters=[
146     ["exit_code", "!=", "0"],
147     ["uuid", "in", list(child_containers.keys())]], limit=1000).execute()
148 for c in cancelled_child_containers["items"]:
149     print("%s (%s)" % (child_containers[c["uuid"]]["name"], child_containers[c["uuid"]]["uuid"]))
150 {% endcodeblock %}
151
152 h2. Get log of a child request
153
154 {% codeblock as python %}
155 import arvados
156 import arvados.collection
157 api = arvados.api()
158 container_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
159 container_request = api.container_requests().get(uuid=container_request_uuid).execute()
160 collection = arvados.collection.CollectionReader(container_request["log_uuid"])
161 for c in collection:
162     if isinstance(collection.find(c), arvados.arvfile.ArvadosFile):
163         print(collection.open(c).read())
164 {% endcodeblock %}
165
166 h2(#sharing_link). Create a collection sharing link
167
168 {% codeblock as python %}
169 import arvados
170 api = arvados.api()
171 download="https://your.download.server"
172 collection_uuid="qr1hi-4zz18-zzzzzzzzzzzzzzz"
173 token = api.api_client_authorizations().create(body={"api_client_authorization":{"scopes": [
174     "GET /arvados/v1/collections/%s" % collection_uuid,
175     "GET /arvados/v1/collections/%s/" % collection_uuid,
176     "GET /arvados/v1/keep_services/accessible"]}}).execute()
177 print("%s/c=%s/t=%s/_/" % (download, collection_uuid, token["api_token"]))
178 {% endcodeblock %}
179
180 h2. Combine two or more collections
181
182 Note, if two collections have files of the same name, the contents will be concatenated in the resulting manifest.
183
184 {% codeblock as python %}
185 import arvados
186 import arvados.collection
187 api = arvados.api()
188 project_uuid = "qr1hi-tpzed-zzzzzzzzzzzzzzz"
189 collection_uuids = ["qr1hi-4zz18-aaaaaaaaaaaaaaa", "qr1hi-4zz18-bbbbbbbbbbbbbbb"]
190 combined_manifest = ""
191 for u in collection_uuids:
192     c = api.collections().get(uuid=u).execute()
193     combined_manifest += c["manifest_text"]
194 newcol = arvados.collection.Collection(combined_manifest)
195 newcol.save_new(name="My combined collection", owner_uuid=project_uuid)
196 {% endcodeblock %}
197
198 h2. Upload a file into a new collection
199
200 {% codeblock as python %}
201 import arvados
202 import arvados.collection
203
204 project_uuid = "qr1hi-j7d0g-zzzzzzzzzzzzzzz"
205 collection_name = "My collection"
206 filename = "file1.txt"
207
208 api = arvados.api()
209 c = arvados.collection.Collection()
210 with open(filename, "rb") as reader:
211     with c.open(filename, "wb") as writer:
212         content = reader.read(128*1024)
213         while content:
214             writer.write(content)
215             content = reader.read(128*1024)
216 c.save_new(name=collection_name, owner_uuid=project_uuid)
217 print("Saved %s to %s" % (collection_name, c.manifest_locator()))
218 {% endcodeblock %}
219
220 h2. Download a file from a collection
221
222 {% codeblock as python %}
223 import arvados
224 import arvados.collection
225
226 collection_uuid = "qr1hi-4zz18-zzzzzzzzzzzzzzz"
227 filename = "file1.txt"
228
229 api = arvados.api()
230 c = arvados.collection.CollectionReader(collection_uuid)
231 with c.open(filename, "rb") as reader:
232     with open(filename, "wb") as writer:
233         content = reader.read(128*1024)
234         while content:
235             writer.write(content)
236             content = reader.read(128*1024)
237 print("Finished downloading %s" % filename)
238 {% endcodeblock %}
239
240 h2. Copy files from a collection a new collection
241
242 {% codeblock as python %}
243 import arvados.collection
244
245 source_collection = "x1u39-4zz18-krzg64ufvehgitl"
246 target_project = "x1u39-j7d0g-67q94einb8ptznm"
247 target_name = "Files copied from source_collection"
248 files_to_copy = ["folder1/sample1/sample1_R1.fastq",
249                  "folder1/sample2/sample2_R1.fastq"]
250
251 source = arvados.collection.CollectionReader(source_collection)
252 target = arvados.collection.Collection()
253
254 for f in files_to_copy:
255     target.copy(f, "", source_collection=source)
256
257 target.save_new(name=target_name, owner_uuid=target_project)
258 print("Created collection %s" % target.manifest_locator())
259 {% endcodeblock %}