16470: Fixes reload() API to match the overridden function.
[arvados.git] / doc / sdk / python / cookbook.html.textile.liquid
1 ---
2 layout: default
3 navsection: sdk
4 navmenu: Python
5 title: Code cookbook
6 ...
7 {% comment %}
8 Copyright (C) The Arvados Authors. All rights reserved.
9
10 SPDX-License-Identifier: CC-BY-SA-3.0
11 {% endcomment %}
12
13 h2. Cancel a container request
14
15 {% codeblock as python %}
16 import arvados
17 arvados.api().container_requests().update(uuid=container_request_uuid, body={"priority": 0}).execute()
18 {% endcodeblock %}
19
20 h2. Cancel all container requests
21
22 {% codeblock as python %}
23 import arvados
24 api = arvados.api()
25 result = api.container_requests().list(filters=[["state", "=", "Committed"], ["priority", ">", "0"]]).execute()["items"]
26 for container_request in result:
27     api.container_requests().update(uuid=container_request["uuid"], body={"priority": 0}).execute()
28 {% endcodeblock %}
29
30 h2. List completed container requests
31
32 {% codeblock as python %}
33 import arvados
34 api = arvados.api()
35 result = api.container_requests().list(filters=[["name", "like", name], ["state", "=", "Final"]]).execute()["items"]
36 container_uuids = [cr["container_uuid"] for cr in result]
37 containers = api.containers().list(filters=[["uuid", "in", container_uuids]]).execute()["items"]
38 container_dict = {c["uuid"]: c for c in containers}
39
40 for container_request in result:
41     container = container_dict[container_request["container_uuid"]]
42     print("%s, %s, %s" % (container_request["uuid"], container_request["name"], "Success" if container["exit_code"] == 0 else "Failed"))
43 {% endcodeblock %}
44
45 h2. Get input of a CWL workflow
46
47 {% codeblock as python %}
48 import arvados
49 api = arvados.api()
50 container_request_uuid="qr1hi-xvhdp-zzzzzzzzzzzzzzz"
51 container_request = api.container_requests().get(uuid=container_request_uuid).execute()
52 print(container_request["mounts"]["/var/lib/cwl/cwl.input.json"])
53 {% endcodeblock %}
54
55 h2. Get output of a CWL workflow
56
57 {% codeblock as python %}
58 import arvados
59 import arvados.collection
60 api = arvados.api()
61 container_request_uuid="qr1hi-xvhdp-zzzzzzzzzzzzzzz"
62 container_request = api.container_requests().get(uuid=container_request_uuid).execute()
63 collection = arvados.collection.CollectionReader(container_request["output_uuid"])
64 print(collection.open("cwl.output.json").read())
65 {% endcodeblock %}
66
67 h2. Get state of a CWL workflow
68
69 {% codeblock as python %}
70 import arvados
71 def get_cr_state(cr_uuid):
72     api = arvados.api()
73     cr = api.container_requests().get(uuid=cr_uuid).execute()
74     if cr['container_uuid'] is None:
75         return cr['state']
76     c = api.containers().get(uuid=cr['container_uuid']).execute()
77     if cr['state'] == 'Final' and c['state'] != 'Complete':
78         return 'Cancelled'
79     elif c['state'] in ['Locked', 'Queued']:
80         if c['priority'] == 0:
81             return 'On hold'
82         else:
83             return 'Queued'
84     elif c['state'] == 'Complete' and c['exit_code'] != 0
85         return 'Failed'
86     elif c['state'] == 'Running':
87         if c['runtime_status'].get('error', None):
88             return 'Failing'
89         elif c['runtime_status'].get('warning', None):
90             return 'Warning'
91     return c['state']
92 container_request_uuid = 'qr1hi-xvhdp-zzzzzzzzzzzzzzz'
93 print(get_cr_state(container_request_uuid))
94 {% endcodeblock %}
95
96 h2. List input of child requests
97
98 {% codeblock as python %}
99 import arvados
100 api = arvados.api()
101 parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
102 namefilter = "bwa%"  # the "like" filter uses SQL pattern match syntax
103 container_request = api.container_requests().get(uuid=parent_request_uuid).execute()
104 parent_container_uuid = container_request["container_uuid"]
105 child_requests = api.container_requests().list(filters=[
106     ["requesting_container_uuid", "=", parent_container_uuid],
107     ["name", "like", namefilter]]).execute()
108 for c in child_requests["items"]:
109     print("%s" % c["name"])
110     for m in c["mounts"].values():
111         if "portable_data_hash" in m:
112             print("  %s" % m["portable_data_hash"])
113 {% endcodeblock %}
114
115 h2. List output of child requests
116
117 {% codeblock as python %}
118 import arvados
119 api = arvados.api()
120 parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
121 namefilter = "bwa%"  # the "like" filter uses SQL pattern match syntax
122 container_request = api.container_requests().get(uuid=parent_request_uuid).execute()
123 parent_container_uuid = container_request["container_uuid"]
124 child_requests = api.container_requests().list(filters=[
125     ["requesting_container_uuid", "=", parent_container_uuid],
126     ["name", "like", namefilter]]).execute()
127 output_uuids = [c["output_uuid"] for c in child_requests["items"]]
128 collections = api.collections().list(filters=[["uuid", "in", output_uuids]]).execute()
129 uuid_to_pdh = {c["uuid"]: c["portable_data_hash"] for c in collections["items"]}
130 for c in child_requests["items"]:
131     print("%s -> %s" % (c["name"], uuid_to_pdh[c["output_uuid"]]))
132 {% endcodeblock %}
133
134 h2. List failed child requests
135
136 {% codeblock as python %}
137 import arvados
138 api = arvados.api()
139 parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
140 container_request = api.container_requests().get(uuid=parent_request_uuid).execute()
141 parent_container_uuid = container_request["container_uuid"]
142 child_requests = api.container_requests().list(filters=[
143     ["requesting_container_uuid", "=", parent_container_uuid]], limit=1000).execute()
144 child_containers = {c["container_uuid"]: c for c in child_requests["items"]}
145 cancelled_child_containers = api.containers().list(filters=[
146     ["exit_code", "!=", "0"],
147     ["uuid", "in", child_containers.keys()]], limit=1000).execute()
148 for c in cancelled_child_containers["items"]:
149     print("%s (%s)" % (child_containers[c["uuid"]]["name"], child_containers[c["uuid"]]["uuid"]))
150 {% endcodeblock %}
151
152 h2. Get log of a child request
153
154 {% codeblock as python %}
155 import arvados
156 import arvados.collection
157 api = arvados.api()
158 container_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
159 container_request = api.container_requests().get(uuid=container_request_uuid).execute()
160 collection = arvados.collection.CollectionReader(container_request["log_uuid"])
161 for c in collection:
162     print(collection.open(c).read())
163 {% endcodeblock %}
164
165 h2(#sharing_link). Create a collection sharing link
166
167 {% codeblock as python %}
168 import arvados
169 api = arvados.api()
170 download="https://your.download.server"
171 collection_uuid="qr1hi-4zz18-zzzzzzzzzzzzzzz"
172 token = api.api_client_authorizations().create(body={"api_client_authorization":{"scopes": [
173     "GET /arvados/v1/collections/%s" % collection_uuid,
174     "GET /arvados/v1/collections/%s/" % collection_uuid,
175     "GET /arvados/v1/keep_services/accessible"]}}).execute()
176 print("%s/c=%s/t=%s/_/" % (download, collection_uuid, token["api_token"]))
177 {% endcodeblock %}
178
179 h2. Combine two or more collections
180
181 Note, if two collections have files of the same name, the contents will be concatenated in the resulting manifest.
182
183 {% codeblock as python %}
184 import arvados
185 import arvados.collection
186 api = arvados.api()
187 project_uuid = "qr1hi-tpzed-zzzzzzzzzzzzzzz"
188 collection_uuids = ["qr1hi-4zz18-aaaaaaaaaaaaaaa", "qr1hi-4zz18-bbbbbbbbbbbbbbb"]
189 combined_manifest = ""
190 for u in collection_uuids:
191     c = api.collections().get(uuid=u).execute()
192     combined_manifest += c["manifest_text"]
193 newcol = arvados.collection.Collection(combined_manifest)
194 newcol.save_new(name="My combined collection", owner_uuid=project_uuid)
195 {% endcodeblock %}
196
197 h2. Upload a file into a new collection
198
199 {% codeblock as python %}
200 import arvados
201 import arvados.collection
202
203 project_uuid = "qr1hi-j7d0g-zzzzzzzzzzzzzzz"
204 collection_name = "My collection"
205 filename = "file1.txt"
206
207 api = arvados.api()
208 c = arvados.collection.Collection()
209 with open(filename, "rb") as reader:
210     with c.open(filename, "wb") as writer:
211         content = reader.read(128*1024)
212         while content:
213             writer.write(content)
214             content = reader.read(128*1024)
215 c.save_new(name=collection_name, owner_uuid=project_uuid)
216 print("Saved %s to %s" % (collection_name, c.manifest_locator()))
217 {% endcodeblock %}
218
219 h2. Download a file from a collection
220
221 {% codeblock as python %}
222 import arvados
223 import arvados.collection
224
225 collection_uuid = "qr1hi-4zz18-zzzzzzzzzzzzzzz"
226 filename = "file1.txt"
227
228 api = arvados.api()
229 c = arvados.collection.CollectionReader(collection_uuid)
230 with c.open(filename, "rb") as reader:
231     with open(filename, "wb") as writer:
232         content = reader.read(128*1024)
233         while content:
234             writer.write(content)
235             content = reader.read(128*1024)
236 print("Finished downloading %s" % filename)
237 {% endcodeblock %}
238
239 h2. Copy files from a collection a new collection
240
241 {% codeblock as python %}
242 import arvados.collection
243
244 source_collection = "x1u39-4zz18-krzg64ufvehgitl"
245 target_project = "x1u39-j7d0g-67q94einb8ptznm"
246 target_name = "Files copied from source_collection"
247 files_to_copy = ["folder1/sample1/sample1_R1.fastq",
248                  "folder1/sample2/sample2_R1.fastq"]
249
250 source = arvados.collection.CollectionReader(source_collection)
251 target = arvados.collection.Collection()
252
253 for f in files_to_copy:
254     target.copy(f, "", source_collection=source)
255
256 target.save_new(name=target_name, owner_uuid=target_project)
257 print("Created collection %s" % target.manifest_locator())
258 {% endcodeblock %}