<div class="releasenotes">
</notextile>
-h2(#main). development main (as of 2023-01-16)
+h2(#main). development main (as of 2023-01-27)
"previous: Upgrading to 2.5.0":#v2_5_0
+h3. Default limit for cloud VM instances
+
+There is a new configuration entry @CloudVMs.MaxInstances@ (default 64) that limits the number of VMs the cloud dispatcher will run at a time. This may need to be adjusted to suit your anticipated workload.
+
+Using the obsolete configuration entry @MaxCloudVMs@, which was previously accepted in config files but not obeyed, will now result in a deprecation warning.
+
h3. Slow migration on upgrade
This upgrade includes a database schema update (changing an integer column in each table from 32-bit to 64-bit) that may be slow on a large installation. Expect the arvados-api-server package upgrade to take longer than usual.
--- /dev/null
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.43.0 (0)
+ -->
+<!-- Title: %3 Pages: 1 -->
+<svg width="748pt" height="818pt"
+ viewBox="0.00 0.00 747.50 818.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 814)">
+<title>%3</title>
+<polygon fill="white" stroke="transparent" points="-4,4 -4,-814 743.5,-814 743.5,4 -4,4"/>
+<!-- invisiblestart -->
+<g id="node1" class="node">
+<title>invisiblestart</title>
+<ellipse fill="none" stroke="white" cx="88" cy="-792" rx="27" ry="18"/>
+</g>
+<!-- uncommitted -->
+<g id="node2" class="node">
+<title>uncommitted</title>
+<polygon fill="lightgrey" stroke="black" points="176,-723 0,-723 0,-685 176,-685 176,-723"/>
+<text text-anchor="start" x="8" y="-707.8" font-family="Times,serif" font-size="14.00">container request:</text>
+<text text-anchor="start" x="8" y="-692.8" font-family="Times,serif" font-size="14.00">   state=Uncommitted</text>
+</g>
+<!-- invisiblestart->uncommitted -->
+<g id="edge1" class="edge">
+<title>invisiblestart->uncommitted</title>
+<path fill="none" stroke="navy" d="M88,-773.6C88,-762.06 88,-746.65 88,-733.36"/>
+<polygon fill="navy" stroke="navy" points="91.5,-733.27 88,-723.27 84.5,-733.27 91.5,-733.27"/>
+<text text-anchor="start" x="88" y="-744.8" font-family="Times,serif" font-size="14.00" fill="navy">   user creates container request</text>
+</g>
+<!-- committed -->
+<g id="node3" class="node">
+<title>committed</title>
+<polygon fill="white" stroke="black" points="167,-604 9,-604 9,-551 167,-551 167,-604"/>
+<text text-anchor="start" x="17" y="-588.8" font-family="Times,serif" font-size="14.00">container request:</text>
+<text text-anchor="start" x="17" y="-573.8" font-family="Times,serif" font-size="14.00">   state=Committed</text>
+<text text-anchor="start" x="17" y="-558.8" font-family="Times,serif" font-size="14.00">   priority>0</text>
+</g>
+<!-- uncommitted->committed -->
+<g id="edge2" class="edge">
+<title>uncommitted->committed</title>
+<path fill="none" stroke="navy" d="M88,-684.9C88,-666.53 88,-637.61 88,-614.55"/>
+<polygon fill="navy" stroke="navy" points="91.5,-614.43 88,-604.43 84.5,-614.43 91.5,-614.43"/>
+<text text-anchor="start" x="88" y="-655.8" font-family="Times,serif" font-size="14.00" fill="navy">   user updates to</text>
+<text text-anchor="start" x="88" y="-640.8" font-family="Times,serif" font-size="14.00" fill="navy">      state=Committed, priority>0</text>
+</g>
+<!-- reused -->
+<g id="node4" class="node">
+<title>reused</title>
+<polygon fill="lightblue" stroke="black" points="739.5,-619 530.5,-619 530.5,-536 739.5,-536 739.5,-619"/>
+<text text-anchor="start" x="538.5" y="-603.8" font-family="Times,serif" font-size="14.00">container request:</text>
+<text text-anchor="start" x="538.5" y="-588.8" font-family="Times,serif" font-size="14.00">   state=Final</text>
+<text text-anchor="start" x="538.5" y="-573.8" font-family="Times,serif" font-size="14.00">container:</text>
+<text text-anchor="start" x="538.5" y="-558.8" font-family="Times,serif" font-size="14.00">   state=Complete</text>
+<text text-anchor="start" x="538.5" y="-543.8" font-family="Times,serif" font-size="14.00">(reused existing container)</text>
+</g>
+<!-- committed->reused -->
+<g id="edge6" class="edge">
+<title>committed->reused</title>
+<path fill="none" stroke="black" d="M167.25,-577.5C260.04,-577.5 414.48,-577.5 520.34,-577.5"/>
+<polygon fill="black" stroke="black" points="520.43,-581 530.43,-577.5 520.43,-574 520.43,-581"/>
+<text text-anchor="middle" x="348.75" y="-584.3" font-family="Times,serif" font-size="14.00">Arvados selects an existing container</text>
+</g>
+<!-- queued -->
+<g id="node5" class="node">
+<title>queued</title>
+<polygon fill="white" stroke="black" points="167,-485 9,-485 9,-402 167,-402 167,-485"/>
+<text text-anchor="start" x="17" y="-469.8" font-family="Times,serif" font-size="14.00">container request:</text>
+<text text-anchor="start" x="17" y="-454.8" font-family="Times,serif" font-size="14.00">   state=Committed</text>
+<text text-anchor="start" x="17" y="-439.8" font-family="Times,serif" font-size="14.00">   priority>0</text>
+<text text-anchor="start" x="17" y="-424.8" font-family="Times,serif" font-size="14.00">container:</text>
+<text text-anchor="start" x="17" y="-409.8" font-family="Times,serif" font-size="14.00">   state=Queued</text>
+</g>
+<!-- committed->queued -->
+<g id="edge3" class="edge">
+<title>committed->queued</title>
+<path fill="none" stroke="black" d="M88,-550.74C88,-534.9 88,-514.07 88,-495.05"/>
+<polygon fill="black" stroke="black" points="91.5,-495.01 88,-485.01 84.5,-495.01 91.5,-495.01"/>
+<text text-anchor="start" x="88" y="-506.8" font-family="Times,serif" font-size="14.00">   Arvados creates a new container</text>
+</g>
+<!-- latecancelled -->
+<g id="node7" class="node">
+<title>latecancelled</title>
+<polygon fill="lightblue" stroke="black" points="709,-343.5 561,-343.5 561,-275.5 709,-275.5 709,-343.5"/>
+<text text-anchor="start" x="569" y="-328.3" font-family="Times,serif" font-size="14.00">container request:</text>
+<text text-anchor="start" x="569" y="-313.3" font-family="Times,serif" font-size="14.00">   state=Final</text>
+<text text-anchor="start" x="569" y="-298.3" font-family="Times,serif" font-size="14.00">container:</text>
+<text text-anchor="start" x="569" y="-283.3" font-family="Times,serif" font-size="14.00">   state=Cancelled</text>
+</g>
+<!-- reused->latecancelled -->
+<!-- locked -->
+<g id="node6" class="node">
+<title>locked</title>
+<polygon fill="white" stroke="black" points="167,-351 9,-351 9,-268 167,-268 167,-351"/>
+<text text-anchor="start" x="17" y="-335.8" font-family="Times,serif" font-size="14.00">container request:</text>
+<text text-anchor="start" x="17" y="-320.8" font-family="Times,serif" font-size="14.00">   state=Committed</text>
+<text text-anchor="start" x="17" y="-305.8" font-family="Times,serif" font-size="14.00">   priority>0</text>
+<text text-anchor="start" x="17" y="-290.8" font-family="Times,serif" font-size="14.00">container:</text>
+<text text-anchor="start" x="17" y="-275.8" font-family="Times,serif" font-size="14.00">   state=Locked</text>
+</g>
+<!-- queued->locked -->
+<g id="edge4" class="edge">
+<title>queued->locked</title>
+<path fill="none" stroke="black" d="M88,-401.82C88,-389.02 88,-374.73 88,-361.32"/>
+<polygon fill="black" stroke="black" points="91.5,-361.27 88,-351.27 84.5,-361.27 91.5,-361.27"/>
+<text text-anchor="start" x="88" y="-372.8" font-family="Times,serif" font-size="14.00">   Arvados is ready to dispatch the container</text>
+</g>
+<!-- queued->latecancelled -->
+<g id="edge7" class="edge">
+<title>queued->latecancelled</title>
+<path fill="none" stroke="navy" d="M167.18,-436.67C233.02,-429.8 328.19,-415.07 406,-384 417.33,-379.47 417.89,-374.06 429,-369 467.51,-351.46 512.51,-337.95 550.6,-328.36"/>
+<polygon fill="navy" stroke="navy" points="551.77,-331.68 560.64,-325.88 550.09,-324.88 551.77,-331.68"/>
+<text text-anchor="middle" x="525" y="-372.8" font-family="Times,serif" font-size="14.00" fill="navy">user updates to priority=0</text>
+</g>
+<!-- locked->latecancelled -->
+<g id="edge8" class="edge">
+<title>locked->latecancelled</title>
+<path fill="none" stroke="navy" d="M167.25,-309.5C269.4,-309.5 446.28,-309.5 550.79,-309.5"/>
+<polygon fill="navy" stroke="navy" points="550.98,-313 560.98,-309.5 550.98,-306 550.98,-313"/>
+<text text-anchor="middle" x="364" y="-316.3" font-family="Times,serif" font-size="14.00" fill="navy">user updates to priority=0</text>
+</g>
+<!-- running -->
+<g id="node8" class="node">
+<title>running</title>
+<polygon fill="white" stroke="black" points="167,-217 9,-217 9,-134 167,-134 167,-217"/>
+<text text-anchor="start" x="17" y="-201.8" font-family="Times,serif" font-size="14.00">container request:</text>
+<text text-anchor="start" x="17" y="-186.8" font-family="Times,serif" font-size="14.00">   state=Committed</text>
+<text text-anchor="start" x="17" y="-171.8" font-family="Times,serif" font-size="14.00">   priority>0</text>
+<text text-anchor="start" x="17" y="-156.8" font-family="Times,serif" font-size="14.00">container:</text>
+<text text-anchor="start" x="17" y="-141.8" font-family="Times,serif" font-size="14.00">   state=Running</text>
+</g>
+<!-- locked->running -->
+<g id="edge5" class="edge">
+<title>locked->running</title>
+<path fill="none" stroke="black" d="M88,-267.82C88,-255.02 88,-240.73 88,-227.32"/>
+<polygon fill="black" stroke="black" points="91.5,-227.27 88,-217.27 84.5,-227.27 91.5,-227.27"/>
+<text text-anchor="start" x="88" y="-238.8" font-family="Times,serif" font-size="14.00">   Arvados starts the container process</text>
+</g>
+<!-- containerfailed -->
+<g id="node9" class="node">
+<title>containerfailed</title>
+<polygon fill="lightblue" stroke="black" points="709,-217 561,-217 561,-134 709,-134 709,-217"/>
+<text text-anchor="start" x="569" y="-201.8" font-family="Times,serif" font-size="14.00">container request:</text>
+<text text-anchor="start" x="569" y="-186.8" font-family="Times,serif" font-size="14.00">   state=Final</text>
+<text text-anchor="start" x="569" y="-171.8" font-family="Times,serif" font-size="14.00">container:</text>
+<text text-anchor="start" x="569" y="-156.8" font-family="Times,serif" font-size="14.00">   state=Complete</text>
+<text text-anchor="start" x="569" y="-141.8" font-family="Times,serif" font-size="14.00">   exit_code≠0</text>
+</g>
+<!-- latecancelled->containerfailed -->
+<!-- running->latecancelled -->
+<g id="edge9" class="edge">
+<title>running->latecancelled</title>
+<path fill="none" stroke="navy" d="M167.03,-191.54C223.51,-202.63 301.19,-218.59 369,-235 430.25,-249.82 498.77,-268.81 550.93,-283.78"/>
+<polygon fill="navy" stroke="navy" points="550.23,-287.22 560.81,-286.62 552.16,-280.49 550.23,-287.22"/>
+<text text-anchor="middle" x="523" y="-238.8" font-family="Times,serif" font-size="14.00" fill="navy">user updates to priority=0</text>
+</g>
+<!-- running->containerfailed -->
+<g id="edge10" class="edge">
+<title>running->containerfailed</title>
+<path fill="none" stroke="black" d="M167.25,-175.5C269.4,-175.5 446.28,-175.5 550.79,-175.5"/>
+<polygon fill="black" stroke="black" points="550.98,-179 560.98,-175.5 550.98,-172 550.98,-179"/>
+<text text-anchor="middle" x="364" y="-182.3" font-family="Times,serif" font-size="14.00">container process fails</text>
+</g>
+<!-- containerfinished -->
+<g id="node10" class="node">
+<title>containerfinished</title>
+<polygon fill="lightblue" stroke="black" points="162,-83 14,-83 14,0 162,0 162,-83"/>
+<text text-anchor="start" x="22" y="-67.8" font-family="Times,serif" font-size="14.00">container request:</text>
+<text text-anchor="start" x="22" y="-52.8" font-family="Times,serif" font-size="14.00">   state=Final</text>
+<text text-anchor="start" x="22" y="-37.8" font-family="Times,serif" font-size="14.00">container:</text>
+<text text-anchor="start" x="22" y="-22.8" font-family="Times,serif" font-size="14.00">   state=Complete</text>
+<text text-anchor="start" x="22" y="-7.8" font-family="Times,serif" font-size="14.00">   exit_code=0</text>
+</g>
+<!-- running->containerfinished -->
+<g id="edge11" class="edge">
+<title>running->containerfinished</title>
+<path fill="none" stroke="black" d="M88,-133.82C88,-121.02 88,-106.73 88,-93.32"/>
+<polygon fill="black" stroke="black" points="91.5,-93.27 88,-83.27 84.5,-93.27 91.5,-93.27"/>
+<text text-anchor="start" x="88" y="-104.8" font-family="Times,serif" font-size="14.00">   container process succeeds</text>
+</g>
+</g>
+</svg>
|output_properties|hash|User metadata properties to set on the output collection. The output collection will also have default properties "type" ("intermediate" or "output") and "container_request" (the uuid of container request that produced the collection).|
|cumulative_cost|number|Estimated cost of the cloud VMs used to satisfy the request, including retried attempts and completed subrequests, but not including reused containers.|0 if container was reused or VM price information was not available.|
+h2(#lifecycle). Container request lifecycle
+
+A container request may be created in the Committed state, or created in the Uncommitted state and then moved into the Committed state.
+
+Once a request is in the Committed state, Arvados locates a suitable existing container or schedules a new one. When the assigned container finishes, the request state changes to Final.
+
+A client may cancel a committed request early (before the assigned container finishes) by setting the request priority to zero.
+
+!{max-width:60em;}{{site.baseurl}}/api/methods/container_request_lifecycle.svg!
+{% comment %}
+# svg generated using `graphviz -Tsvg -O`
+digraph {
+ graph [nojustify=true] [labeljust=l]
+
+ invisiblestart [label = ""] [color=white] [group=lifecycle];
+ node [color=black] [fillcolor=white] [style=filled] [shape=box] [nojustify=true];
+ uncommitted [label = "container request:\l state=Uncommitted\l"] [fillcolor=lightgrey] [group=lifecycle];
+ {
+ rank=same;
+ committed [label = "container request:\l state=Committed\l priority>0\l"] [group=lifecycle];
+ reused [label = "container request:\l state=Final\lcontainer:\l state=Complete\l(reused existing container)\l"] [fillcolor=lightblue] [group=endstate];
+ }
+ invisiblestart -> uncommitted [label = " user creates container request\l"] [color=navy] [fontcolor=navy];
+ uncommitted -> committed [label = " user updates to\l state=Committed, priority>0\l"] [color=navy] [fontcolor=navy];
+ queued [label = "container request:\l state=Committed\l priority>0\lcontainer:\l state=Queued\l"] [group=lifecycle];
+ committed -> queued [label = " Arvados creates a new container\l"];
+ {
+ rank=same;
+ locked [label = "container request:\l state=Committed\l priority>0\lcontainer:\l state=Locked\l"] [group=lifecycle];
+ latecancelled [label = "container request:\l state=Final\lcontainer:\l state=Cancelled\l"] [fillcolor=lightblue] [group=endstate];
+ }
+ queued -> locked [label = " Arvados is ready to dispatch the container\l"];
+ {
+ rank=same;
+ running [label = "container request:\l state=Committed\l priority>0\lcontainer:\l state=Running\l"] [group=lifecycle];
+ containerfailed [label = "container request:\l state=Final\lcontainer:\l state=Complete\l exit_code≠0\l"] [fillcolor=lightblue] [group=endstate];
+ }
+ locked -> running [label = " Arvados starts the container process\l"];
+ containerfinished [label = "container request:\l state=Final\lcontainer:\l state=Complete\l exit_code=0\l"] [fillcolor=lightblue] [group=lifecycle];
+
+ committed -> reused [label = "Arvados selects an existing container"] [constraint=false] [labeldistance=0.5];
+ queued -> latecancelled [label = "user updates to priority=0"] [color=navy] [fontcolor=navy];
+ locked -> latecancelled [label = "user updates to priority=0"] [color=navy] [fontcolor=navy] [constraint=false];
+ running -> latecancelled [label = "user updates to priority=0"] [color=navy] [fontcolor=navy] [constraint=false];
+ running -> containerfailed [label = "container process fails"];
+ running -> containerfinished [label = " container process succeeds\l"];
+
+ # layout hacks
+ reused -> latecancelled [style=invis];
+ latecancelled -> containerfailed [style=invis];
+}
+{% endcomment %}
+
h2(#priority). Priority
The @priority@ field has a range of 0-1000.
A container request may be canceled by setting its priority to 0, using an update call.
-When a container request is canceled, it will still reflect the state of the Container it is associated with via the container_uuid attribute. If that Container is being reused by any other container_requests that are still active, i.e., not yet canceled, that Container may continue to run or be scheduled to run by the system in future. However, if no other container_requests are using that Contianer, then the Container will get canceled as well.
+When a container request is canceled, it will still reflect the state of the Container it is associated with via the container_uuid attribute. If that Container is being reused by any other container_requests that are still active, i.e., not yet canceled, that Container may continue to run or be scheduled to run by the system in future. However, if no other container_requests are using that Container, then the Container will get canceled as well.
h2. Methods
!{max-width:40em}{{site.baseurl}}/architecture/dispatchcloud.svg!
{% comment %}
-# svg generated using https://graphviz.it/
+# svg generated using https://dreampuf.github.io/
digraph {
subgraph cluster_cloudvm {
node [color=black] [fillcolor=white] [style=filled];
* @tail_uuid@ identifies the user or role group that receives the permission.
* @head_uuid@ identifies the Arvados object this permission grants access to.
-For details, refer to the "Permissions model documentation":{{ site.baseurl }}/api/permission-model.html. Managing permissions is just a matter of ensuring the desired links exist with the standard @create@, @update@, and @delete@ methods.
+For details, refer to the "Permissions model documentation":{{ site.baseurl }}/api/permission-model.html. Managing permissions is just a matter of ensuring the desired links exist using the standard @create@, @update@, and @delete@ methods.
h3(#grant-permission). Grant permission to an object
h3(#download-a-file-from-a-collection). Download a file from a collection
-Once you have a @Collection@ object, the "@Collection.open@ method":{{ site.baseurl }}/sdk/python/arvados/collection.html#arvados.collection.RichCollectionBase.open lets you open files from a collection the same way you would open files from disk using Python's built-in @open@ function. It returns a file-like object that you can use in many of the same ways you would use any other file object. You can pass it as a source to Python's standard "@shutil.copyfileobj@ function":https://docs.python.org/3/library/shutil.html#shutil.copyfileobj to download it. This code downloads @ExampleFile@ from your collection and saves it to the current working directory as @ExampleDownload@:
+Once you have a @Collection@ object, the "@Collection.open@ method":{{ site.baseurl }}/sdk/python/arvados/collection.html#arvados.collection.RichCollectionBase.open lets you open files from a collection the same way you would open files from disk using Python's built-in @open@ function. You pass a second mode argument like @'rb'@ to open the file in binary mode. It returns a file-like object that you can use in many of the same ways you would use any other file object. You can pass it as a source to Python's standard "@shutil.copyfileobj@ function":https://docs.python.org/3/library/shutil.html#shutil.copyfileobj to download it. This code downloads @ExampleFile@ from your collection and saves it to the current working directory as @ExampleDownload@:
{% codeblock as python %}
import arvados.collection
import shutil
collection = arvados.collection.Collection(...)
with (
- collection.open('ExampleFile') as src_file,
- open('ExampleDownload', 'w') as dst_file,
+ collection.open('ExampleFile', 'rb') as src_file,
+ open('ExampleDownload', 'wb') as dst_file,
):
shutil.copyfileobj(src_file, dst_file)
{% endcodeblock %}
collection = arvados.collection.Collection(...)
with collection.open('ExampleFile', 'w') as my_file:
# Write to my_file as desired.
- # This example writes "Hello, world!" to the file.
+ # This example writes "Hello, Arvados!" to the file.
print("Hello, Arvados!", file=my_file)
collection.save_new(...) # or collection.save() to update an existing collection
{% endcodeblock %}
import shutil
collection = arvados.collection.Collection(...)
with (
- open('ExampleFile') as src_file,
- collection.open('ExampleUpload', 'w') as dst_file,
+ open('ExampleFile', 'rb') as src_file,
+ collection.open('ExampleUpload', 'wb') as dst_file,
):
shutil.copyfileobj(src_file, dst_file)
collection.save_new(...) # or collection.save() to update an existing collection
pprint.pprint(mount_source.get('content'))
{% endcodeblock %}
-h3(#get-input-of-a-cwl-workflow). Get input of a container or CWL workflow run
+h3(#get-input-of-a-cwl-workflow). Get input of a CWL workflow run
When you run a CWL workflow, the CWL inputs are stored in the container request's @mounts@ field as a JSON mount named @/var/lib/cwl/cwl.input.json@.
If you are logged in to an Arvados VM, the Python SDK should be installed.
-To use the Python SDK elsewhere, you can install from PyPI or a distribution package.
+To use the Python SDK elsewhere, you can install it "from an Arvados distribution package":#package-install or "from PyPI using pip":#pip-install.
+{% include 'notebox_begin_warning' %}
As of Arvados 2.2, the Python SDK requires Python 3.6+. The last version to support Python 2.7 is Arvados 2.0.4.
+{% include 'notebox_end' %}
-h2. Option 1: Install from a distribution package
+h2(#package-install). Install from a distribution package
-This installation method is recommended to make the CLI tools available system-wide. It can coexist with the installation method described in option 2, below.
+This installation method is recommended to make the CLI tools available system-wide. It can coexist with the pip installation method described below.
First, configure the "Arvados package repositories":../../install/packages.html
{% include 'install_packages' %}
-h2. Option 2: Install with pip
-
-This installation method is recommended to use the SDK in your own Python programs. If installed into a @virtualenv@, it can coexist with the system-wide installation method from a distribution package.
-
-Run @pip install arvados-python-client@ in an appropriate installation environment, such as a @virtualenv@.
-
-Note:
-
-The SDK uses @pycurl@ which depends on the @libcurl@ C library. To build the module you may have to first install additional packages. On Debian 10 this is:
-
-<pre>
-$ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl-dev
-</pre>
-
-If your version of @pip@ is 1.4 or newer, the @pip install@ command might give an error: "Could not find a version that satisfies the requirement arvados-python-client". If this happens, try @pip install --pre arvados-python-client@.
-
-h2. Test installation
-
-If the SDK is installed and your @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ environment variables are set up correctly (see "api-tokens":{{site.baseurl}}/user/reference/api-tokens.html for details), @import arvados@ should produce no errors.
+{% include 'notebox_begin_warning' %}
+If you are on Ubuntu 18.04, please note that the Arvados packages that use Python depend on the python-3.8 package. This means they are installed under @/usr/share/python3.8@, not @/usr/share/python3@. You will need to update the commands below accordingly.
+{% include 'notebox_end' %}
-If you installed with pip (option 1, above):
+The package includes a virtualenv, which means the correct Python environment needs to be loaded before the Arvados SDK can be imported. You can test the installation by doing that, then creating a client object. Ensure your "@ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ credentials are set up correctly":{{site.baseurl}}/user/reference/api-tokens.html. Then you should be able to run the following without any errors:
<notextile>
-<pre>~$ <code class="userinput">python</code>
+<pre>~$ <code class="userinput">source /usr/share/python3/dist/python3-arvados-python-client/bin/activate</code>
+(python-arvados-python-client) ~$ <code class="userinput">python</code>
Python 3.7.3 (default, Jul 25 2020, 13:03:44)
[GCC 8.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
</pre>
</notextile>
-If you installed from a distribution package (option 2): the package includes a virtualenv, which means the correct Python environment needs to be loaded before the Arvados SDK can be imported. This can be done by activating the virtualenv first:
-
-{% include 'notebox_begin_warning' %}
-If you are on Ubuntu 18.04, please note that the Arvados packages that use Python depend on the python-3.8 package. This means they are installed under @/usr/share/python3.8@, not @/usr/share/python3@. You will need to update the commands below accordingly.
-{% include 'notebox_end' %}
+Alternatively, you can run the Python executable inside the @virtualenv@ directly:
<notextile>
-<pre>~$ <code class="userinput">source /usr/share/python3/dist/python3-arvados-python-client/bin/activate</code>
-(python-arvados-python-client) ~$ <code class="userinput">python</code>
+<pre>~$ <code class="userinput">/usr/share/python3/dist/python3-arvados-python-client/bin/python</code>
Python 3.7.3 (default, Jul 25 2020, 13:03:44)
[GCC 8.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
</pre>
</notextile>
-Or alternatively, by using the Python executable from the virtualenv directly:
+After you have successfully tested your installation, proceed to the the "API client overview":api-client.html and "cookbook":cookbook.html to learn how to use the SDK.
+
+h2(#pip-install). Install from PyPI with pip
+
+This installation method is recommended to use the SDK in your own Python programs. If installed into a @virtualenv@, it can coexist with the system-wide installation method from a distribution package.
+
+The SDK uses @pycurl@ which depends on the @libcurl@ C library. To build the module you may have to first install additional packages. On Debian 10 you can do this by running:
+
+<pre>
+$ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl-dev
+</pre>
+
+Run @python3 -m pip install arvados-python-client@ in an appropriate installation environment, such as a @virtualenv@.
+
+{% include 'notebox_begin_warning' %}
+If your version of @pip@ is 1.4 or newer, the @pip install@ command might give an error: "Could not find a version that satisfies the requirement arvados-python-client". If this happens, try @python3 -m pip install --pre arvados-python-client@.
+{% include 'notebox_end' %}
+
+You can test the installation by creating a client object. Ensure your "@ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ credentials are set up correctly":{{site.baseurl}}/user/reference/api-tokens.html. Then you should be able to run the following without any errors:
<notextile>
-<pre>~$ <code class="userinput">/usr/share/python3/dist/python3-arvados-python-client/bin/python</code>
+<pre>~$ <code class="userinput">python3</code>
Python 3.7.3 (default, Jul 25 2020, 13:03:44)
[GCC 8.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
</pre>
</notextile>
-h2. Usage
-
-Check out the "API client overview":api-client.html and "cookbook":cookbook.html.
+After you have successfully tested your installation, proceed to the the "API client overview":api-client.html and "cookbook":cookbook.html to learn how to use the SDK.
# with the cancelled container.
MaxRetryAttempts: 3
- # The maximum number of compute nodes that can be in use simultaneously
- # If this limit is reduced, any existing nodes with slot number >= new limit
- # will not be counted against the new limit. In other words, the new limit
- # won't be strictly enforced until those nodes with higher slot numbers
- # go down.
- MaxComputeVMs: 64
-
# Schedule all child containers on preemptible instances (e.g. AWS
# Spot Instances) even if not requested by the submitter.
#
# providers too, if desired.
MaxConcurrentInstanceCreateOps: 1
+ # The maximum number of instances to run at a time, or 0 for
+ # unlimited.
+ #
+ # If more instances than this are already running and busy
+ # when the dispatcher starts up, the running containers will
+ # be allowed to finish before the excess instances are shut
+ # down.
+ MaxInstances: 64
+
# Interval between cloud provider syncs/updates ("list all
# instances").
SyncInterval: 1m
"Containers.Logging": false,
"Containers.LogReuseDecisions": false,
"Containers.LSF": false,
- "Containers.MaxComputeVMs": false,
"Containers.MaxDispatchAttempts": false,
"Containers.MaxRetryAttempts": true,
"Containers.MinRetryPeriod": true,
instanceTypes: cluster.InstanceTypes,
maxProbesPerSecond: cluster.Containers.CloudVMs.MaxProbesPerSecond,
maxConcurrentInstanceCreateOps: cluster.Containers.CloudVMs.MaxConcurrentInstanceCreateOps,
+ maxInstances: cluster.Containers.CloudVMs.MaxInstances,
probeInterval: duration(cluster.Containers.CloudVMs.ProbeInterval, defaultProbeInterval),
syncInterval: duration(cluster.Containers.CloudVMs.SyncInterval, defaultSyncInterval),
timeoutIdle: duration(cluster.Containers.CloudVMs.TimeoutIdle, defaultTimeoutIdle),
probeInterval time.Duration
maxProbesPerSecond int
maxConcurrentInstanceCreateOps int
+ maxInstances int
timeoutIdle time.Duration
timeoutBooting time.Duration
timeoutProbe time.Duration
// pool. The worker is added immediately; instance creation runs in
// the background.
//
-// Create returns false if a pre-existing error state prevents it from
-// even attempting to create a new instance. Those errors are logged
-// by the Pool, so the caller does not need to log anything in such
-// cases.
+// Create returns false if a pre-existing error or a configuration
+// setting prevents it from even attempting to create a new
+// instance. Those errors are logged by the Pool, so the caller does
+// not need to log anything in such cases.
func (wp *Pool) Create(it arvados.InstanceType) bool {
logger := wp.logger.WithField("InstanceType", it.Name)
wp.setupOnce.Do(wp.setup)
}
wp.mtx.Lock()
defer wp.mtx.Unlock()
- if time.Now().Before(wp.atQuotaUntil) || wp.instanceSet.throttleCreate.Error() != nil {
+ if time.Now().Before(wp.atQuotaUntil) ||
+ wp.instanceSet.throttleCreate.Error() != nil ||
+ (wp.maxInstances > 0 && wp.maxInstances <= len(wp.workers)+len(wp.creating)) {
return false
}
// The maxConcurrentInstanceCreateOps knob throttles the number of node create
}
wp.updateWorker(inst, it)
}()
+ if len(wp.creating)+len(wp.workers) == wp.maxInstances {
+ logger.Infof("now at MaxInstances limit of %d instances", wp.maxInstances)
+ }
return true
}
// AtQuota returns true if Create is not expected to work at the
-// moment.
+// moment (e.g., cloud provider has reported quota errors, or we are
+// already at our own configured quota).
func (wp *Pool) AtQuota() bool {
wp.mtx.Lock()
defer wp.mtx.Unlock()
- return time.Now().Before(wp.atQuotaUntil)
+ return time.Now().Before(wp.atQuotaUntil) || (wp.maxInstances > 0 && wp.maxInstances <= len(wp.workers)+len(wp.creating))
}
// SetIdleBehavior determines how the indicated instance will behave
DefaultKeepCacheRAM ByteSize
DispatchPrivateKey string
LogReuseDecisions bool
- MaxComputeVMs int
MaxDispatchAttempts int
MaxRetryAttempts int
MinRetryPeriod Duration
MaxCloudOpsPerSecond int
MaxProbesPerSecond int
MaxConcurrentInstanceCreateOps int
+ MaxInstances int
PollInterval Duration
ProbeInterval Duration
SSHPort string
attr_accessor :job_readable
UNUSED_NODE_IP = '127.40.4.0'
+ MAX_VMS = 3
api_accessible :user, :extend => :common do |t|
t.add :hostname
# query label:
'Node.available_slot_number',
# [col_id, val] for $1 vars:
- [[nil, Rails.configuration.Containers.MaxComputeVMs]],
+ [[nil, MAX_VMS]],
).rows.first.andand.first
end
!Rails.configuration.Containers.SLURM.Managed.DNSServerConfTemplate.to_s.empty? and
!Rails.configuration.Containers.SLURM.Managed.AssignNodeHostname.empty?)
- (0..Rails.configuration.Containers.MaxComputeVMs-1).each do |slot_number|
+ (0..MAX_VMS-1).each do |slot_number|
hostname = hostname_for_slot(slot_number)
hostfile = File.join Rails.configuration.Containers.SLURM.Managed.DNSServerConfDir, "#{hostname}.conf"
if !File.exist? hostfile
arvcfg.declare_config "Containers.MaxDispatchAttempts", Integer, :max_container_dispatch_attempts
arvcfg.declare_config "Containers.MaxRetryAttempts", Integer, :container_count_max
arvcfg.declare_config "Containers.AlwaysUsePreemptibleInstances", Boolean, :preemptible_instances
-arvcfg.declare_config "Containers.MaxComputeVMs", Integer, :max_compute_nodes
arvcfg.declare_config "Containers.Logging.LogBytesPerEvent", Integer, :crunch_log_bytes_per_event
arvcfg.declare_config "Containers.Logging.LogSecondsBetweenEvents", ActiveSupport::Duration, :crunch_log_seconds_between_events
arvcfg.declare_config "Containers.Logging.LogThrottlePeriod", ActiveSupport::Duration, :crunch_log_throttle_period
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-require 'test_helper'
-require 'tmpdir'
-require 'tempfile'
-
-class NodeTest < ActiveSupport::TestCase
- def ping_node(node_name, ping_data)
- set_user_from_auth :admin
- node = nodes(node_name)
- node.ping({ping_secret: node.info['ping_secret'],
- ip: node.ip_address}.merge(ping_data))
- node
- end
-
- test "pinging a node can add and update stats" do
- node = ping_node(:idle, {total_cpu_cores: '12', total_ram_mb: '512'})
- assert_equal(12, node.properties['total_cpu_cores'])
- assert_equal(512, node.properties['total_ram_mb'])
- end
-
- test "stats disappear if not in a ping" do
- node = ping_node(:idle, {total_ram_mb: '256'})
- refute_includes(node.properties, 'total_cpu_cores')
- assert_equal(256, node.properties['total_ram_mb'])
- end
-
- test "worker state is down for node with no slot" do
- node = nodes(:was_idle_now_down)
- assert_nil node.slot_number, "fixture is not what I expected"
- assert_equal 'down', node.crunch_worker_state, "wrong worker state"
- end
-
- test "dns_server_conf_template" do
- Rails.configuration.Containers.SLURM.Managed.DNSServerConfDir = Rails.root.join 'tmp'
- Rails.configuration.Containers.SLURM.Managed.DNSServerConfTemplate = Rails.root.join 'config', 'unbound.template'
- conffile = Rails.root.join 'tmp', 'compute65535.conf'
- File.unlink conffile rescue nil
- assert Node.dns_server_update 'compute65535', '127.0.0.1'
- assert_match(/\"1\.0\.0\.127\.in-addr\.arpa\. IN PTR compute65535\.zzzzz\.arvadosapi\.com\"/, IO.read(conffile))
- File.unlink conffile
- end
-
- test "dns_server_restart_command" do
- Rails.configuration.Containers.SLURM.Managed.DNSServerConfDir = Rails.root.join 'tmp'
- Rails.configuration.Containers.SLURM.Managed.DNSServerReloadCommand = 'foobar'
- restartfile = Rails.root.join 'tmp', 'restart.txt'
- File.unlink restartfile rescue nil
- assert Node.dns_server_update 'compute65535', '127.0.0.127'
- assert_equal "foobar\n", IO.read(restartfile)
- File.unlink restartfile
- end
-
- test "dns_server_restart_command fail" do
- Rails.configuration.Containers.SLURM.Managed.DNSServerConfDir = Rails.root.join 'tmp', 'bogusdir'
- Rails.configuration.Containers.SLURM.Managed.DNSServerReloadCommand = 'foobar'
- refute Node.dns_server_update 'compute65535', '127.0.0.127'
- end
-
- test "dns_server_update_command with valid command" do
- testfile = Rails.root.join('tmp', 'node_test_dns_server_update_command.txt')
- Rails.configuration.Containers.SLURM.Managed.DNSServerUpdateCommand =
- ('echo -n "%{hostname} == %{ip_address}" >' +
- testfile.to_s.shellescape)
- assert Node.dns_server_update 'compute65535', '127.0.0.1'
- assert_equal 'compute65535 == 127.0.0.1', IO.read(testfile)
- File.unlink testfile
- end
-
- test "dns_server_update_command with failing command" do
- Rails.configuration.Containers.SLURM.Managed.DNSServerUpdateCommand = 'false %{hostname}'
- refute Node.dns_server_update 'compute65535', '127.0.0.1'
- end
-
- test "dns update with no commands/dirs configured" do
- Rails.configuration.Containers.SLURM.Managed.DNSServerUpdateCommand = ""
- Rails.configuration.Containers.SLURM.Managed.DNSServerConfDir = ""
- Rails.configuration.Containers.SLURM.Managed.DNSServerConfTemplate = 'ignored!'
- Rails.configuration.Containers.SLURM.Managed.DNSServerReloadCommand = 'ignored!'
- assert Node.dns_server_update 'compute65535', '127.0.0.127'
- end
-
- test "don't leave temp files behind if there's an error writing them" do
- Rails.configuration.Containers.SLURM.Managed.DNSServerConfTemplate = Rails.root.join 'config', 'unbound.template'
- Tempfile.any_instance.stubs(:puts).raises(IOError)
- Dir.mktmpdir do |tmpdir|
- Rails.configuration.Containers.SLURM.Managed.DNSServerConfDir = tmpdir
- refute Node.dns_server_update 'compute65535', '127.0.0.127'
- assert_empty Dir.entries(tmpdir).select{|f| File.file? f}
- end
- end
-
- test "ping new node with no hostname and default config" do
- node = ping_node(:new_with_no_hostname, {})
- slot_number = node.slot_number
- refute_nil slot_number
- assert_equal("compute#{slot_number}", node.hostname)
- end
-
- test "ping new node with no hostname and no config" do
- Rails.configuration.Containers.SLURM.Managed.AssignNodeHostname = false
- node = ping_node(:new_with_no_hostname, {})
- refute_nil node.slot_number
- assert_nil node.hostname
- end
-
- test "ping new node with zero padding config" do
- Rails.configuration.Containers.SLURM.Managed.AssignNodeHostname = 'compute%<slot_number>04d'
- node = ping_node(:new_with_no_hostname, {})
- slot_number = node.slot_number
- refute_nil slot_number
- assert_equal("compute000#{slot_number}", node.hostname)
- end
-
- test "ping node with hostname and config and expect hostname unchanged" do
- node = ping_node(:new_with_custom_hostname, {})
- assert_equal(23, node.slot_number)
- assert_equal("custom1", node.hostname)
- end
-
- test "ping node with hostname and no config and expect hostname unchanged" do
- Rails.configuration.Containers.SLURM.Managed.AssignNodeHostname = false
- node = ping_node(:new_with_custom_hostname, {})
- assert_equal(23, node.slot_number)
- assert_equal("custom1", node.hostname)
- end
-
- # Ping two nodes: one without a hostname and the other with a hostname.
- # Verify that the first one gets a hostname and second one is unchanged.
- test "ping two nodes one with no hostname and one with hostname and check hostnames" do
- # ping node with no hostname and expect it set with config format
- node = ping_node(:new_with_no_hostname, {})
- refute_nil node.slot_number
- assert_equal "compute#{node.slot_number}", node.hostname
-
- # ping node with a hostname and expect it to be unchanged
- node2 = ping_node(:new_with_custom_hostname, {})
- refute_nil node2.slot_number
- assert_equal "custom1", node2.hostname
- end
-
- test "update dns when hostname and ip_address are cleared" do
- act_as_system_user do
- node = ping_node(:new_with_custom_hostname, {})
- Node.expects(:dns_server_update).with(node.hostname, Node::UNUSED_NODE_IP)
- node.update_attributes(hostname: nil, ip_address: nil)
- end
- end
-
- test "update dns when hostname changes" do
- act_as_system_user do
- node = ping_node(:new_with_custom_hostname, {})
-
- Node.expects(:dns_server_update).with(node.hostname, Node::UNUSED_NODE_IP)
- Node.expects(:dns_server_update).with('foo0', node.ip_address)
- node.update_attributes!(hostname: 'foo0')
-
- Node.expects(:dns_server_update).with('foo0', Node::UNUSED_NODE_IP)
- node.update_attributes!(hostname: nil, ip_address: nil)
-
- Node.expects(:dns_server_update).with('foo0', '10.11.12.13')
- node.update_attributes!(hostname: 'foo0', ip_address: '10.11.12.13')
-
- Node.expects(:dns_server_update).with('foo0', '10.11.12.14')
- node.update_attributes!(hostname: 'foo0', ip_address: '10.11.12.14')
- end
- end
-
- test 'newest ping wins IP address conflict' do
- act_as_system_user do
- n1, n2 = Node.create!, Node.create!
-
- n1.ping(ip: '10.5.5.5', ping_secret: n1.info['ping_secret'])
- n1.reload
-
- Node.expects(:dns_server_update).with(n1.hostname, Node::UNUSED_NODE_IP)
- Node.expects(:dns_server_update).with(Not(equals(n1.hostname)), '10.5.5.5')
- n2.ping(ip: '10.5.5.5', ping_secret: n2.info['ping_secret'])
-
- n1.reload
- n2.reload
- assert_nil n1.ip_address
- assert_equal '10.5.5.5', n2.ip_address
-
- Node.expects(:dns_server_update).with(n2.hostname, Node::UNUSED_NODE_IP)
- Node.expects(:dns_server_update).with(n1.hostname, '10.5.5.5')
- n1.ping(ip: '10.5.5.5', ping_secret: n1.info['ping_secret'])
-
- n1.reload
- n2.reload
- assert_nil n2.ip_address
- assert_equal '10.5.5.5', n1.ip_address
- end
- end
-
- test 'run out of slots' do
- Rails.configuration.Containers.MaxComputeVMs = 3
- act_as_system_user do
- Node.destroy_all
- (1..4).each do |i|
- n = Node.create!
- args = { ip: "10.0.0.#{i}", ping_secret: n.info['ping_secret'] }
- if i <= Rails.configuration.Containers.MaxComputeVMs
- n.ping(args)
- else
- assert_raises do
- n.ping(args)
- end
- end
- end
- end
- end
-end
raise
except EnvironmentError as e:
raise llfuse.FUSEError(e.errno)
+ except NotImplementedError:
+ raise llfuse.FUSEError(errno.ENOTSUP)
except arvados.errors.KeepWriteError as e:
_logger.error("Keep write error: " + str(e))
raise llfuse.FUSEError(errno.EIO)
from future.utils import viewitems
from builtins import str
from builtins import object
+from pathlib import Path
from six import assertRegex
+import errno
import json
import llfuse
import logging
import arvados
import arvados_fuse as fuse
+from arvados_fuse import fusedir
from . import run_test_server
from .integration_test import IntegrationTest
self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection, enable_write=False)
self.pool.apply(_readonlyCollectionTestHelper, (self.mounttmp,))
+
+
+@parameterized.parameterized_class([
+ {'root_class': fusedir.ProjectDirectory, 'root_kwargs': {
+ 'project_object': run_test_server.fixture('users')['admin'],
+ }},
+ {'root_class': fusedir.ProjectDirectory, 'root_kwargs': {
+ 'project_object': run_test_server.fixture('groups')['public'],
+ }},
+])
+class UnsupportedCreateTest(MountTestBase):
+ root_class = None
+ root_kwargs = {}
+
+ def setUp(self):
+ super().setUp()
+ if 'prefs' in self.root_kwargs.get('project_object', ()):
+ self.root_kwargs['project_object']['prefs'] = {}
+ self.make_mount(self.root_class, **self.root_kwargs)
+ # Make sure the directory knows about its top-level ents.
+ os.listdir(self.mounttmp)
+
+ def test_create(self):
+ test_path = Path(self.mounttmp, 'test_create')
+ with self.assertRaises(OSError) as exc_check:
+ with test_path.open('w'):
+ pass
+ self.assertEqual(exc_check.exception.errno, errno.ENOTSUP)
+
+
+# FIXME: IMO, for consistency with the "create inside a project" case,
+# these operations should also return ENOTSUP instead of EPERM.
+# Right now they're returning EPERM because the clasess' writable() method
+# usually returns False, and the Operations class transforms that accordingly.
+# However, for cases where the mount will never be writable, I think ENOTSUP
+# is a clearer error: it lets the user know they can't fix the problem by
+# adding permissions in Arvados, etc.
+@parameterized.parameterized_class([
+ {'root_class': fusedir.MagicDirectory,
+ 'preset_dir': 'by_id',
+ 'preset_file': 'README',
+ },
+
+ {'root_class': fusedir.SharedDirectory,
+ 'root_kwargs': {
+ 'exclude': run_test_server.fixture('users')['admin']['uuid'],
+ },
+ 'preset_dir': 'Active User',
+ },
+
+ {'root_class': fusedir.TagDirectory,
+ 'root_kwargs': {
+ 'tag': run_test_server.fixture('links')['foo_collection_tag']['name'],
+ },
+ 'preset_dir': run_test_server.fixture('collections')['foo_collection_in_aproject']['uuid'],
+ },
+
+ {'root_class': fusedir.TagsDirectory,
+ 'preset_dir': run_test_server.fixture('links')['foo_collection_tag']['name'],
+ },
+])
+class UnsupportedOperationsTest(UnsupportedCreateTest):
+ preset_dir = None
+ preset_file = None
+
+ def test_create(self):
+ test_path = Path(self.mounttmp, 'test_create')
+ with self.assertRaises(OSError) as exc_check:
+ with test_path.open('w'):
+ pass
+ self.assertEqual(exc_check.exception.errno, errno.EPERM)
+
+ def test_mkdir(self):
+ test_path = Path(self.mounttmp, 'test_mkdir')
+ with self.assertRaises(OSError) as exc_check:
+ test_path.mkdir()
+ self.assertEqual(exc_check.exception.errno, errno.EPERM)
+
+ def test_rename(self):
+ src_name = self.preset_dir or self.preset_file
+ if src_name is None:
+ return
+ test_src = Path(self.mounttmp, src_name)
+ test_dst = test_src.with_name('test_dst')
+ with self.assertRaises(OSError) as exc_check:
+ test_src.rename(test_dst)
+ self.assertEqual(exc_check.exception.errno, errno.EPERM)
+
+ def test_rmdir(self):
+ if self.preset_dir is None:
+ return
+ test_path = Path(self.mounttmp, self.preset_dir)
+ with self.assertRaises(OSError) as exc_check:
+ test_path.rmdir()
+ self.assertEqual(exc_check.exception.errno, errno.EPERM)
+
+ def test_unlink(self):
+ if self.preset_file is None:
+ return
+ test_path = Path(self.mounttmp, self.preset_file)
+ with self.assertRaises(OSError) as exc_check:
+ test_path.unlink()
+ self.assertEqual(exc_check.exception.errno, errno.EPERM)
projects / arvados.git / commitdiff
