From 48562a3a2c4b4ca877ddb06726c82cdc303e681e Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 18 Mar 2022 15:44:23 -0400 Subject: [PATCH] 18180: Documentation improvements discussing spot instance support. Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- doc/admin/spot-instances.html.textile.liquid | 19 +++++--- .../cwl/cwl-run-options.html.textile.liquid | 19 ++++++++ doc/user/cwl/cwl-style.html.textile.liquid | 43 ++++++++++++++----- sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml | 2 +- sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml | 2 +- sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml | 2 +- 6 files changed, 69 insertions(+), 18 deletions(-) diff --git a/doc/admin/spot-instances.html.textile.liquid b/doc/admin/spot-instances.html.textile.liquid index 7ca57df0ab..3837f30d6d 100644 --- a/doc/admin/spot-instances.html.textile.liquid +++ b/doc/admin/spot-instances.html.textile.liquid @@ -16,13 +16,11 @@ Currently Arvados supports preemptible instances using AWS and Azure spot instan h2. Configuration -First, ensure automatic selection of preemptible instances is not disabled in your configuration file (this is enabled by default, but can be disabled with @AlwaysUsePreemptibleInstances: false@), and add entries to @InstanceTypes@ that have @Preemptible: true@. Typically you want to add both preemptible and non-preemptible entries for each cloud provider VM type. The @Price@ for preemptible instances is the maximum bid price, the actual price paid is dynamic and will likely be lower. For example: +Add entries to @InstanceTypes@ that have @Preemptible: true@. Typically you want to add both preemptible and non-preemptible entries for each cloud provider VM type. The @Price@ for preemptible instances is the maximum bid price, the actual price paid is dynamic and will likely be lower. For example: 
 Clusters:
-  ClusterID: 
-    Containers:
-      AlwaysUsePreemptibleInstances: true
+  ClusterID:
     InstanceTypes:
       m4.large:
         Preemptible: false
@@ -40,7 +38,18 @@ Clusters:
         Price: 0.1
-When @AlwaysUsePreemptibleInstances@ is enabled, child containers (workflow steps) will automatically be made preemptible. Note that because preempting the workflow runner would cancel the entire workflow, the workflow runner runs in a reserved (non-preemptible) instance. +Next, you can choose to enable automatic use of preemptible instances: + +
+    Containers:
+      AlwaysUsePreemptibleInstances: true
+
+ +If @AlwaysUsePreemptibleInstances@ is "true", child containers (workflow steps) will always select preemptible instances, regardless of user option. + +If @AlwaysUsePreemptibleInstances@ is "false" (the default) or unspecified, preemptible instance are "used when requested by the user.":{{site.baseurl}}/user/cwl/cwl-run-options.html#preemptible + +Note that regardless of the value of @AlwaysUsePreemptibleInstances@, the top level workflow runner container always runs in a reserved (non-preemptible) instance, to avoid situations where the workflow runner is killed requiring the entire to be restarted. No additional configuration is required, "arvados-dispatch-cloud":{{site.baseurl}}/install/crunch2-cloud/install-dispatch-cloud.html will now start preemptible instances where appropriate. diff --git a/doc/user/cwl/cwl-run-options.html.textile.liquid b/doc/user/cwl/cwl-run-options.html.textile.liquid index d331dad871..94e46ae1bc 100644 --- a/doc/user/cwl/cwl-run-options.html.textile.liquid +++ b/doc/user/cwl/cwl-run-options.html.textile.liquid @@ -63,6 +63,9 @@ table(table table-bordered table-condensed). |==--priority== PRIORITY|Workflow priority (range 1..1000, higher has precedence over lower)| |==--thread-count== THREAD_COUNT|Number of threads to use for container submit and output collection.| |==--http-timeout== HTTP_TIMEOUT|API request timeout in seconds. Default is 300 seconds (5 minutes).| +|==--enable-preemptible==|Use preemptible instances. Control individual steps with "arv:UsePreemptible":cwl-extensions.html#UsePreemptible hint.| +|==--disable-preemptible==|Don't use preemptible instances.| +|==--skip-schemas==|Skip loading of extension schemas (the $schemas section).| |==--trash-intermediate==|Immediately trash intermediate outputs on workflow success.| |==--no-trash-intermediate==|Do not trash intermediate outputs (default).| @@ -143,3 +146,19 @@ Using @--intermediate-output-ttl@ without @--trash-intermediate@ means that inte h3(#federation). Run workflow on a remote federated cluster By default, the workflow runner will run on the local (home) cluster. Using @--submit-runner-cluster@ you can specify that the runner should be submitted to a remote federated cluster. When doing this, @--project-uuid@ should specify a project on that cluster. Steps making up the workflow will be submitted to the remote federated cluster by default, but the behavior of @arv:ClusterTarget@ is unchanged. Note: when using this option, any resources that need to be uploaded in order to run the workflow (such as files or Docker images) will be uploaded to the local (home) cluster, and streamed to the federated cluster on demand. + +h3(#preemptible). Using preemptible (spot) instances + +Preemptible instances typically offer lower cost computation with a tradeoff of lower service guarantees. If a compute node is preempted, Arvados will restart the computation on a new instance. + +If the sitewide configuration @Containers.AlwaysUsePreemptibleInstances@ is true, workflow steps will always select preemptible instances, regardless of user option. + +If @Containers.AlwaysUsePreemptibleInstances@ is false, you can request preemptible instances for a specific run with the @arvados-cwl-runner --enable-preemptible@ option. + +Within the workflow, you can control whether individual steps should be preemptible with the "arv:UsePreemptible":cwl-extensions.html#UsePreemptible hint. + +If a workflow requests preemptible instances with "arv:UsePreemptible":cwl-extensions.html#UsePreemptible , but you _do not_ want to use preemptible instances, you can override it for a specific run with the @arvados-cwl-runner --disable-preemptible@ option. + +h3(#gpu). Use CUDA GPU instances + +See "cwltool:CUDARequirement":cwl-extensions.html#CUDARequirement . diff --git a/doc/user/cwl/cwl-style.html.textile.liquid b/doc/user/cwl/cwl-style.html.textile.liquid index 853ed3b3e2..303ae37e9e 100644 --- a/doc/user/cwl/cwl-style.html.textile.liquid +++ b/doc/user/cwl/cwl-style.html.textile.liquid @@ -13,7 +13,15 @@ h2(#performance). Performance To get the best perfomance from your workflows, be aware of the following Arvados features, behaviors, and best practices. -Does your application support NVIDIA GPU acceleration? Use "cwltool:CUDARequirement":cwl-extensions.html#CUDARequirement to request nodes with GPUs. +h3. Does your application support NVIDIA GPU acceleration? + +Use "cwltool:CUDARequirement":cwl-extensions.html#CUDARequirement to request nodes with GPUs. + +h3. Trying to reduce costs? + +Try "using preemptible (spot) instances":cwl-run-options.html#preemptible . + +h3. You have a sequence of short-running steps If you have a sequence of short-running steps (less than 1-2 minutes each), use the Arvados extension "arv:RunInSingleContainer":cwl-extensions.html#RunInSingleContainer to avoid scheduling and data transfer overhead by running all the steps together in the same container on the same node. To use this feature, @cwltool@ must be installed in the container image. Example: @@ -42,10 +50,16 @@ steps: run: subworkflow-with-short-steps.cwl {% endcodeblock %} +h3. Avoid declaring @InlineJavascriptRequirement@ or @ShellCommandRequirement@ + Avoid declaring @InlineJavascriptRequirement@ or @ShellCommandRequirement@ unless you specifically need them. Don't include them "just in case" because they change the default behavior and may add extra overhead. +h3. Prefer text substitution to Javascript + When combining a parameter value with a string, such as adding a filename extension, write @$(inputs.file.basename).ext@ instead of @$(inputs.file.basename + 'ext')@. The first form is evaluated as a simple text substitution, the second form (using the @+@ operator) is evaluated as an arbitrary Javascript expression and requires that you declare @InlineJavascriptRequirement@. +h3. Use @ExpressionTool@ to efficiently rearrange input files + Use @ExpressionTool@ to efficiently rearrange input files between steps of a Workflow. For example, the following expression accepts a directory containing files paired by @_R1_@ and @_R2_@ and produces an array of Directories containing each pair. {% codeblock as yaml %} @@ -80,9 +94,13 @@ expression: | } {% endcodeblock %} -Available compute nodes types vary over time and across different cloud providers, so try to limit the RAM requirement to what the program actually needs. However, if you need to target a specific compute node type, see this discussion on "calculating RAM request and choosing instance type for containers.":{{site.baseurl}}/api/execution.html#RAM +h3. Limit RAM requests to what you really need + +Available compute nodes types vary over time and across different cloud providers, so it is important to limit the RAM requirement to what the program actually needs. However, if you need to target a specific compute node type, see this discussion on "calculating RAM request and choosing instance type for containers.":{{site.baseurl}}/api/execution.html#RAM -Instead of scattering separate steps, prefer to scatter over a subworkflow. +h3. Avoid scattering by step by step + +Instead of a scatter step that feeds into another scatter step, prefer to scatter over a subworkflow. With the following pattern, @step1@ has to wait for all samples to complete before @step2@ can start computing on any samples. This means a single long-running sample can prevent the rest of the workflow from moving on: @@ -148,10 +166,16 @@ h2. Portability To write workflows that are easy to modify and portable across CWL runners (in the event you need to share your workflow with others), there are several best practices to follow: +h3. Always provide @DockerRequirement@ + Workflows should always provide @DockerRequirement@ in the @hints@ or @requirements@ section. +h3. Build a reusable library of components + Build a reusable library of components. Share tool wrappers and subworkflows between projects. Make use of and contribute to "community maintained workflows and tools":https://github.com/common-workflow-library and tool registries such as "Dockstore":http://dockstore.org . +h3. Supply scripts as input parameters + CommandLineTools wrapping custom scripts should represent the script as an input parameter with the script file as a default value. Use @secondaryFiles@ for scripts that consist of multiple files. For example: {% codeblock as yaml %} @@ -180,22 +204,21 @@ outputs: glob: "*.fastq" {% endcodeblock %} +h3. Getting the temporary and output directories + You can get the designated temporary directory using @$(runtime.tmpdir)@ in your CWL file, or from the @$TMPDIR@ environment variable in your script. Similarly, you can get the designated output directory using $(runtime.outdir), or from the @HOME@ environment variable in your script. -Avoid specifying resource requirements in CommandLineTool. Prefer to specify them in the workflow. You can provide a default resource requirement in the top level @hints@ section, and individual steps can override it with their own resource requirement. +h3. Specifying @ResourceRequirement@ + +Avoid specifying resources in the @requirements@ section of a @CommandLineTool@, put it in the @hints@ section instead. This enables you to override the tool resource hint with a workflow step level requirement: {% codeblock as yaml %} cwlVersion: v1.0 class: Workflow inputs: inp: File -hints: - ResourceRequirement: - ramMin: 1000 - coresMin: 1 - tmpdirMin: 45000 steps: step1: in: {inp: inp} @@ -205,7 +228,7 @@ steps: in: {inp: step1/inp} out: [out] run: tool2.cwl - hints: + requirements: ResourceRequirement: ramMin: 2000 coresMin: 2 diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml index 443a027aea..af75481431 100644 --- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml +++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml @@ -395,7 +395,7 @@ $graph: fields: class: type: string - doc: "Always 'arv:ProcessProperties" + doc: "Always 'arv:UsePreemptible" jsonldPredicate: _id: "@type" _type: "@vocab" diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml index 633edaad23..0ae451ccaa 100644 --- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml +++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml @@ -338,7 +338,7 @@ $graph: fields: class: type: string - doc: "Always 'arv:ProcessProperties" + doc: "Always 'arv:UsePreemptible" jsonldPredicate: _id: "@type" _type: "@vocab" diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml index 8ca064a6e5..de5e55ca01 100644 --- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml +++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml @@ -340,7 +340,7 @@ $graph: fields: class: type: string - doc: "Always 'arv:ProcessProperties" + doc: "Always 'arv:UsePreemptible" jsonldPredicate: _id: "@type" _type: "@vocab" -- 2.30.2