if current_user && !profile_config.empty?
current_user_profile = current_user.prefs[:profile]
profile_config.each do |k, entry|
- if entry['Required']
+ if entry[:Required]
if !current_user_profile ||
!current_user_profile[k] ||
current_user_profile[k].empty?
def profile
params[:offer_return_to] ||= params[:return_to]
+
+ # In a federation situation, when you get a user record using
+ # "current user of token" it can fetch a stale user record from
+ # the local cluster. So even if profile settings were just written
+ # to the user record on the login cluster (because the user just
+ # filled out the profile), those profile settings may not appear
+ # in the "current user" response because it is returning a cached
+ # record from the local cluster.
+ #
+ # In this case, explicitly fetching user record forces it to get a
+ # fresh record from the login cluster.
+ Thread.current[:user] = User.find(current_user.uuid)
end
def activity
</div>
<% profile_config.kind_of?(Array) && profile_config.andand.each do |entry| %>
- <% if entry['Key'] %>
+ <% if entry[:Key] %>
<%
show_save_button = true
- label = entry['Required'] ? '* ' : ''
- label += entry['FormFieldTitle']
- value = current_user_profile[entry['Key'].to_sym] if current_user_profile
+ label = entry[:Required] ? '* ' : ''
+ label += entry[:FormFieldTitle]
+ value = current_user_profile[entry[:Key].to_sym] if current_user_profile
%>
<div class="form-group">
- <label for="<%=entry['Key']%>"
+ <label for="<%=entry[:Key]%>"
class="col-sm-3 control-label"
- style=<%="color:red" if entry['Required']&&(!value||value.empty?)%>> <%=label%>
+ style=<%="color:red" if entry[:Required]&&(!value||value.empty?)%>> <%=label%>
</label>
- <% if entry['Type'] == 'select' %>
+ <% if entry[:Type] == 'select' %>
<div class="col-sm-8">
- <select class="form-control" name="user[prefs][profile][<%=entry['Key']%>]">
- <% entry['Options'].each do |option, _| %>
+ <select class="form-control" name="user[prefs][profile][<%=entry[:Key]%>]">
+ <% entry[:Options].each do |option, _| %>
+ <% option = option.to_s %>
<option value="<%=option%>" <%='selected' if option==value%>><%=option%></option>
<% end %>
</select>
</div>
<% else %>
<div class="col-sm-8">
- <input type="text" class="form-control" name="user[prefs][profile][<%=entry['Key']%>]" placeholder="<%=entry['FormFieldDescription']%>" value="<%=value%>" ></input>
+ <input type="text" class="form-control" name="user[prefs][profile][<%=entry[:Key]%>]" placeholder="<%=entry[:FormFieldDescription]%>" value="<%=value%>" ></input>
</div>
<% end %>
</div>
. build/run-library.sh
+# This defines python_sdk_version and cwl_runner_version with python-style
+# package suffixes (.dev/rc)
calculate_python_sdk_cwl_package_versions
-cwl_runner_version=$(echo -n $cwl_runner_version | sed s/~dev/.dev/g | sed s/~rc/rc/g)
-
set -x
docker build --no-cache --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py --build-arg pipcmd=$pipcmd -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
echo arv-keepdocker arvados/jobs $cwl_runner_version
ARVADOS_BUILDING_ITERATION="1"
fi
+# This defines python_sdk_version and cwl_runner_version with python-style
+# package suffixes (.dev/rc)
calculate_python_sdk_cwl_package_versions
-echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
-
-if [[ "${python_sdk_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
- python_sdk_version="${python_sdk_version}-1"
-else
- python_sdk_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
+if [[ -z "$cwl_runner_version" ]]; then
+ echo "ERROR: cwl_runner_version is empty";
+ exit 1
fi
-# What we use to tag the Docker image. For development and release
-# candidate packages, the OS package has a "~dev" or "~rc" suffix, but
-# Python requires a ".dev" or "rc" suffix. Arvados-cwl-runner will be
-# expecting the Python-compatible version string when it tries to pull
-# the Docker image, but --build-arg is expecting the OS package
+echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
+
+# For development and release candidate packages, the OS package has a "~dev"
+# or "~rc" suffix, but Python requires a ".dev" or "rc" suffix.
+#
+# Arvados-cwl-runner will be expecting the Python-compatible version string
+# when it tries to pull the Docker image, so we use that to tag the Docker
+# image.
+#
+# The --build-arg docker invocation arguments are expecting the OS package
# version.
-cwl_runner_version_tag=$(echo -n $cwl_runner_version | sed s/~dev/.dev/g | sed s/~rc/rc/g)
+python_sdk_version_os=$(echo -n $python_sdk_version | sed s/.dev/~dev/g | sed s/rc/~rc/g)
+cwl_runner_version_os=$(echo -n $cwl_runner_version | sed s/.dev/~dev/g | sed s/rc/~rc/g)
-if [[ -z "$cwl_runner_version_tag" ]]; then
- echo "ERROR: cwl_runner_version_tag is empty";
- exit 1
+if [[ "${python_sdk_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
+ python_sdk_version_os="${python_sdk_version_os}-1"
+else
+ python_sdk_version_os="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
fi
-if [[ "${cwl_runner_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
- cwl_runner_version="${cwl_runner_version}-1"
+if [[ "${cwl_runner_version_os}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
+ cwl_runner_version_os="${cwl_runner_version_os}-1"
else
- cwl_runner_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
+ cwl_runner_version_os="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
fi
cd docker/jobs
docker build $NOCACHE \
- --build-arg python_sdk_version=${python_sdk_version} \
- --build-arg cwl_runner_version=${cwl_runner_version} \
+ --build-arg python_sdk_version=${python_sdk_version_os} \
+ --build-arg cwl_runner_version=${cwl_runner_version_os} \
--build-arg repo_version=${REPO} \
- -t arvados/jobs:$cwl_runner_version_tag .
+ -t arvados/jobs:$cwl_runner_version .
ECODE=$?
## 20150526 nico -- *sometimes* dockerhub needs re-login
## even though credentials are already in .dockercfg
docker login -u arvados
- docker_push arvados/jobs:$cwl_runner_version_tag
+ docker_push arvados/jobs:$cwl_runner_version
title "upload arvados images finished (`timer`)"
else
title "upload arvados images SKIPPED because no --upload option set (`timer`)"
COLUMNS=80
. `dirname "$(readlink -f "$0")"`/run-library.sh
-#. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh
read -rd "\000" helpmessage <<EOF
$(basename $0): Build Arvados Python packages and Ruby gems
title "End of $gem_name gem build (`timer`)"
}
+handle_python_package () {
+ # This function assumes the current working directory is the python package directory
+ if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
+ echo "This package doesn't need rebuilding."
+ return
+ fi
+ # Make sure only to use sdist - that's the only format pip can deal with (sigh)
+ python3 setup.py $DASHQ_UNLESS_DEBUG sdist
+}
+
python_wrapper() {
local package_name="$1"; shift
local package_directory="$1"; shift
cwl_runner_version=$(cd sdk/cwl && python3 arvados_version.py)
}
-handle_python_package () {
- # This function assumes the current working directory is the python package directory
- if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
- # This package doesn't need rebuilding.
- return
- fi
- # Make sure only to use sdist - that's the only format pip can deal with (sigh)
- python setup.py $DASHQ_UNLESS_DEBUG sdist
-}
-
handle_ruby_gem() {
local gem_name="$1"; shift
local gem_version="$(nohash_version_from_git)"
done
fi
- # the python-arvados-cwl-runner package comes with cwltool, expose that version
- if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool" ]]; then
- COMMAND_ARR+=("usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool=/usr/bin/")
+ # the python3-arvados-cwl-runner package comes with cwltool, expose that version
+ if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/python-arvados-cwl-runner/bin/cwltool" ]]; then
+ COMMAND_ARR+=("usr/share/$python/dist/python-arvados-cwl-runner/bin/cwltool=/usr/bin/")
fi
COMMAND_ARR+=(".")
"git.arvados.org/arvados.git/lib/cli"
"git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/lib/costanalyzer"
"git.arvados.org/arvados.git/lib/deduplicationreport"
"git.arvados.org/arvados.git/lib/mount"
)
"mount": mount.Command,
"deduplication-report": deduplicationreport.Command,
+ "costanalyzer": costanalyzer.Command,
})
)
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-The master Arvados configuration is stored at @/etc/arvados/config.yml@
+The Arvados configuration is stored at @/etc/arvados/config.yml@
See "Migrating Configuration":config-migration.html for information about migrating from legacy component-specific configuration files.
LoginCluster: clsr1
</pre>
-The @LoginCluster@ configuration redirects all user logins to the LoginCluster, and the LoginCluster will issue API tokens which will be accepted by the federation. Users are activated or deactivated across the entire federation based on their status on the master cluster.
+The @LoginCluster@ configuration redirects all user logins to the LoginCluster, and the LoginCluster will issue API tokens which will be accepted by the federation. Users are activated or deactivated across the entire federation based on their status on the login cluster.
-Note: tokens issued by the master cluster need to be periodically re-validated when used on other clusters in the federation. The period between revalidation attempts is configured with @Login.RemoteTokenRefresh@. The default is 5 minutes. A longer period reduces overhead from validating tokens, but means it may take longer for other clusters to notice when a token has been revoked or a user has changed status (being activated/deactivated, admin flag changed).
+Note: tokens issued by the login cluster need to be periodically re-validated when used on other clusters in the federation. The period between revalidation attempts is configured with @Login.RemoteTokenRefresh@. The default is 5 minutes. A longer period reduces overhead from validating tokens, but means it may take longer for other clusters to notice when a token has been revoked or a user has changed status (being activated/deactivated, admin flag changed).
To migrate users of existing clusters with separate user databases to use a single LoginCluster, use "arv-federation-migrate":merge-remote-account.html .
<div class="releasenotes">
</notextile>
-h2(#master). development master (as of 2020-10-28)
+h2(#main). development main (as of 2020-10-28)
"Upgrading from 2.1.0":#v2_1_0
The "bucket name" is an Arvados collection uuid, portable data hash, or project uuid.
-The bucket name must be encoded as the first path segment of every request. This is what the S3 documentation calls "Path-Style Requests".
+Path-style and virtual host-style requests are supported.
+* A path-style request uses the hostname indicated by @Services.WebDAVDownload.ExternalURL@, with the bucket name in the first path segment: @https://download.example.com/zzzzz-4zz18-asdfgasdfgasdfg/@.
+* A virtual host-style request uses the hostname pattern indicated by @Services.WebDAV.ExternalURL@, with a bucket name in place of the leading @*@: @https://zzzzz-4zz18-asdfgasdfgasdfg.collections.example.com/@.
+
+If you have wildcard DNS, TLS, and routing set up, an S3 client configured with endpoint @collections.example.com@ should work regardless of which request style it uses.
h3. Supported Operations
Keep-web accepts AWS Signature Version 4 (AWS4-HMAC-SHA256) as well as the older V2 AWS signature.
-* If your client uses V4 signatures exclusively: use the Arvados token's UUID part as AccessKey, and its secret part as SecretKey. This is preferred.
-* If your client uses V2 signatures, or a combination of V2 and V4, or the Arvados token UUID is unknown: use the secret part of the Arvados token for both AccessKey and SecretKey.
+If your client uses V4 signatures exclusively _and_ your Arvados token was issued by the same cluster you are connecting to, you can use the Arvados token's UUID part as your S3 Access Key, and its secret part as your S3 Secret Key. This is preferred, where applicable.
+
+Example using cluster @zzzzz@:
+* Arvados token: @v2/zzzzz-gj3su-yyyyyyyyyyyyyyy/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
+* Access Key: @zzzzz-gj3su-yyyyyyyyyyyyyyy@
+* Secret Key: @xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
+
+In all other cases, replace every @/@ character in your Arvados token with @_@, and use the resulting string as both Access Key and Secret Key.
+
+Example using a cluster other than @zzzzz@ _or_ an S3 client that uses V2 signatures:
+* Arvados token: @v2/zzzzz-gj3su-yyyyyyyyyyyyyyy/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
+* Access Key: @v2_zzzzz-gj3su-yyyyyyyyyyyyyyy_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
+* Secret Key: @v2_zzzzz-gj3su-yyyyyyyyyyyyyyy_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
|@=@, @!=@|string, number, timestamp, or null|Equality comparison|@["tail_uuid","=","xyzzy-j7d0g-fffffffffffffff"]@ @["tail_uuid","!=",null]@|
|@<@, @<=@, @>=@, @>@|string, number, or timestamp|Ordering comparison|@["script_version",">","123"]@|
|@like@, @ilike@|string|SQL pattern match. Single character match is @_@ and wildcard is @%@. The @ilike@ operator is case-insensitive|@["script_version","like","d00220fb%"]@|
-|@in@, @not in@|array of strings|Set membership|@["script_version","in",["master","d00220fb38d4b85ca8fc28a8151702a2b9d1dec5"]]@|
+|@in@, @not in@|array of strings|Set membership|@["script_version","in",["main","d00220fb38d4b85ca8fc28a8151702a2b9d1dec5"]]@|
|@is_a@|string|Arvados object type|@["head_uuid","is_a","arvados#collection"]@|
|@exists@|string|Test if a subproperty is present.|@["properties","exists","my_subproperty"]@|
h3(#script_version). Specifying Git versions
-The script_version attribute and arvados_sdk_version runtime constraint are typically given as a branch, tag, or commit hash, but there are many more ways to specify a Git commit. The "specifying revisions" section of the "gitrevisions manual page":http://git-scm.com/docs/gitrevisions.html has a definitive list. Arvados accepts Git versions in any format listed there that names a single commit (not a tree, a blob, or a range of commits). However, some kinds of names can be expected to resolve differently in Arvados than they do in your local repository. For example, <code>HEAD@{1}</code> refers to the local reflog, and @origin/master@ typically refers to a remote branch: neither is likely to work as desired if given as a Git version.
+The script_version attribute and arvados_sdk_version runtime constraint are typically given as a branch, tag, or commit hash, but there are many more ways to specify a Git commit. The "specifying revisions" section of the "gitrevisions manual page":http://git-scm.com/docs/gitrevisions.html has a definitive list. Arvados accepts Git versions in any format listed there that names a single commit (not a tree, a blob, or a range of commits). However, some kinds of names can be expected to resolve differently in Arvados than they do in your local repository. For example, <code>HEAD@{1}</code> refers to the local reflog, and @origin/main@ typically refers to a remote branch: neither is likely to work as desired if given as a Git version.
h3. Runtime constraints
h4. Examples
-Run the script "crunch_scripts/hash.py" in the repository "you" using the "master" commit. Arvados should re-use a previous job if the script_version of the previous job is the same as the current "master" commit. This works irrespective of whether the previous job was submitted using the name "master", a different branch name or tag indicating the same commit, a SHA-1 commit hash, etc.
+Run the script "crunch_scripts/hash.py" in the repository "you" using the "main" commit. Arvados should re-use a previous job if the script_version of the previous job is the same as the current "main" commit. This works irrespective of whether the previous job was submitted using the name "main", a different branch name or tag indicating the same commit, a SHA-1 commit hash, etc.
<notextile><pre>
{
"job": {
"script": "hash.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": "c1bad4b39ca5a924e481008009d94e32+210"
}
}
</pre></notextile>
-Arvados should re-use a previous job if the "script_version" of the previous job is between "earlier_version_tag" and the "master" commit (inclusive), but not the commit indicated by "blacklisted_version_tag". If there are no previous jobs matching these criteria, run the job using the "master" commit.
+Arvados should re-use a previous job if the "script_version" of the previous job is between "earlier_version_tag" and the "main" commit (inclusive), but not the commit indicated by "blacklisted_version_tag". If there are no previous jobs matching these criteria, run the job using the "main" commit.
<notextile><pre>
{
"job": {
"script": "hash.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": "c1bad4b39ca5a924e481008009d94e32+210"
}
"job": {
"script": "hash.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": "c1bad4b39ca5a924e481008009d94e32+210"
}
}
</pre></notextile>
-Run the script "crunch_scripts/monte-carlo.py" in the repository "you/you" using the current "master" commit. Because it is marked as "nondeterministic", this job will not be considered as a suitable candidate for future job submissions that use the "find_or_create" feature.
+Run the script "crunch_scripts/monte-carlo.py" in the repository "you/you" using the current "main" commit. Because it is marked as "nondeterministic", this job will not be considered as a suitable candidate for future job submissions that use the "find_or_create" feature.
<notextile><pre>
{
"job": {
"script": "monte-carlo.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"nondeterministic": true,
"script_parameters": {
"input": "c1bad4b39ca5a924e481008009d94e32+210"
"do_hash": {
"script": "hash.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": {
"required": true,
"filter": {
"script": "0-filter.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": {
"output_of": "do_hash"
"cat_in_the_hat": {
"script": "cat.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": { }
},
"thing1": {
"script": "thing1.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": {
"output_of": "cat_in_the_hat"
"thing2": {
"script": "thing2.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": {
"output_of": "cat_in_the_hat"
"thing1": {
"script": "thing1.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": { }
},
"thing2": {
"script": "thing2.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": { }
},
"cleanup": {
"script": "cleanup.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"mess1": {
"output_of": "thing1"
This page documents setting up and running the "Arvados on Kubernetes":/install/arvados-on-kubernetes.html @Helm@ chart on @Google Kubernetes Engine@ (GKE).
-{% include 'notebox_begin_warning' %}
-This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
-{% include 'notebox_end' %}
-
h2. Prerequisites
h3. Install tooling
h2. Shut down
-{% include 'notebox_begin_warning' %}
-This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and <strong>all data stored on the Arvados cluster will be deleted</strong> when it is shut down. This will be fixed in a future version.
-{% include 'notebox_end' %}
-
<pre>
$ helm del arvados
</pre>
This page documents setting up and running the "Arvados on Kubernetes":/install/arvados-on-kubernetes.html @Helm@ chart on @Minikube@.
-{% include 'notebox_begin_warning' %}
-This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
-{% include 'notebox_end' %}
-
h2. Prerequisites
h3. Install tooling
h2. Shut down
{% include 'notebox_begin_warning' %}
-This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and <strong>all data stored on the Arvados cluster will be deleted</strong> when it is shut down. This will be fixed in a future version.
+This Helm chart uses Kubernetes <i>persistent volumes</i> for the Postgresql and Keepstore data volumes. These volumes will be retained after you delete the Arvados helm chart with the command below. Because those volumes are stored in the local Minikube Kubernetes cluster, if you delete that cluster (e.g. with <i>minikube delete</i>) the Kubernetes persistent volumes will also be deleted.
{% include 'notebox_end' %}
<pre>
Arvados on Kubernetes is implemented as a @Helm 3@ chart.
-{% include 'notebox_begin_warning' %}
-This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
-{% include 'notebox_end' %}
-
h2(#overview). Overview
This Helm chart provides a basic, small Arvados cluster.
h2(#introduction). Introduction
-The Keep-web server provides read/write HTTP (WebDAV) access to files stored in Keep. This makes it easy to access files in Keep from a browser, or mount Keep as a network folder using WebDAV support in various operating systems. It serves public data to unauthenticated clients, and serves private data to clients that supply Arvados API tokens. It can be installed anywhere with access to Keep services, typically behind a web proxy that provides TLS support. See the "godoc page":http://godoc.org/github.com/curoverse/arvados/services/keep-web for more detail.
+The Keep-web server provides read/write access to files stored in Keep using WebDAV and S3 protocols. This makes it easy to access files in Keep from a browser, or mount Keep as a network folder using WebDAV support in various operating systems. It serves public data to unauthenticated clients, and serves private data to clients that supply Arvados API tokens. It can be installed anywhere with access to Keep services, typically behind a web proxy that provides TLS support. See the "godoc page":http://godoc.org/github.com/curoverse/arvados/services/keep-web for more detail.
h2(#dns). Configure DNS
</code></pre>
</notextile>
+This option is preferred if you plan to access Keep using third-party S3 client software, because it accommodates S3 virtual host-style requests and path-style requests without any special client configuration.
+
h4. Under the main domain
Alternately, they can go under the main domain by including @--@:
# "Install Saltstack":#saltstack
# "Single host install using the provision.sh script":#single_host
-# "Local testing Arvados in a Vagrant box":#vagrant
# "DNS configuration":#final_steps
# "Initial user and login":#initial_user
+# "Test the installed cluster running a simple workflow":#test_install
h2(#saltstack). Install Saltstack
* User: 'admin'
* Password: 'password'
* Email: 'admin@arva2.arv.local'
+
+h2(#test_install). Test the installed cluster running a simple workflow
+
+The @provision.sh@ script saves a simple example test workflow in the @/tmp/cluster_tests@. If you want to run it, just change to that directory and run:
+
+<notextile>
+<pre><code>cd /tmp/cluster_tests
+./run-test.sh
+</code></pre>
+</notextile>
+
+It will create a test user, upload a small workflow and run it. If everything goes OK, the output should similar to this (some output was shortened for clarity):
+
+<notextile>
+<pre><code>Creating Arvados Standard Docker Images project
+Arvados project uuid is 'arva2-j7d0g-0prd8cjlk6kfl7y'
+{
+ ...
+ "uuid":"arva2-o0j2j-n4zu4cak5iifq2a",
+ "owner_uuid":"arva2-tpzed-000000000000000",
+ ...
+}
+Uploading arvados/jobs' docker image to the project
+2.1.1: Pulling from arvados/jobs
+8559a31e96f4: Pulling fs layer
+...
+Status: Downloaded newer image for arvados/jobs:2.1.1
+docker.io/arvados/jobs:2.1.1
+2020-11-23 21:43:39 arvados.arv_put[32678] INFO: Creating new cache file at /home/vagrant/.cache/arvados/arv-put/c59256eda1829281424c80f588c7cc4d
+2020-11-23 21:43:46 arvados.arv_put[32678] INFO: Collection saved as 'Docker image arvados jobs:2.1.1 sha256:0dd50'
+arva2-4zz18-1u5pvbld7cvxuy2
+Creating initial user ('admin')
+Setting up user ('admin')
+{
+ "items":[
+ {
+ ...
+ "owner_uuid":"arva2-tpzed-000000000000000",
+ ...
+ "uuid":"arva2-o0j2j-1ownrdne0ok9iox"
+ },
+ {
+ ...
+ "owner_uuid":"arva2-tpzed-000000000000000",
+ ...
+ "uuid":"arva2-o0j2j-1zbeyhcwxc1tvb7"
+ },
+ {
+ ...
+ "email":"admin@arva2.arv.local",
+ ...
+ "owner_uuid":"arva2-tpzed-000000000000000",
+ ...
+ "username":"admin",
+ "uuid":"arva2-tpzed-3wrm93zmzpshrq2",
+ ...
+ }
+ ],
+ "kind":"arvados#HashList"
+}
+Activating user 'admin'
+{
+ ...
+ "email":"admin@arva2.arv.local",
+ ...
+ "username":"admin",
+ "uuid":"arva2-tpzed-3wrm93zmzpshrq2",
+ ...
+}
+Running test CWL workflow
+INFO /usr/bin/cwl-runner 2.1.1, arvados-python-client 2.1.1, cwltool 3.0.20200807132242
+INFO Resolved 'hasher-workflow.cwl' to 'file:///tmp/cluster_tests/hasher-workflow.cwl'
+...
+INFO Using cluster arva2 (https://arva2.arv.local:8443/)
+INFO Upload local files: "test.txt"
+INFO Uploaded to ea34d971b71d5536b4f6b7d6c69dc7f6+50 (arva2-4zz18-c8uvwqdry4r8jao)
+INFO Using collection cache size 256 MiB
+INFO [container hasher-workflow.cwl] submitted container_request arva2-xvhdp-v1bkywd58gyocwm
+INFO [container hasher-workflow.cwl] arva2-xvhdp-v1bkywd58gyocwm is Final
+INFO Overall process status is success
+INFO Final output collection d6c69a88147dde9d52a418d50ef788df+123
+{
+ "hasher_out": {
+ "basename": "hasher3.md5sum.txt",
+ "class": "File",
+ "location": "keep:d6c69a88147dde9d52a418d50ef788df+123/hasher3.md5sum.txt",
+ "size": 95
+ }
+}
+INFO Final process status is success
+</code></pre>
+</notextile>
# "Vagrant":#vagrant
# "DNS configuration":#final_steps
# "Initial user and login":#initial_user
+# "Test the installed cluster running a simple workflow":#test_install
h2(#vagrant). Vagrant
* User: 'admin'
* Password: 'password'
* Email: 'admin@arva2.arv.local'
+
+h2(#test_install). Test the installed cluster running a simple workflow
+
+As documented in the <a href="{{ site.baseurl }}/install/salt-single-host.html">Single Host installation</a> page, You can run a test workflow to verify the installation finished correctly. To do so, you can follow these steps:
+
+<notextile>
+<pre><code>vagrant ssh</code></pre>
+</notextile>
+
+and once in the instance:
+
+<notextile>
+<pre><code>cd /tmp/cluster_tests
+./run-test.sh
+</code></pre>
+</notextile>
** Contains about resource consumption (RAM, cpu, disk, network) on the node while it was running
This is different from the log crunchstat.txt because it includes resource consumption of Arvados components that run on the node outside the container such as crunch-run and other processes related to the Keep file system.
-For the highest level logs, the logs are tracking the container that ran the @arvados-cwl-runner@ process which you can think of as the “mastermind” behind tracking which parts of the CWL workflow need to be run when, which have been run already, what order they need to be run, which can be run simultaneously, and so forth and then sending out the related container requests. Each step then has their own logs related to containers running a CWL step of the workflow including a log of standard error that contains the standard error of the code run in that CWL step. Those logs can be found by expanding the steps and clicking on the link to the log collection.
+For the highest level logs, the logs are tracking the container that ran the @arvados-cwl-runner@ process which you can think of as the “workflow runner”. It tracks which parts of the CWL workflow need to be run when, which have been run already, what order they need to be run, which can be run simultaneously, and so forth and then creates the necessary container requests. Each step has its own logs related to containers running a CWL step of the workflow including a log of standard error that contains the standard error of the code run in that CWL step. Those logs can be found by expanding the steps and clicking on the link to the log collection.
-Let’s take a peek at a few of these logs to get you more familiar with them. First, we can look at the @stderr.txt@ of the highest level process. Again recall this should be of the “mastermind” @arvados-cwl-runner@ process. You can click on the log to download it to your local machine, and when you look at the contents - you should see something like the following...
+Let’s take a peek at a few of these logs to get you more familiar with them. First, we can look at the @stderr.txt@ of the highest level process. Again recall this should be of the “workflow runner” @arvados-cwl-runner@ process. You can click on the log to download it to your local machine, and when you look at the contents - you should see something like the following...
<pre><code>2020-06-22T20:30:04.737703197Z INFO /usr/bin/arvados-cwl-runner 2.0.3, arvados-python-client 2.0.3, cwltool 1.0.20190831161204
2020-06-22T20:30:04.743250012Z INFO Resolved '/var/lib/cwl/workflow.json#main' to 'file:///var/lib/cwl/workflow.json#main'
if err != nil {
return fmt.Errorf("user.Lookup(\"postgres\"): %s", err)
}
- postgresUid, err := strconv.Atoi(postgresUser.Uid)
+ postgresUID, err := strconv.Atoi(postgresUser.Uid)
if err != nil {
return fmt.Errorf("user.Lookup(\"postgres\"): non-numeric uid?: %q", postgresUser.Uid)
}
if err != nil {
return err
}
- err = os.Chown(datadir, postgresUid, 0)
+ err = os.Chown(datadir, postgresUID, 0)
if err != nil {
return err
}
crString, ok := request["container_request"].(string)
if ok {
- var crJson map[string]interface{}
- err := json.Unmarshal([]byte(crString), &crJson)
+ var crJSON map[string]interface{}
+ err := json.Unmarshal([]byte(crString), &crJSON)
if err != nil {
httpserver.Error(w, err.Error(), http.StatusBadRequest)
return true
}
- request["container_request"] = crJson
+ request["container_request"] = crJSON
}
containerRequest, ok := request["container_request"].(map[string]interface{})
if options.BypassFederation {
return conn.local.UserUpdate(ctx, options)
}
- return conn.chooseBackend(options.UUID).UserUpdate(ctx, options)
+ resp, err := conn.chooseBackend(options.UUID).UserUpdate(ctx, options)
+ if err != nil {
+ return resp, err
+ }
+ if !strings.HasPrefix(options.UUID, conn.cluster.ClusterID) {
+ // Copy the updated user record to the local cluster
+ err = conn.batchUpdateUsers(ctx, arvados.ListOptions{}, []arvados.User{resp})
+ if err != nil {
+ return arvados.User{}, err
+ }
+ }
+ return resp, err
}
func (conn *Conn) UserUpdateUUID(ctx context.Context, options arvados.UpdateUUIDOptions) (arvados.User, error) {
"git.arvados.org/arvados.git/sdk/go/health"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"github.com/jmoiron/sqlx"
+ // sqlx needs lib/pq to talk to PostgreSQL
_ "github.com/lib/pq"
)
"bytes"
"context"
"encoding/json"
+ "fmt"
"io"
"io/ioutil"
"math"
"net/http"
"net/url"
"os"
+ "os/exec"
"path/filepath"
+ "strconv"
+ "strings"
"git.arvados.org/arvados.git/lib/boot"
"git.arvados.org/arvados.git/lib/config"
c.Check(coll.PortableDataHash, check.Equals, pdh)
}
+func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) {
+ if _, err := exec.LookPath("s3cmd"); err != nil {
+ c.Skip("s3cmd not in PATH")
+ return
+ }
+
+ testText := "IntegrationSuite.TestS3WithFederatedToken"
+
+ conn1 := s.conn("z1111")
+ rootctx1, _, _ := s.rootClients("z1111")
+ userctx1, ac1, _, _ := s.userClients(rootctx1, c, conn1, "z1111", true)
+ conn3 := s.conn("z3333")
+
+ createColl := func(clusterID string) arvados.Collection {
+ _, ac, kc := s.clientsWithToken(clusterID, ac1.AuthToken)
+ var coll arvados.Collection
+ fs, err := coll.FileSystem(ac, kc)
+ c.Assert(err, check.IsNil)
+ f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = io.WriteString(f, testText)
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+ mtxt, err := fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ coll, err = s.conn(clusterID).CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
+ "manifest_text": mtxt,
+ }})
+ c.Assert(err, check.IsNil)
+ return coll
+ }
+
+ for _, trial := range []struct {
+ clusterID string // create the collection on this cluster (then use z3333 to access it)
+ token string
+ }{
+ // Try the hardest test first: z3333 hasn't seen
+ // z1111's token yet, and we're just passing the
+ // opaque secret part, so z3333 has to guess that it
+ // belongs to z1111.
+ {"z1111", strings.Split(ac1.AuthToken, "/")[2]},
+ {"z3333", strings.Split(ac1.AuthToken, "/")[2]},
+ {"z1111", strings.Replace(ac1.AuthToken, "/", "_", -1)},
+ {"z3333", strings.Replace(ac1.AuthToken, "/", "_", -1)},
+ } {
+ c.Logf("================ %v", trial)
+ coll := createColl(trial.clusterID)
+
+ cfgjson, err := conn3.ConfigGet(userctx1)
+ c.Assert(err, check.IsNil)
+ var cluster arvados.Cluster
+ err = json.Unmarshal(cfgjson, &cluster)
+ c.Assert(err, check.IsNil)
+
+ c.Logf("TokenV2 is %s", ac1.AuthToken)
+ host := cluster.Services.WebDAV.ExternalURL.Host
+ s3args := []string{
+ "--ssl", "--no-check-certificate",
+ "--host=" + host, "--host-bucket=" + host,
+ "--access_key=" + trial.token, "--secret_key=" + trial.token,
+ }
+ buf, err := exec.Command("s3cmd", append(s3args, "ls", "s3://"+coll.UUID)...).CombinedOutput()
+ c.Check(err, check.IsNil)
+ c.Check(string(buf), check.Matches, `.* `+fmt.Sprintf("%d", len(testText))+` +s3://`+coll.UUID+`/test.txt\n`)
+
+ buf, err = exec.Command("s3cmd", append(s3args, "get", "s3://"+coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput()
+ // Command fails because we don't return Etag header.
+ // c.Check(err, check.IsNil)
+ flen := strconv.Itoa(len(testText))
+ c.Check(string(buf), check.Matches, `(?ms).*`+flen+` of `+flen+`.*`)
+ }
+}
+
func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) {
conn1 := s.conn("z1111")
conn3 := s.conn("z3333")
var _ = check.Suite(&RPCSuite{})
-const contextKeyTestTokens = "testTokens"
+type key int
+
+const (
+ contextKeyTestTokens key = iota
+)
type RPCSuite struct {
log logrus.FieldLogger
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package costanalyzer
+
+import (
+ "io"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/sirupsen/logrus"
+)
+
+var Command command
+
+type command struct{}
+
+type NoPrefixFormatter struct{}
+
+func (f *NoPrefixFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+ return []byte(entry.Message), nil
+}
+
+// RunCommand implements the subcommand "costanalyzer <collection> <collection> ..."
+func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ var err error
+ logger := ctxlog.New(stderr, "text", "info")
+ defer func() {
+ if err != nil {
+ logger.Error("\n" + err.Error() + "\n")
+ }
+ }()
+
+ logger.SetFormatter(new(NoPrefixFormatter))
+
+ loader := config.NewLoader(stdin, logger)
+ loader.SkipLegacy = true
+
+ exitcode, err := costanalyzer(prog, args, loader, logger, stdout, stderr)
+
+ return exitcode
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package costanalyzer
+
+import (
+ "encoding/json"
+ "errors"
+ "flag"
+ "fmt"
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "os"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/sirupsen/logrus"
+)
+
+type nodeInfo struct {
+ // Legacy (records created by Arvados Node Manager with Arvados <= 1.4.3)
+ Properties struct {
+ CloudNode struct {
+ Price float64
+ Size string
+ } `json:"cloud_node"`
+ }
+ // Modern
+ ProviderType string
+ Price float64
+}
+
+type arrayFlags []string
+
+func (i *arrayFlags) String() string {
+ return ""
+}
+
+func (i *arrayFlags) Set(value string) error {
+ *i = append(*i, value)
+ return nil
+}
+
+func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string, cache bool, err error) {
+ flags := flag.NewFlagSet("", flag.ContinueOnError)
+ flags.SetOutput(stderr)
+ flags.Usage = func() {
+ fmt.Fprintf(flags.Output(), `
+Usage:
+ %s [options ...]
+
+ This program analyzes the cost of Arvados container requests. For each uuid
+ supplied, it creates a CSV report that lists all the containers used to
+ fulfill the container request, together with the machine type and cost of
+ each container.
+
+ When supplied with the uuid of a container request, it will calculate the
+ cost of that container request and all its children. When suplied with a
+ project uuid or when supplied with multiple container request uuids, it will
+ create a CSV report for each supplied uuid, as well as a CSV file with
+ aggregate cost accounting for all supplied uuids. The aggregate cost report
+ takes container reuse into account: if a container was reused between several
+ container requests, its cost will only be counted once.
+
+ To get the node costs, the progam queries the Arvados API for current cost
+ data for each node type used. This means that the reported cost always
+ reflects the cost data as currently defined in the Arvados API configuration
+ file.
+
+ Caveats:
+ - the Arvados API configuration cost data may be out of sync with the cloud
+ provider.
+ - when generating reports for older container requests, the cost data in the
+ Arvados API configuration file may have changed since the container request
+ was fulfilled. This program uses the cost data stored at the time of the
+ execution of the container, stored in the 'node.json' file in its log
+ collection.
+
+ In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
+ ARVADOS_API_TOKEN environment variables must be set.
+
+Options:
+`, prog)
+ flags.PrintDefaults()
+ }
+ loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)")
+ flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports (required)")
+ flags.Var(&uuids, "uuid", "Toplevel `project or container request` uuid. May be specified more than once. (required)")
+ flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects")
+ err = flags.Parse(args)
+ if err == flag.ErrHelp {
+ err = nil
+ exitCode = 1
+ return
+ } else if err != nil {
+ exitCode = 2
+ return
+ }
+
+ if len(uuids) < 1 {
+ flags.Usage()
+ err = fmt.Errorf("Error: no uuid(s) provided")
+ exitCode = 2
+ return
+ }
+
+ if resultsDir == "" {
+ flags.Usage()
+ err = fmt.Errorf("Error: output directory must be specified")
+ exitCode = 2
+ return
+ }
+
+ lvl, err := logrus.ParseLevel(*loglevel)
+ if err != nil {
+ exitCode = 2
+ return
+ }
+ logger.SetLevel(lvl)
+ if !cache {
+ logger.Debug("Caching disabled\n")
+ }
+ return
+}
+
+func ensureDirectory(logger *logrus.Logger, dir string) (err error) {
+ statData, err := os.Stat(dir)
+ if os.IsNotExist(err) {
+ err = os.MkdirAll(dir, 0700)
+ if err != nil {
+ return fmt.Errorf("error creating directory %s: %s", dir, err.Error())
+ }
+ } else {
+ if !statData.IsDir() {
+ return fmt.Errorf("the path %s is not a directory", dir)
+ }
+ }
+ return
+}
+
+func addContainerLine(logger *logrus.Logger, node nodeInfo, cr arvados.ContainerRequest, container arvados.Container) (csv string, cost float64) {
+ csv = cr.UUID + ","
+ csv += cr.Name + ","
+ csv += container.UUID + ","
+ csv += string(container.State) + ","
+ if container.StartedAt != nil {
+ csv += container.StartedAt.String() + ","
+ } else {
+ csv += ","
+ }
+
+ var delta time.Duration
+ if container.FinishedAt != nil {
+ csv += container.FinishedAt.String() + ","
+ delta = container.FinishedAt.Sub(*container.StartedAt)
+ csv += strconv.FormatFloat(delta.Seconds(), 'f', 0, 64) + ","
+ } else {
+ csv += ",,"
+ }
+ var price float64
+ var size string
+ if node.Properties.CloudNode.Price != 0 {
+ price = node.Properties.CloudNode.Price
+ size = node.Properties.CloudNode.Size
+ } else {
+ price = node.Price
+ size = node.ProviderType
+ }
+ cost = delta.Seconds() / 3600 * price
+ csv += size + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"
+ return
+}
+
+func loadCachedObject(logger *logrus.Logger, file string, uuid string, object interface{}) (reload bool) {
+ reload = true
+ if strings.Contains(uuid, "-j7d0g-") {
+ // We do not cache projects, they have no final state
+ return
+ }
+ // See if we have a cached copy of this object
+ _, err := os.Stat(file)
+ if err != nil {
+ return
+ }
+ data, err := ioutil.ReadFile(file)
+ if err != nil {
+ logger.Errorf("error reading %q: %s", file, err)
+ return
+ }
+ err = json.Unmarshal(data, &object)
+ if err != nil {
+ logger.Errorf("failed to unmarshal json: %s: %s", data, err)
+ return
+ }
+
+ // See if it is in a final state, if that makes sense
+ switch v := object.(type) {
+ case *arvados.ContainerRequest:
+ if v.State == arvados.ContainerRequestStateFinal {
+ reload = false
+ logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file)
+ }
+ case *arvados.Container:
+ if v.State == arvados.ContainerStateComplete || v.State == arvados.ContainerStateCancelled {
+ reload = false
+ logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file)
+ }
+ }
+ return
+}
+
+// Load an Arvados object.
+func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid string, cache bool, object interface{}) (err error) {
+ file := uuid + ".json"
+
+ var reload bool
+ var cacheDir string
+
+ if !cache {
+ reload = true
+ } else {
+ homeDir, err := os.UserHomeDir()
+ if err != nil {
+ reload = true
+ logger.Info("Unable to determine current user home directory, not using cache")
+ } else {
+ cacheDir = homeDir + "/.cache/arvados/costanalyzer/"
+ err = ensureDirectory(logger, cacheDir)
+ if err != nil {
+ reload = true
+ logger.Infof("Unable to create cache directory at %s, not using cache: %s", cacheDir, err.Error())
+ } else {
+ reload = loadCachedObject(logger, cacheDir+file, uuid, object)
+ }
+ }
+ }
+ if !reload {
+ return
+ }
+
+ if strings.Contains(uuid, "-j7d0g-") {
+ err = ac.RequestAndDecode(&object, "GET", "arvados/v1/groups/"+uuid, nil, nil)
+ } else if strings.Contains(uuid, "-xvhdp-") {
+ err = ac.RequestAndDecode(&object, "GET", "arvados/v1/container_requests/"+uuid, nil, nil)
+ } else if strings.Contains(uuid, "-dz642-") {
+ err = ac.RequestAndDecode(&object, "GET", "arvados/v1/containers/"+uuid, nil, nil)
+ } else {
+ err = fmt.Errorf("unsupported object type with UUID %q:\n %s", uuid, err)
+ return
+ }
+ if err != nil {
+ err = fmt.Errorf("error loading object with UUID %q:\n %s", uuid, err)
+ return
+ }
+ encoded, err := json.MarshalIndent(object, "", " ")
+ if err != nil {
+ err = fmt.Errorf("error marshaling object with UUID %q:\n %s", uuid, err)
+ return
+ }
+ if cacheDir != "" {
+ err = ioutil.WriteFile(cacheDir+file, encoded, 0644)
+ if err != nil {
+ err = fmt.Errorf("error writing file %s:\n %s", file, err)
+ return
+ }
+ }
+ return
+}
+
+func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, cr arvados.ContainerRequest) (node nodeInfo, err error) {
+ if cr.LogUUID == "" {
+ err = errors.New("No log collection")
+ return
+ }
+
+ var collection arvados.Collection
+ err = ac.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil)
+ if err != nil {
+ err = fmt.Errorf("error getting collection: %s", err)
+ return
+ }
+
+ var fs arvados.CollectionFileSystem
+ fs, err = collection.FileSystem(ac, kc)
+ if err != nil {
+ err = fmt.Errorf("error opening collection as filesystem: %s", err)
+ return
+ }
+ var f http.File
+ f, err = fs.Open("node.json")
+ if err != nil {
+ err = fmt.Errorf("error opening file 'node.json' in collection %s: %s", cr.LogUUID, err)
+ return
+ }
+
+ err = json.NewDecoder(f).Decode(&node)
+ if err != nil {
+ err = fmt.Errorf("error reading file 'node.json' in collection %s: %s", cr.LogUUID, err)
+ return
+ }
+ return
+}
+
+func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) {
+
+ cost = make(map[string]float64)
+
+ var project arvados.Group
+ err = loadObject(logger, ac, uuid, uuid, cache, &project)
+ if err != nil {
+ return nil, fmt.Errorf("error loading object %s: %s", uuid, err.Error())
+ }
+
+ var childCrs map[string]interface{}
+ filterset := []arvados.Filter{
+ {
+ Attr: "owner_uuid",
+ Operator: "=",
+ Operand: project.UUID,
+ },
+ {
+ Attr: "requesting_container_uuid",
+ Operator: "=",
+ Operand: nil,
+ },
+ }
+ err = ac.RequestAndDecode(&childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{
+ "filters": filterset,
+ "limit": 10000,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
+ }
+ if value, ok := childCrs["items"]; ok {
+ logger.Infof("Collecting top level container requests in project %s\n", uuid)
+ items := value.([]interface{})
+ for _, item := range items {
+ itemMap := item.(map[string]interface{})
+ crCsv, err := generateCrCsv(logger, itemMap["uuid"].(string), arv, ac, kc, resultsDir, cache)
+ if err != nil {
+ return nil, fmt.Errorf("error generating container_request CSV: %s", err.Error())
+ }
+ for k, v := range crCsv {
+ cost[k] = v
+ }
+ }
+ } else {
+ logger.Infof("No top level container requests found in project %s\n", uuid)
+ }
+ return
+}
+
+func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) {
+
+ cost = make(map[string]float64)
+
+ csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n"
+ var tmpCsv string
+ var tmpTotalCost float64
+ var totalCost float64
+
+ // This is a container request, find the container
+ var cr arvados.ContainerRequest
+ err = loadObject(logger, ac, uuid, uuid, cache, &cr)
+ if err != nil {
+ return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
+ }
+ var container arvados.Container
+ err = loadObject(logger, ac, uuid, cr.ContainerUUID, cache, &container)
+ if err != nil {
+ return nil, fmt.Errorf("error loading container object %s: %s", cr.ContainerUUID, err)
+ }
+
+ topNode, err := getNode(arv, ac, kc, cr)
+ if err != nil {
+ return nil, fmt.Errorf("error getting node %s: %s", cr.UUID, err)
+ }
+ tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container)
+ csv += tmpCsv
+ totalCost += tmpTotalCost
+ cost[container.UUID] = totalCost
+
+ // Find all container requests that have the container we found above as requesting_container_uuid
+ var childCrs arvados.ContainerRequestList
+ filterset := []arvados.Filter{
+ {
+ Attr: "requesting_container_uuid",
+ Operator: "=",
+ Operand: container.UUID,
+ }}
+ err = ac.RequestAndDecode(&childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{
+ "filters": filterset,
+ "limit": 10000,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
+ }
+ logger.Infof("Collecting child containers for container request %s", uuid)
+ for _, cr2 := range childCrs.Items {
+ logger.Info(".")
+ node, err := getNode(arv, ac, kc, cr2)
+ if err != nil {
+ return nil, fmt.Errorf("error getting node %s: %s", cr2.UUID, err)
+ }
+ logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n")
+ var c2 arvados.Container
+ err = loadObject(logger, ac, uuid, cr2.ContainerUUID, cache, &c2)
+ if err != nil {
+ return nil, fmt.Errorf("error loading object %s: %s", cr2.ContainerUUID, err)
+ }
+ tmpCsv, tmpTotalCost = addContainerLine(logger, node, cr2, c2)
+ cost[cr2.ContainerUUID] = tmpTotalCost
+ csv += tmpCsv
+ totalCost += tmpTotalCost
+ }
+ logger.Info(" done\n")
+
+ csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n"
+
+ // Write the resulting CSV file
+ fName := resultsDir + "/" + uuid + ".csv"
+ err = ioutil.WriteFile(fName, []byte(csv), 0644)
+ if err != nil {
+ return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
+ }
+ logger.Infof("\nUUID report in %s\n\n", fName)
+
+ return
+}
+
+func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) {
+ exitcode, uuids, resultsDir, cache, err := parseFlags(prog, args, loader, logger, stderr)
+ if exitcode != 0 {
+ return
+ }
+ err = ensureDirectory(logger, resultsDir)
+ if err != nil {
+ exitcode = 3
+ return
+ }
+
+ // Arvados Client setup
+ arv, err := arvadosclient.MakeArvadosClient()
+ if err != nil {
+ err = fmt.Errorf("error creating Arvados object: %s", err)
+ exitcode = 1
+ return
+ }
+ kc, err := keepclient.MakeKeepClient(arv)
+ if err != nil {
+ err = fmt.Errorf("error creating Keep object: %s", err)
+ exitcode = 1
+ return
+ }
+
+ ac := arvados.NewClientFromEnv()
+
+ cost := make(map[string]float64)
+ for _, uuid := range uuids {
+ if strings.Contains(uuid, "-j7d0g-") {
+ // This is a project (group)
+ cost, err = handleProject(logger, uuid, arv, ac, kc, resultsDir, cache)
+ if err != nil {
+ exitcode = 1
+ return
+ }
+ for k, v := range cost {
+ cost[k] = v
+ }
+ } else if strings.Contains(uuid, "-xvhdp-") {
+ // This is a container request
+ var crCsv map[string]float64
+ crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
+ if err != nil {
+ err = fmt.Errorf("Error generating container_request CSV for uuid %s: %s", uuid, err.Error())
+ exitcode = 2
+ return
+ }
+ for k, v := range crCsv {
+ cost[k] = v
+ }
+ } else if strings.Contains(uuid, "-tpzed-") {
+ // This is a user. The "Home" project for a user is not a real project.
+ // It is identified by the user uuid. As such, cost analysis for the
+ // "Home" project is not supported by this program. Skip this uuid, but
+ // keep going.
+ logger.Errorf("Cost analysis is not supported for the 'Home' project: %s", uuid)
+ }
+ }
+
+ if len(cost) == 0 {
+ logger.Info("Nothing to do!\n")
+ return
+ }
+
+ var csv string
+
+ csv = "# Aggregate cost accounting for uuids:\n"
+ for _, uuid := range uuids {
+ csv += "# " + uuid + "\n"
+ }
+
+ var total float64
+ for k, v := range cost {
+ csv += k + "," + strconv.FormatFloat(v, 'f', 8, 64) + "\n"
+ total += v
+ }
+
+ csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n"
+
+ // Write the resulting CSV file
+ aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
+ err = ioutil.WriteFile(aFile, []byte(csv), 0644)
+ if err != nil {
+ err = fmt.Errorf("Error writing file with path %s: %s", aFile, err.Error())
+ exitcode = 1
+ return
+ }
+ logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile)
+ return
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package costanalyzer
+
+import (
+ "bytes"
+ "io"
+ "io/ioutil"
+ "os"
+ "regexp"
+ "testing"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
+ "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&Suite{})
+
+type Suite struct{}
+
+func (s *Suite) TearDownSuite(c *check.C) {
+ // Undo any changes/additions to the database so they don't affect subsequent tests.
+ arvadostest.ResetEnv()
+}
+
+func (s *Suite) SetUpSuite(c *check.C) {
+ arvadostest.StartAPI()
+ arvadostest.StartKeep(2, true)
+
+ // Get the various arvados, arvadosclient, and keep client objects
+ ac := arvados.NewClientFromEnv()
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.Equals, nil)
+ arv.ApiToken = arvadostest.ActiveToken
+ kc, err := keepclient.MakeKeepClient(arv)
+ c.Assert(err, check.Equals, nil)
+
+ standardE4sV3JSON := `{
+ "Name": "Standard_E4s_v3",
+ "ProviderType": "Standard_E4s_v3",
+ "VCPUs": 4,
+ "RAM": 34359738368,
+ "Scratch": 64000000000,
+ "IncludedScratch": 64000000000,
+ "AddedScratch": 0,
+ "Price": 0.292,
+ "Preemptible": false
+}`
+ standardD32sV3JSON := `{
+ "Name": "Standard_D32s_v3",
+ "ProviderType": "Standard_D32s_v3",
+ "VCPUs": 32,
+ "RAM": 137438953472,
+ "Scratch": 256000000000,
+ "IncludedScratch": 256000000000,
+ "AddedScratch": 0,
+ "Price": 1.76,
+ "Preemptible": false
+}`
+
+ standardA1V2JSON := `{
+ "Name": "a1v2",
+ "ProviderType": "Standard_A1_v2",
+ "VCPUs": 1,
+ "RAM": 2147483648,
+ "Scratch": 10000000000,
+ "IncludedScratch": 10000000000,
+ "AddedScratch": 0,
+ "Price": 0.043,
+ "Preemptible": false
+}`
+
+ standardA2V2JSON := `{
+ "Name": "a2v2",
+ "ProviderType": "Standard_A2_v2",
+ "VCPUs": 2,
+ "RAM": 4294967296,
+ "Scratch": 20000000000,
+ "IncludedScratch": 20000000000,
+ "AddedScratch": 0,
+ "Price": 0.091,
+ "Preemptible": false
+}`
+
+ legacyD1V2JSON := `{
+ "properties": {
+ "cloud_node": {
+ "price": 0.073001,
+ "size": "Standard_D1_v2"
+ },
+ "total_cpu_cores": 1,
+ "total_ram_mb": 3418,
+ "total_scratch_mb": 51170
+ }
+}`
+
+ // Our fixtures do not actually contain file contents. Populate the log collections we're going to use with the node.json file
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID, arvadostest.LogCollectionUUID, standardE4sV3JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID2, arvadostest.LogCollectionUUID2, standardD32sV3JSON)
+
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsContainerRequest1UUID, arvadostest.DiagnosticsContainerRequest1LogCollectionUUID, standardA1V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsContainerRequest2UUID, arvadostest.DiagnosticsContainerRequest2LogCollectionUUID, standardA1V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher1ContainerRequestUUID, arvadostest.Hasher1LogCollectionUUID, standardA1V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher2ContainerRequestUUID, arvadostest.Hasher2LogCollectionUUID, standardA2V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher3ContainerRequestUUID, arvadostest.Hasher3LogCollectionUUID, legacyD1V2JSON)
+}
+
+func createNodeJSON(c *check.C, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, crUUID string, logUUID string, nodeJSON string) {
+ // Get the CR
+ var cr arvados.ContainerRequest
+ err := ac.RequestAndDecode(&cr, "GET", "arvados/v1/container_requests/"+crUUID, nil, nil)
+ c.Assert(err, check.Equals, nil)
+ c.Assert(cr.LogUUID, check.Equals, logUUID)
+
+ // Get the log collection
+ var coll arvados.Collection
+ err = ac.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil)
+ c.Assert(err, check.IsNil)
+
+ // Create a node.json file -- the fixture doesn't actually contain the contents of the collection.
+ fs, err := coll.FileSystem(ac, kc)
+ c.Assert(err, check.IsNil)
+ f, err := fs.OpenFile("node.json", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = io.WriteString(f, nodeJSON)
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+
+ // Flush the data to Keep
+ mtxt, err := fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ c.Assert(mtxt, check.NotNil)
+
+ // Update collection record
+ err = ac.RequestAndDecode(&coll, "PUT", "arvados/v1/collections/"+cr.LogUUID, nil, map[string]interface{}{
+ "collection": map[string]interface{}{
+ "manifest_text": mtxt,
+ },
+ })
+ c.Assert(err, check.IsNil)
+}
+
+func (*Suite) TestUsage(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-help", "-log-level=debug"}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 1)
+ c.Check(stdout.String(), check.Equals, "")
+ c.Check(stderr.String(), check.Matches, `(?ms).*Usage:.*`)
+}
+
+func (*Suite) TestContainerRequestUUID(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ // Run costanalyzer with 1 container request uuid
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889")
+}
+
+func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ // Run costanalyzer with 2 container request uuids
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID, "-uuid", arvadostest.CompletedContainerRequestUUID2, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+
+ uuidReport2, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
+
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000")
+ stdout.Truncate(0)
+ stderr.Truncate(0)
+
+ // Now move both container requests into an existing project, and then re-run
+ // the analysis with the project uuid. The results should be identical.
+ ac := arvados.NewClientFromEnv()
+ var cr arvados.ContainerRequest
+ err = ac.RequestAndDecode(&cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID, nil, map[string]interface{}{
+ "container_request": map[string]interface{}{
+ "owner_uuid": arvadostest.AProjectUUID,
+ },
+ })
+ c.Assert(err, check.IsNil)
+ err = ac.RequestAndDecode(&cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID2, nil, map[string]interface{}{
+ "container_request": map[string]interface{}{
+ "owner_uuid": arvadostest.AProjectUUID,
+ },
+ })
+ c.Assert(err, check.IsNil)
+
+ // Run costanalyzer with the project uuid
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.AProjectUUID, "-cache=false", "-log-level", "debug", "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err = ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+
+ uuidReport2, err = ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
+
+ re = regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches = re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err = ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000")
+}
+
+func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ // Run costanalyzer with 2 container request uuids
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID, "-uuid", arvadostest.CompletedDiagnosticsContainerRequest2UUID, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00916192")
+
+ uuidReport2, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00588088")
+
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,0.01492030")
+}
var pi procinfo
err = json.NewDecoder(f).Decode(&pi)
if err != nil {
- return fmt.Errorf("decode %s: %s\n", path, err)
+ return fmt.Errorf("decode %s: %s", path, err)
}
if pi.UUID != uuid || pi.PID == 0 {
}
for bind := range runner.SecretMounts {
if _, ok := runner.Container.Mounts[bind]; ok {
- return fmt.Errorf("Secret mount %q conflicts with regular mount", bind)
+ return fmt.Errorf("secret mount %q conflicts with regular mount", bind)
}
if runner.SecretMounts[bind].Kind != "json" &&
runner.SecretMounts[bind].Kind != "text" {
- return fmt.Errorf("Secret mount %q type is %q but only 'json' and 'text' are permitted.",
+ return fmt.Errorf("secret mount %q type is %q but only 'json' and 'text' are permitted",
bind, runner.SecretMounts[bind].Kind)
}
binds = append(binds, bind)
if bind == "stdout" || bind == "stderr" {
// Is it a "file" mount kind?
if mnt.Kind != "file" {
- return fmt.Errorf("Unsupported mount kind '%s' for %s. Only 'file' is supported.", mnt.Kind, bind)
+ return fmt.Errorf("unsupported mount kind '%s' for %s: only 'file' is supported", mnt.Kind, bind)
}
// Does path start with OutputPath?
if bind == "stdin" {
// Is it a "collection" mount kind?
if mnt.Kind != "collection" && mnt.Kind != "json" {
- return fmt.Errorf("Unsupported mount kind '%s' for stdin. Only 'collection' or 'json' are supported.", mnt.Kind)
+ return fmt.Errorf("unsupported mount kind '%s' for stdin: only 'collection' and 'json' are supported", mnt.Kind)
}
}
if strings.HasPrefix(bind, runner.Container.OutputPath+"/") && bind != runner.Container.OutputPath+"/" {
if mnt.Kind != "collection" && mnt.Kind != "text" && mnt.Kind != "json" {
- return fmt.Errorf("Only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path for %q, was %q", bind, mnt.Kind)
+ return fmt.Errorf("only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path for %q, was %q", bind, mnt.Kind)
}
}
case mnt.Kind == "collection" && bind != "stdin":
var src string
if mnt.UUID != "" && mnt.PortableDataHash != "" {
- return fmt.Errorf("Cannot specify both 'uuid' and 'portable_data_hash' for a collection mount")
+ return fmt.Errorf("cannot specify both 'uuid' and 'portable_data_hash' for a collection mount")
}
if mnt.UUID != "" {
if mnt.Writable {
- return fmt.Errorf("Writing to existing collections currently not permitted.")
+ return fmt.Errorf("writing to existing collections currently not permitted")
}
pdhOnly = false
src = fmt.Sprintf("%s/by_id/%s", runner.ArvMountPoint, mnt.UUID)
} else if mnt.PortableDataHash != "" {
if mnt.Writable && !strings.HasPrefix(bind, runner.Container.OutputPath+"/") {
- return fmt.Errorf("Can never write to a collection specified by portable data hash")
+ return fmt.Errorf("can never write to a collection specified by portable data hash")
}
idx := strings.Index(mnt.PortableDataHash, "/")
if idx > 0 {
var tmpdir string
tmpdir, err = runner.MkTempDir(runner.parentTemp, "tmp")
if err != nil {
- return fmt.Errorf("While creating mount temp dir: %v", err)
+ return fmt.Errorf("while creating mount temp dir: %v", err)
}
st, staterr := os.Stat(tmpdir)
if staterr != nil {
- return fmt.Errorf("While Stat on temp dir: %v", staterr)
+ return fmt.Errorf("while Stat on temp dir: %v", staterr)
}
err = os.Chmod(tmpdir, st.Mode()|os.ModeSetgid|0777)
if staterr != nil {
- return fmt.Errorf("While Chmod temp dir: %v", err)
+ return fmt.Errorf("while Chmod temp dir: %v", err)
}
runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", tmpdir, bind))
if bind == runner.Container.OutputPath {
}
if runner.HostOutputDir == "" {
- return fmt.Errorf("Output path does not correspond to a writable mount point")
+ return fmt.Errorf("output path does not correspond to a writable mount point")
}
if wantAPI := runner.Container.RuntimeConstraints.API; needCertMount && wantAPI != nil && *wantAPI {
runner.ArvMount, err = runner.RunArvMount(arvMountCmd, token)
if err != nil {
- return fmt.Errorf("While trying to start arv-mount: %v", err)
+ return fmt.Errorf("while trying to start arv-mount: %v", err)
}
for _, p := range collectionPaths {
_, err = os.Stat(p)
if err != nil {
- return fmt.Errorf("While checking that input files exist: %v", err)
+ return fmt.Errorf("while checking that input files exist: %v", err)
}
}
for _, cp := range copyFiles {
st, err := os.Stat(cp.src)
if err != nil {
- return fmt.Errorf("While staging writable file from %q to %q: %v", cp.src, cp.bind, err)
+ return fmt.Errorf("while staging writable file from %q to %q: %v", cp.src, cp.bind, err)
}
if st.IsDir() {
err = filepath.Walk(cp.src, func(walkpath string, walkinfo os.FileInfo, walkerr error) error {
}
return os.Chmod(target, walkinfo.Mode()|os.ModeSetgid|0777)
} else {
- return fmt.Errorf("Source %q is not a regular file or directory", cp.src)
+ return fmt.Errorf("source %q is not a regular file or directory", cp.src)
}
})
} else if st.Mode().IsRegular() {
}
}
if err != nil {
- return fmt.Errorf("While staging writable file from %q to %q: %v", cp.src, cp.bind, err)
+ return fmt.Errorf("while staging writable file from %q to %q: %v", cp.src, cp.bind, err)
}
}
err := cr.SetupMounts()
c.Check(err, NotNil)
- c.Check(err, ErrorMatches, `Only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
+ c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
checkEmpty()
err := cr.SetupMounts()
c.Check(err, NotNil)
- c.Check(err, ErrorMatches, `Unsupported mount kind 'tmp' for stdin.*`)
+ c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
checkEmpty()
}`, func(t *TestDockerClient) {})
c.Check(err, NotNil)
- c.Check(strings.Contains(err.Error(), "Unsupported mount kind 'tmp' for stdout"), Equals, true)
+ c.Check(strings.Contains(err.Error(), "unsupported mount kind 'tmp' for stdout"), Equals, true)
}
func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
}`, func(t *TestDockerClient) {})
c.Check(err, NotNil)
- c.Check(strings.Contains(err.Error(), "Unsupported mount kind 'collection' for stdout"), Equals, true)
+ c.Check(strings.Contains(err.Error(), "unsupported mount kind 'collection' for stdout"), Equals, true)
}
func (s *TestSuite) TestFullRunWithAPI(c *C) {
"git.arvados.org/arvados.git/lib/controller/api"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"github.com/jmoiron/sqlx"
+ // sqlx needs lib/pq to talk to PostgreSQL
_ "github.com/lib/pq"
)
DataDirectory string
LogFile string
}
- if pg_lsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil {
+ if pgLsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil {
err = fmt.Errorf("pg_lsclusters: %s", err2)
return 1
- } else if pgclusters := strings.Split(strings.TrimSpace(string(pg_lsclusters)), "\n"); len(pgclusters) != 1 {
+ } else if pgclusters := strings.Split(strings.TrimSpace(string(pgLsclusters)), "\n"); len(pgclusters) != 1 {
logger.Warnf("pg_lsclusters returned %d postgresql clusters -- skipping postgresql initdb/startup, hope that's ok", len(pgclusters))
} else if _, err = fmt.Sscanf(pgclusters[0], "%s %s %d %s %s %s %s", &pgc.Version, &pgc.Cluster, &pgc.Port, &pgc.Status, &pgc.Owner, &pgc.DataDirectory, &pgc.LogFile); err != nil {
err = fmt.Errorf("error parsing pg_lsclusters output: %s", err)
"io"
"log"
"net/http"
+ // pprof is only imported to register its HTTP handlers
_ "net/http/pprof"
"os"
. /usr/src/arvados/build/run-library.sh
TMPHERE=\$(pwd)
cd /usr/src/arvados
+
+ # This defines python_sdk_version and cwl_runner_version with python-style
+ # package suffixes (.dev/rc)
calculate_python_sdk_cwl_package_versions
- cwl_runner_version=\$(echo -n \$cwl_runner_version | sed s/~dev/.dev/g | sed s/~rc/rc/g)
cd \$TMPHERE
set -u
"git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"github.com/jmoiron/sqlx"
+ // sqlx needs lib/pq to talk to PostgreSQL
_ "github.com/lib/pq"
"gopkg.in/check.v1"
)
RunningContainerUUID = "zzzzz-dz642-runningcontainr"
- CompletedContainerUUID = "zzzzz-dz642-compltcontainer"
+ CompletedContainerUUID = "zzzzz-dz642-compltcontainer"
+ CompletedContainerRequestUUID = "zzzzz-xvhdp-cr4completedctr"
+ CompletedContainerRequestUUID2 = "zzzzz-xvhdp-cr4completedcr2"
+
+ CompletedDiagnosticsContainerRequest1UUID = "zzzzz-xvhdp-diagnostics0001"
+ CompletedDiagnosticsContainerRequest2UUID = "zzzzz-xvhdp-diagnostics0002"
+ CompletedDiagnosticsContainer1UUID = "zzzzz-dz642-diagcompreq0001"
+ CompletedDiagnosticsContainer2UUID = "zzzzz-dz642-diagcompreq0002"
+ DiagnosticsContainerRequest1LogCollectionUUID = "zzzzz-4zz18-diagcompreqlog1"
+ DiagnosticsContainerRequest2LogCollectionUUID = "zzzzz-4zz18-diagcompreqlog2"
+
+ CompletedDiagnosticsHasher1ContainerRequestUUID = "zzzzz-xvhdp-diag1hasher0001"
+ CompletedDiagnosticsHasher2ContainerRequestUUID = "zzzzz-xvhdp-diag1hasher0002"
+ CompletedDiagnosticsHasher3ContainerRequestUUID = "zzzzz-xvhdp-diag1hasher0003"
+ CompletedDiagnosticsHasher1ContainerUUID = "zzzzz-dz642-diagcomphasher1"
+ CompletedDiagnosticsHasher2ContainerUUID = "zzzzz-dz642-diagcomphasher2"
+ CompletedDiagnosticsHasher3ContainerUUID = "zzzzz-dz642-diagcomphasher3"
+
+ Hasher1LogCollectionUUID = "zzzzz-4zz18-dlogcollhash001"
+ Hasher2LogCollectionUUID = "zzzzz-4zz18-dlogcollhash002"
+ Hasher3LogCollectionUUID = "zzzzz-4zz18-dlogcollhash003"
ArvadosRepoUUID = "zzzzz-s0uqq-arvadosrepo0123"
ArvadosRepoName = "arvados"
TestVMUUID = "zzzzz-2x53u-382brsig8rp3064"
CollectionWithUniqueWordsUUID = "zzzzz-4zz18-mnt690klmb51aud"
+
+ LogCollectionUUID = "zzzzz-4zz18-logcollection01"
+ LogCollectionUUID2 = "zzzzz-4zz18-logcollection02"
)
// PathologicalManifest : A valid manifest designed to test
func getStackTrace() string {
buf := make([]byte, 1000)
- bytes_written := runtime.Stack(buf, false)
- return "Stack Trace:\n" + string(buf[:bytes_written])
+ bytesWritten := runtime.Stack(buf, false)
+ return "Stack Trace:\n" + string(buf[:bytesWritten])
}
func expectEqual(t *testing.T, actual interface{}, expected interface{}) {
// Reads from the underlying reader, update the hashing function, and
// pass the results through. Returns BadChecksum (instead of EOF) on
// the last read if the checksum doesn't match.
-func (this HashCheckingReader) Read(p []byte) (n int, err error) {
- n, err = this.Reader.Read(p)
+func (hcr HashCheckingReader) Read(p []byte) (n int, err error) {
+ n, err = hcr.Reader.Read(p)
if n > 0 {
- this.Hash.Write(p[:n])
+ hcr.Hash.Write(p[:n])
}
if err == io.EOF {
- sum := this.Hash.Sum(nil)
- if fmt.Sprintf("%x", sum) != this.Check {
+ sum := hcr.Hash.Sum(nil)
+ if fmt.Sprintf("%x", sum) != hcr.Check {
err = BadChecksum
}
}
return n, err
}
-// WriteTo writes the entire contents of this.Reader to dest. Returns
+// WriteTo writes the entire contents of hcr.Reader to dest. Returns
// BadChecksum if writing is successful but the checksum doesn't
// match.
-func (this HashCheckingReader) WriteTo(dest io.Writer) (written int64, err error) {
- if writeto, ok := this.Reader.(io.WriterTo); ok {
- written, err = writeto.WriteTo(io.MultiWriter(dest, this.Hash))
+func (hcr HashCheckingReader) WriteTo(dest io.Writer) (written int64, err error) {
+ if writeto, ok := hcr.Reader.(io.WriterTo); ok {
+ written, err = writeto.WriteTo(io.MultiWriter(dest, hcr.Hash))
} else {
- written, err = io.Copy(io.MultiWriter(dest, this.Hash), this.Reader)
+ written, err = io.Copy(io.MultiWriter(dest, hcr.Hash), hcr.Reader)
}
if err != nil {
return written, err
}
- sum := this.Hash.Sum(nil)
- if fmt.Sprintf("%x", sum) != this.Check {
+ sum := hcr.Hash.Sum(nil)
+ if fmt.Sprintf("%x", sum) != hcr.Check {
return written, BadChecksum
}
// Close reads all remaining data from the underlying Reader and
// returns BadChecksum if the checksum doesn't match. It also closes
// the underlying Reader if it implements io.ReadCloser.
-func (this HashCheckingReader) Close() (err error) {
- _, err = io.Copy(this.Hash, this.Reader)
+func (hcr HashCheckingReader) Close() (err error) {
+ _, err = io.Copy(hcr.Hash, hcr.Reader)
- if closer, ok := this.Reader.(io.Closer); ok {
+ if closer, ok := hcr.Reader.(io.Closer); ok {
closeErr := closer.Close()
if err == nil {
err = closeErr
if err != nil {
return err
}
- if fmt.Sprintf("%x", this.Hash.Sum(nil)) != this.Check {
+ if fmt.Sprintf("%x", hcr.Hash.Sum(nil)) != hcr.Check {
return BadChecksum
}
return nil
type StubPutHandler struct {
c *C
expectPath string
- expectApiToken string
+ expectAPIToken string
expectBody string
expectStorageClass string
handled chan string
func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
- sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
+ sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectAPIToken))
sph.c.Check(req.Header.Get("X-Keep-Storage-Classes"), Equals, sph.expectStorageClass)
body, err := ioutil.ReadAll(req.Body)
sph.c.Check(err, Equals, nil)
func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
ks = make([]KeepServer, n)
- for i := 0; i < n; i += 1 {
+ for i := 0; i < n; i++ {
ks[i] = RunFakeKeepServer(st)
}
type StubGetHandler struct {
c *C
expectPath string
- expectApiToken string
+ expectAPIToken string
httpStatus int
body []byte
}
func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath)
- sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectApiToken))
+ sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectAPIToken))
resp.WriteHeader(sgh.httpStatus)
resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body)))
resp.Write(sgh.body)
response string
}
-func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Reader,
+func (kc *KeepClient) uploadToKeepServer(host string, hash string, body io.Reader,
uploadStatusChan chan<- uploadStatus, expectedLength int64, reqid string) {
var req *http.Request
}
req.Header.Add("X-Request-Id", reqid)
- req.Header.Add("Authorization", "OAuth2 "+this.Arvados.ApiToken)
+ req.Header.Add("Authorization", "OAuth2 "+kc.Arvados.ApiToken)
req.Header.Add("Content-Type", "application/octet-stream")
- req.Header.Add(XKeepDesiredReplicas, fmt.Sprint(this.Want_replicas))
- if len(this.StorageClasses) > 0 {
- req.Header.Add("X-Keep-Storage-Classes", strings.Join(this.StorageClasses, ", "))
+ req.Header.Add(XKeepDesiredReplicas, fmt.Sprint(kc.Want_replicas))
+ if len(kc.StorageClasses) > 0 {
+ req.Header.Add("X-Keep-Storage-Classes", strings.Join(kc.StorageClasses, ", "))
}
var resp *http.Response
- if resp, err = this.httpClient().Do(req); err != nil {
+ if resp, err = kc.httpClient().Do(req); err != nil {
DebugPrintf("DEBUG: [%s] Upload failed %v error: %v", reqid, url, err.Error())
uploadStatusChan <- uploadStatus{err, url, 0, 0, err.Error()}
return
}
}
-func (this *KeepClient) putReplicas(
+func (kc *KeepClient) putReplicas(
hash string,
getReader func() io.Reader,
expectedLength int64) (locator string, replicas int, err error) {
- reqid := this.getRequestID()
+ reqid := kc.getRequestID()
// Calculate the ordering for uploading to servers
- sv := NewRootSorter(this.WritableLocalRoots(), hash).GetSortedRoots()
+ sv := NewRootSorter(kc.WritableLocalRoots(), hash).GetSortedRoots()
// The next server to try contacting
nextServer := 0
}()
replicasDone := 0
- replicasTodo := this.Want_replicas
+ replicasTodo := kc.Want_replicas
- replicasPerThread := this.replicasPerService
+ replicasPerThread := kc.replicasPerService
if replicasPerThread < 1 {
// unlimited or unknown
replicasPerThread = replicasTodo
}
- retriesRemaining := 1 + this.Retries
+ retriesRemaining := 1 + kc.Retries
var retryServers []string
lastError := make(map[string]string)
// Start some upload requests
if nextServer < len(sv) {
DebugPrintf("DEBUG: [%s] Begin upload %s to %s", reqid, hash, sv[nextServer])
- go this.uploadToKeepServer(sv[nextServer], hash, getReader(), uploadStatusChan, expectedLength, reqid)
+ go kc.uploadToKeepServer(sv[nextServer], hash, getReader(), uploadStatusChan, expectedLength, reqid)
nextServer++
active++
} else {
token_uuid = ''
secret = token
+ stored_secret = nil # ...if different from secret
optional = nil
case token[0..2]
# below. If so, we'll stuff the database with hmac instead of
# the real OIDC token.
upstream_cluster_id = Rails.configuration.Login.LoginCluster
- token_uuid = upstream_cluster_id + generate_uuid[5..27]
- secret = hmac
+ stored_secret = hmac
else
return nil
end
remote_user_prefix = remote_user['uuid'][0..4]
+ if token_uuid == ''
+ # Use the same UUID as the remote when caching the token.
+ begin
+ remote_token = SafeJSON.load(
+ clnt.get_content('https://' + host + '/arvados/v1/api_client_authorizations/current',
+ {'remote' => Rails.configuration.ClusterID},
+ {'Authorization' => 'Bearer ' + token}))
+ token_uuid = remote_token['uuid']
+ if !token_uuid.match(HasUuid::UUID_REGEX) || token_uuid[0..4] != upstream_cluster_id
+ raise "remote cluster #{upstream_cluster_id} returned invalid token uuid #{token_uuid.inspect}"
+ end
+ rescue => e
+ Rails.logger.warn "error getting remote token details for #{token.inspect}: #{e}"
+ return nil
+ end
+ end
+
# Clusters can only authenticate for their own users.
if remote_user_prefix != upstream_cluster_id
Rails.logger.warn "remote authentication rejected: claimed remote user #{remote_user_prefix} but token was issued by #{upstream_cluster_id}"
auth.user = user
auth.api_client_id = 0
end
+ # If stored_secret is set, we save stored_secret in the database
+ # but return the real secret to the caller. This way, if we end
+ # up returning the auth record to the client, they see the same
+ # secret they supplied, instead of the HMAC we saved in the
+ # database.
+ stored_secret = stored_secret || secret
auth.update_attributes!(user: user,
- api_token: secret,
+ api_token: stored_secret,
api_client_id: 0,
expires_at: Time.now + Rails.configuration.Login.RemoteTokenRefresh)
- Rails.logger.debug "cached remote token #{token_uuid} with secret #{secret} in local db"
+ Rails.logger.debug "cached remote token #{token_uuid} with secret #{stored_secret} in local db"
+ auth.api_token = secret
return auth
end
properties:
"http://schema.org/example": "value1"
+log_collection:
+ uuid: zzzzz-4zz18-logcollection01
+ current_version_uuid: zzzzz-4zz18-logcollection01
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-10-29T00:51:44.075594000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-10-29T00:51:44.072109000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: a real log collection for a completed container
+
+log_collection2:
+ uuid: zzzzz-4zz18-logcollection02
+ current_version_uuid: zzzzz-4zz18-logcollection02
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-10-29T00:51:44.075594000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-10-29T00:51:44.072109000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: another real log collection for a completed container
+
+diagnostics_request_container_log_collection:
+ uuid: zzzzz-4zz18-diagcompreqlog1
+ current_version_uuid: zzzzz-4zz18-diagcompreqlog1
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-02T00:20:44.007557000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-02T00:20:44.005381000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: Container log for request zzzzz-xvhdp-diagnostics0001
+
+hasher1_log_collection:
+ uuid: zzzzz-4zz18-dlogcollhash001
+ current_version_uuid: zzzzz-4zz18-dlogcollhash001
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-02T00:16:55.272606000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-02T00:16:55.267006000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: hasher1 log collection
+
+hasher2_log_collection:
+ uuid: zzzzz-4zz18-dlogcollhash002
+ current_version_uuid: zzzzz-4zz18-dlogcollhash002
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-02T00:20:23.547251000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-02T00:20:23.545275000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: hasher2 log collection
+
+hasher3_log_collection:
+ uuid: zzzzz-4zz18-dlogcollhash003
+ current_version_uuid: zzzzz-4zz18-dlogcollhash003
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-02T00:20:38.789204000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-02T00:20:38.787329000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: hasher3 log collection
+
+diagnostics_request_container_log_collection2:
+ uuid: zzzzz-4zz18-diagcompreqlog2
+ current_version_uuid: zzzzz-4zz18-diagcompreqlog2
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-03T16:17:53.351593000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-03T16:17:53.346969000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: Container log for request zzzzz-xvhdp-diagnostics0002
+
# Test Helper trims the rest of the file
# Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
output_path: test
command: ["echo", "hello"]
container_uuid: zzzzz-dz642-compltcontainer
- log_uuid: zzzzz-4zz18-y9vne9npefyxh8g
+ log_uuid: zzzzz-4zz18-logcollection01
output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
runtime_constraints:
vcpus: 1
output_path: test
command: ["arvados-cwl-runner", "echo", "hello"]
container_uuid: zzzzz-dz642-compltcontainr2
+ log_uuid: zzzzz-4zz18-logcollection02
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
runtime_constraints:
vcpus: 1
ram: 123
+completed_diagnostics:
+ name: CWL diagnostics hasher
+ uuid: zzzzz-xvhdp-diagnostics0001
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 1
+ created_at: 2020-11-02T00:03:50.229364000Z
+ modified_at: 2020-11-02T00:20:44.041122000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_path: /var/spool/cwl
+ command: [
+ "arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--no-log-timestamps",
+ "--disable-validate",
+ "--disable-color",
+ "--eval-timeout=20",
+ "--thread-count=1",
+ "--disable-reuse",
+ "--collection-cache-size=256",
+ "--on-error=continue",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"
+ ]
+ container_uuid: zzzzz-dz642-diagcompreq0001
+ log_uuid: zzzzz-4zz18-diagcompreqlog1
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 1342177280
+ API: true
+
+completed_diagnostics_hasher1:
+ name: hasher1
+ uuid: zzzzz-xvhdp-diag1hasher0001
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:03:50.229364000Z
+ modified_at: 2020-11-02T00:20:44.041122000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher1
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/9f26a86b6030a69ad222cf67d71c9502+65/hasher-input-file.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher1
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0001
+ log_uuid: zzzzz-4zz18-dlogcollhash001
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 2684354560
+ API: true
+
+completed_diagnostics_hasher2:
+ name: hasher2
+ uuid: zzzzz-xvhdp-diag1hasher0002
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:17:07.067464000Z
+ modified_at: 2020-11-02T00:20:23.557498000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher2
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/d3a687732e84061f3bae15dc7e313483+62/hasher1.md5sum.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher2
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0001
+ log_uuid: zzzzz-4zz18-dlogcollhash002
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 2
+ ram: 2684354560
+ API: true
+
+completed_diagnostics_hasher3:
+ name: hasher3
+ uuid: zzzzz-xvhdp-diag1hasher0003
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:20:30.960251000Z
+ modified_at: 2020-11-02T00:20:38.799377000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher3
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/6bd770f6cf8f83e7647c602eecfaeeb8+62/hasher2.md5sum.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher3
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0001
+ log_uuid: zzzzz-4zz18-dlogcollhash003
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 2684354560
+ API: true
+
+completed_diagnostics2:
+ name: Copy of CWL diagnostics hasher
+ uuid: zzzzz-xvhdp-diagnostics0002
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 1
+ created_at: 2020-11-03T15:54:30.098485000Z
+ modified_at: 2020-11-03T16:17:53.406809000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_path: /var/spool/cwl
+ command: [
+ "arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--no-log-timestamps",
+ "--disable-validate",
+ "--disable-color",
+ "--eval-timeout=20",
+ "--thread-count=1",
+ "--disable-reuse",
+ "--collection-cache-size=256",
+ "--on-error=continue",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"
+ ]
+ container_uuid: zzzzz-dz642-diagcompreq0002
+ log_uuid: zzzzz-4zz18-diagcompreqlog2
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 1342177280
+ API: true
+
+completed_diagnostics_hasher1_reuse:
+ name: hasher1
+ uuid: zzzzz-xvhdp-diag2hasher0001
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:03:50.229364000Z
+ modified_at: 2020-11-02T00:20:44.041122000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher1
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/9f26a86b6030a69ad222cf67d71c9502+65/hasher-input-file.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher1
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0002
+ log_uuid: zzzzz-4zz18-dlogcollhash001
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 2684354560
+ API: true
+
+completed_diagnostics_hasher2_reuse:
+ name: hasher2
+ uuid: zzzzz-xvhdp-diag2hasher0002
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:17:07.067464000Z
+ modified_at: 2020-11-02T00:20:23.557498000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher2
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/d3a687732e84061f3bae15dc7e313483+62/hasher1.md5sum.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher2
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0002
+ log_uuid: zzzzz-4zz18-dlogcollhash002
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 2
+ ram: 2684354560
+ API: true
+
+completed_diagnostics_hasher3_reuse:
+ name: hasher3
+ uuid: zzzzz-xvhdp-diag2hasher0003
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:20:30.960251000Z
+ modified_at: 2020-11-02T00:20:38.799377000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher3
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/6bd770f6cf8f83e7647c602eecfaeeb8+62/hasher2.md5sum.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher3
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0002
+ log_uuid: zzzzz-4zz18-dlogcollhash003
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 2684354560
+ API: true
+
requester:
uuid: zzzzz-xvhdp-9zacv3o1xw6sxz5
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
vcpus: 1
ram: 123
container_uuid: zzzzz-dz642-compltcontainer
- log_uuid: zzzzz-4zz18-y9vne9npefyxh8g
+ log_uuid: zzzzz-4zz18-logcollection01
output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
mounts:
/var/lib/cwl/cwl.input.json:
output_path: test
command: ["echo", "hello"]
container_uuid: zzzzz-dz642-compltcontainer
- log_uuid: zzzzz-4zz18-y9vne9npefyxh8g
+ log_uuid: zzzzz-4zz18-logcollection01
output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
runtime_constraints:
vcpus: 1
secret_mounts: {}
secret_mounts_md5: 99914b932bd37a50b983c5e7c90ae93b
+diagnostics_completed_requester:
+ uuid: zzzzz-dz642-diagcompreq0001
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 562948349145881771
+ created_at: 2020-11-02T00:03:50.192697000Z
+ modified_at: 2020-11-02T00:20:43.987275000Z
+ started_at: 2020-11-02T00:08:07.186711000Z
+ finished_at: 2020-11-02T00:20:43.975416000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: 6129e376cb05c942f75a0c36083383e8+244
+ output: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
+ output_path: /var/spool/cwl
+ command: [
+ "arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--no-log-timestamps",
+ "--disable-validate",
+ "--disable-color",
+ "--eval-timeout=20",
+ "--thread-count=1",
+ "--disable-reuse",
+ "--collection-cache-size=256",
+ "--on-error=continue",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 1342177280
+ vcpus: 1
+
+diagnostics_completed_hasher1:
+ uuid: zzzzz-dz642-diagcomphasher1
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 562948349145881771
+ created_at: 2020-11-02T00:08:18.829222000Z
+ modified_at: 2020-11-02T00:16:55.142023000Z
+ started_at: 2020-11-02T00:16:52.375871000Z
+ finished_at: 2020-11-02T00:16:55.105985000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: fed8fb19fe8e3a320c29fed0edab12dd+220
+ output: d3a687732e84061f3bae15dc7e313483+62
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/9f26a86b6030a69ad222cf67d71c9502+65/hasher-input-file.txt"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 268435456
+ vcpus: 1
+
+diagnostics_completed_hasher2:
+ uuid: zzzzz-dz642-diagcomphasher2
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 562948349145881771
+ created_at: 2020-11-02T00:17:07.026493000Z
+ modified_at: 2020-11-02T00:20:23.505908000Z
+ started_at: 2020-11-02T00:20:21.513185000Z
+ finished_at: 2020-11-02T00:20:23.478317000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: 4fc03b95fc2646b0dec7383dbb7d56d8+221
+ output: 6bd770f6cf8f83e7647c602eecfaeeb8+62
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/d3a687732e84061f3bae15dc7e313483+62/hasher1.md5sum.txt"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 268435456
+ vcpus: 2
+
+diagnostics_completed_hasher3:
+ uuid: zzzzz-dz642-diagcomphasher3
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 562948349145881771
+ created_at: 2020-11-02T00:20:30.943856000Z
+ modified_at: 2020-11-02T00:20:38.746541000Z
+ started_at: 2020-11-02T00:20:36.748957000Z
+ finished_at: 2020-11-02T00:20:38.732199000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: 1eeaf70de0f65b1346e54c59f09e848d+210
+ output: 11b5fdaa380102e760c3eb6de80a9876+62
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/6bd770f6cf8f83e7647c602eecfaeeb8+62/hasher2.md5sum.txt"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 268435456
+ vcpus: 1
+
+diagnostics_completed_requester2:
+ uuid: zzzzz-dz642-diagcompreq0002
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 1124295487972526
+ created_at: 2020-11-03T15:54:36.504661000Z
+ modified_at: 2020-11-03T16:17:53.242868000Z
+ started_at: 2020-11-03T16:09:51.123659000Z
+ finished_at: 2020-11-03T16:17:53.220358000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: f1933bf5191f576613ea7f65bd0ead53+244
+ output: 941b71a57208741ce8742eca62352fb1+123
+ output_path: /var/spool/cwl
+ command: [
+ "arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--no-log-timestamps",
+ "--disable-validate",
+ "--disable-color",
+ "--eval-timeout=20",
+ "--thread-count=1",
+ "--disable-reuse",
+ "--collection-cache-size=256",
+ "--on-error=continue",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 1342177280
+ vcpus: 1
+
requester:
uuid: zzzzz-dz642-requestingcntnr
owner_uuid: zzzzz-tpzed-000000000000000
// Cancelled or Complete. See https://dev.arvados.org/issues/10979
func (disp *Dispatcher) checkSqueueForOrphans() {
for _, uuid := range disp.sqCheck.All() {
- if !containerUuidPattern.MatchString(uuid) {
+ if !containerUuidPattern.MatchString(uuid) || !strings.HasPrefix(uuid, disp.cluster.ClusterID) {
continue
}
err := disp.TrackContainer(uuid)
var urlPDHDecoder = strings.NewReplacer(" ", "+", "-", "+")
+var notFoundMessage = "404 Not found\r\n\r\nThe requested path was not found, or you do not have permission to access it.\r"
+var unauthorizedMessage = "401 Unauthorized\r\n\r\nA valid Arvados token must be provided to access this resource.\r"
+
// parseCollectionIDFromURL returns a UUID or PDH if s is a UUID or a
// PDH (even if it is a PDH with "+" replaced by " " or "-");
// otherwise "".
}
if collectionID == "" && !useSiteFS {
- w.WriteHeader(http.StatusNotFound)
+ http.Error(w, notFoundMessage, http.StatusNotFound)
return
}
// for additional credentials would just be
// confusing), or we don't even accept
// credentials at this path.
- w.WriteHeader(http.StatusNotFound)
+ http.Error(w, notFoundMessage, http.StatusNotFound)
return
}
for _, t := range reqTokens {
if tokenResult[t] == 404 {
// The client provided valid token(s), but the
// collection was not found.
- w.WriteHeader(http.StatusNotFound)
+ http.Error(w, notFoundMessage, http.StatusNotFound)
return
}
}
// data that has been deleted. Allow a referrer to
// provide this context somehow?
w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
- w.WriteHeader(http.StatusUnauthorized)
+ http.Error(w, unauthorizedMessage, http.StatusUnauthorized)
return
}
openPath := "/" + strings.Join(targetPath, "/")
if f, err := fs.Open(openPath); os.IsNotExist(err) {
// Requested non-existent path
- w.WriteHeader(http.StatusNotFound)
+ http.Error(w, notFoundMessage, http.StatusNotFound)
} else if err != nil {
// Some other (unexpected) error
http.Error(w, "open: "+err.Error(), http.StatusInternalServerError)
func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string, credentialsOK, attachment bool) {
if len(tokens) == 0 {
w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
- http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
+ http.Error(w, unauthorizedMessage, http.StatusUnauthorized)
return
}
if writeMethod[r.Method] {
}
s.testServer.Handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNotFound)
- c.Check(resp.Body.String(), check.Equals, "")
+ c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
}
}
// depending on the authz method.
c.Check(code, check.Equals, failCode)
}
- c.Check(body, check.Equals, "")
+ if code == 404 {
+ c.Check(body, check.Equals, notFoundMessage+"\n")
+ } else {
+ c.Check(body, check.Equals, unauthorizedMessage+"\n")
+ }
}
}
}
"",
"",
http.StatusNotFound,
- "",
+ notFoundMessage+"\n",
)
}
"",
"",
http.StatusUnauthorized,
- "",
+ unauthorizedMessage+"\n",
)
}
"application/x-www-form-urlencoded",
url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
http.StatusNotFound,
- "",
+ notFoundMessage+"\n",
)
}
"",
"",
http.StatusNotFound,
- "",
+ notFoundMessage+"\n",
)
}
"net/url"
"os"
"path/filepath"
+ "regexp"
"sort"
"strconv"
"strings"
return hashdigest(hmac.New(sha256.New, key), stringToSign), nil
}
+var v2tokenUnderscore = regexp.MustCompile(`^v2_[a-z0-9]{5}-gj3su-[a-z0-9]{15}_`)
+
+func unescapeKey(key string) string {
+ if v2tokenUnderscore.MatchString(key) {
+ // Entire Arvados token, with "/" replaced by "_" to
+ // avoid colliding with the Authorization header
+ // format.
+ return strings.Replace(key, "_", "/", -1)
+ } else if s, err := url.PathUnescape(key); err == nil {
+ return s
+ } else {
+ return key
+ }
+}
+
// checks3signature verifies the given S3 V4 signature and returns the
// Arvados token that corresponds to the given accessKey. An error is
// returned if accessKey is not a valid token UUID or the signature
} else {
// Access key and secret key are both an entire
// Arvados token or OIDC access token.
- ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+key)
+ ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+unescapeKey(key))
err = client.RequestAndDecodeContext(ctx, &aca, "GET", "arvados/v1/api_client_authorizations/current", nil, nil)
secret = key
}
} else if expect != signature {
return "", fmt.Errorf("signature does not match (scope %q signedHeaders %q stringToSign %q)", scope, signedHeaders, stringToSign)
}
- return secret, nil
+ return aca.TokenV2(), nil
}
+func s3ErrorResponse(w http.ResponseWriter, s3code string, message string, resource string, code int) {
+ w.Header().Set("Content-Type", "application/xml")
+ w.Header().Set("X-Content-Type-Options", "nosniff")
+ w.WriteHeader(code)
+ var errstruct struct {
+ Code string
+ Message string
+ Resource string
+ RequestId string
+ }
+ errstruct.Code = s3code
+ errstruct.Message = message
+ errstruct.Resource = resource
+ errstruct.RequestId = ""
+ enc := xml.NewEncoder(w)
+ fmt.Fprint(w, xml.Header)
+ enc.EncodeElement(errstruct, xml.StartElement{Name: xml.Name{Local: "Error"}})
+}
+
+var NoSuchKey = "NoSuchKey"
+var NoSuchBucket = "NoSuchBucket"
+var InvalidArgument = "InvalidArgument"
+var InternalError = "InternalError"
+var UnauthorizedAccess = "UnauthorizedAccess"
+var InvalidRequest = "InvalidRequest"
+var SignatureDoesNotMatch = "SignatureDoesNotMatch"
+
// serveS3 handles r and returns true if r is a request from an S3
// client, otherwise it returns false.
func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
if auth := r.Header.Get("Authorization"); strings.HasPrefix(auth, "AWS ") {
split := strings.SplitN(auth[4:], ":", 2)
if len(split) < 2 {
- http.Error(w, "malformed Authorization header", http.StatusUnauthorized)
+ s3ErrorResponse(w, InvalidRequest, "malformed Authorization header", r.URL.Path, http.StatusUnauthorized)
return true
}
- token = split[0]
+ token = unescapeKey(split[0])
} else if strings.HasPrefix(auth, s3SignAlgorithm+" ") {
t, err := h.checks3signature(r)
if err != nil {
- http.Error(w, "signature verification failed: "+err.Error(), http.StatusForbidden)
+ s3ErrorResponse(w, SignatureDoesNotMatch, "signature verification failed: "+err.Error(), r.URL.Path, http.StatusForbidden)
return true
}
token = t
_, kc, client, release, err := h.getClients(r.Header.Get("X-Request-Id"), token)
if err != nil {
- http.Error(w, "Pool failed: "+h.clientPool.Err().Error(), http.StatusInternalServerError)
+ s3ErrorResponse(w, InternalError, "Pool failed: "+h.clientPool.Err().Error(), r.URL.Path, http.StatusInternalServerError)
return true
}
defer release()
fs := client.SiteFileSystem(kc)
fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
- objectNameGiven := strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
+ var objectNameGiven bool
+ fspath := "/by_id"
+ if id := parseCollectionIDFromDNSName(r.Host); id != "" {
+ fspath += "/" + id
+ objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 0
+ } else {
+ objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
+ }
+ fspath += r.URL.Path
switch {
case r.Method == http.MethodGet && !objectNameGiven:
}
return true
case r.Method == http.MethodGet || r.Method == http.MethodHead:
- fspath := "/by_id" + r.URL.Path
fi, err := fs.Stat(fspath)
if r.Method == "HEAD" && !objectNameGiven {
// HeadBucket
if err == nil && fi.IsDir() {
w.WriteHeader(http.StatusOK)
} else if os.IsNotExist(err) {
- w.WriteHeader(http.StatusNotFound)
+ s3ErrorResponse(w, NoSuchBucket, "The specified bucket does not exist.", r.URL.Path, http.StatusNotFound)
} else {
- http.Error(w, err.Error(), http.StatusBadGateway)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusBadGateway)
}
return true
}
if os.IsNotExist(err) ||
(err != nil && err.Error() == "not a directory") ||
(fi != nil && fi.IsDir()) {
- http.Error(w, "not found", http.StatusNotFound)
+ s3ErrorResponse(w, NoSuchKey, "The specified key does not exist.", r.URL.Path, http.StatusNotFound)
return true
}
// shallow copy r, and change URL path
return true
case r.Method == http.MethodPut:
if !objectNameGiven {
- http.Error(w, "missing object name in PUT request", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "Missing object name in PUT request.", r.URL.Path, http.StatusBadRequest)
return true
}
- fspath := "by_id" + r.URL.Path
var objectIsDir bool
if strings.HasSuffix(fspath, "/") {
if !h.Config.cluster.Collections.S3FolderObjects {
- http.Error(w, "invalid object name: trailing slash", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "invalid object name: trailing slash", r.URL.Path, http.StatusBadRequest)
return true
}
n, err := r.Body.Read(make([]byte, 1))
if err != nil && err != io.EOF {
- http.Error(w, fmt.Sprintf("error reading request body: %s", err), http.StatusInternalServerError)
+ s3ErrorResponse(w, InternalError, fmt.Sprintf("error reading request body: %s", err), r.URL.Path, http.StatusInternalServerError)
return true
} else if n > 0 {
- http.Error(w, "cannot create object with trailing '/' char unless content is empty", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "cannot create object with trailing '/' char unless content is empty", r.URL.Path, http.StatusBadRequest)
return true
} else if strings.SplitN(r.Header.Get("Content-Type"), ";", 2)[0] != "application/x-directory" {
- http.Error(w, "cannot create object with trailing '/' char unless Content-Type is 'application/x-directory'", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "cannot create object with trailing '/' char unless Content-Type is 'application/x-directory'", r.URL.Path, http.StatusBadRequest)
return true
}
// Given PUT "foo/bar/", we'll use "foo/bar/."
fi, err := fs.Stat(fspath)
if err != nil && err.Error() == "not a directory" {
// requested foo/bar, but foo is a file
- http.Error(w, "object name conflicts with existing object", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "object name conflicts with existing object", r.URL.Path, http.StatusBadRequest)
return true
}
if strings.HasSuffix(r.URL.Path, "/") && err == nil && !fi.IsDir() {
// requested foo/bar/, but foo/bar is a file
- http.Error(w, "object name conflicts with existing object", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "object name conflicts with existing object", r.URL.Path, http.StatusBadRequest)
return true
}
// create missing parent/intermediate directories, if any
dir := fspath[:i]
if strings.HasSuffix(dir, "/") {
err = errors.New("invalid object name (consecutive '/' chars)")
- http.Error(w, err.Error(), http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, err.Error(), r.URL.Path, http.StatusBadRequest)
return true
}
err = fs.Mkdir(dir, 0755)
// Cannot create a directory
// here.
err = fmt.Errorf("mkdir %q failed: %w", dir, err)
- http.Error(w, err.Error(), http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, err.Error(), r.URL.Path, http.StatusBadRequest)
return true
} else if err != nil && !os.IsExist(err) {
err = fmt.Errorf("mkdir %q failed: %w", dir, err)
- http.Error(w, err.Error(), http.StatusInternalServerError)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
return true
}
}
}
if err != nil {
err = fmt.Errorf("open %q failed: %w", r.URL.Path, err)
- http.Error(w, err.Error(), http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, err.Error(), r.URL.Path, http.StatusBadRequest)
return true
}
defer f.Close()
_, err = io.Copy(f, r.Body)
if err != nil {
err = fmt.Errorf("write to %q failed: %w", r.URL.Path, err)
- http.Error(w, err.Error(), http.StatusBadGateway)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusBadGateway)
return true
}
err = f.Close()
if err != nil {
err = fmt.Errorf("write to %q failed: close: %w", r.URL.Path, err)
- http.Error(w, err.Error(), http.StatusBadGateway)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusBadGateway)
return true
}
}
err = fs.Sync()
if err != nil {
err = fmt.Errorf("sync failed: %w", err)
- http.Error(w, err.Error(), http.StatusInternalServerError)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
return true
}
w.WriteHeader(http.StatusOK)
return true
case r.Method == http.MethodDelete:
if !objectNameGiven || r.URL.Path == "/" {
- http.Error(w, "missing object name in DELETE request", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "missing object name in DELETE request", r.URL.Path, http.StatusBadRequest)
return true
}
- fspath := "by_id" + r.URL.Path
if strings.HasSuffix(fspath, "/") {
fspath = strings.TrimSuffix(fspath, "/")
fi, err := fs.Stat(fspath)
w.WriteHeader(http.StatusNoContent)
return true
} else if err != nil {
- http.Error(w, err.Error(), http.StatusInternalServerError)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
return true
} else if !fi.IsDir() {
// if "foo" exists and is a file, then
}
if err != nil {
err = fmt.Errorf("rm failed: %w", err)
- http.Error(w, err.Error(), http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, err.Error(), r.URL.Path, http.StatusBadRequest)
return true
}
err = fs.Sync()
if err != nil {
err = fmt.Errorf("sync failed: %w", err)
- http.Error(w, err.Error(), http.StatusInternalServerError)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
return true
}
w.WriteHeader(http.StatusNoContent)
return true
default:
- http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+ s3ErrorResponse(w, InvalidRequest, "method not allowed", r.URL.Path, http.StatusMethodNotAllowed)
+
return true
}
}
"fmt"
"io/ioutil"
"net/http"
+ "net/url"
"os"
"os/exec"
"strings"
secretkey string
}{
{true, aws.V2Signature, arvadostest.ActiveToken, "none"},
+ {true, aws.V2Signature, url.QueryEscape(arvadostest.ActiveTokenV2), "none"},
+ {true, aws.V2Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), "none"},
{false, aws.V2Signature, "none", "none"},
{false, aws.V2Signature, "none", arvadostest.ActiveToken},
{true, aws.V4Signature, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken},
{true, aws.V4Signature, arvadostest.ActiveToken, arvadostest.ActiveToken},
+ {true, aws.V4Signature, url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2)},
+ {true, aws.V4Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1)},
{false, aws.V4Signature, arvadostest.ActiveToken, ""},
{false, aws.V4Signature, arvadostest.ActiveToken, "none"},
{false, aws.V4Signature, "none", arvadostest.ActiveToken},
// GetObject
rdr, err = bucket.GetReader(prefix + "missingfile")
- c.Check(err, check.ErrorMatches, `404 Not Found`)
+ c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+ c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
+ c.Check(err, check.ErrorMatches, `The specified key does not exist.`)
// HeadObject
exists, err := bucket.Exists(prefix + "missingfile")
objname := prefix + trial.path
_, err := bucket.GetReader(objname)
- c.Assert(err, check.ErrorMatches, `404 Not Found`)
+ c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+ c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
+ c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
buf := make([]byte, trial.size)
rand.Read(buf)
c.Logf("=== %v", trial)
_, err := bucket.GetReader(trial.path)
- c.Assert(err, check.ErrorMatches, `404 Not Found`)
+ c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+ c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
+ c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
buf := make([]byte, trial.size)
rand.Read(buf)
err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
- c.Check(err, check.ErrorMatches, `400 Bad Request`)
+ c.Check(err.(*s3.Error).StatusCode, check.Equals, 400)
+ c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`)
+ c.Check(err, check.ErrorMatches, `(mkdir "by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid argument`)
_, err = bucket.GetReader(trial.path)
- c.Assert(err, check.ErrorMatches, `404 Not Found`)
+ c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+ c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
+ c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
}
}
rand.Read(buf)
err := bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), "application/octet-stream", s3.Private, s3.Options{})
- if !c.Check(err, check.ErrorMatches, `400 Bad.*`, check.Commentf("PUT %q should fail", objname)) {
+ if !c.Check(err, check.ErrorMatches, `(invalid object name.*|open ".*" failed.*|object name conflicts with existing object|Missing object name in PUT request.)`, check.Commentf("PUT %q should fail", objname)) {
return
}
if objname != "" && objname != "/" {
_, err = bucket.GetReader(objname)
- c.Check(err, check.ErrorMatches, `404 Not Found`, check.Commentf("GET %q should return 404", objname))
+ c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+ c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
+ c.Check(err, check.ErrorMatches, `The specified key does not exist.`, check.Commentf("GET %q should return 404", objname))
}
}()
}
c.Check(err, check.IsNil)
c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
}
+
+func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) {
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+
+ hdr, body, _ := s.runCurl(c, "AWS "+arvadostest.ActiveTokenV2+":none", stage.coll.UUID+".collections.example.com", "/sailboat.txt")
+ c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
+ c.Check(body, check.Equals, "⛵\n")
+}
} {
hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
- c.Check(body, check.Equals, "")
+ c.Check(body, check.Equals, notFoundMessage+"\n")
if token != "" {
hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
- c.Check(body, check.Equals, "")
+ c.Check(body, check.Equals, notFoundMessage+"\n")
}
hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
- c.Check(body, check.Equals, "")
+ c.Check(body, check.Equals, notFoundMessage+"\n")
}
}
hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
if len(body) > 0 {
- c.Check(body, check.Equals, "404 page not found\n")
+ c.Check(body, check.Equals, notFoundMessage+"\n")
}
}
}
}
// Return header block and body.
-func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
+func (s *IntegrationSuite) runCurl(c *check.C, auth, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
curlArgs := []string{"--silent", "--show-error", "--include"}
testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
- if token != "" {
- curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token)
+ if strings.Contains(auth, " ") {
+ // caller supplied entire Authorization header value
+ curlArgs = append(curlArgs, "-H", "Authorization: "+auth)
+ } else if auth != "" {
+ // caller supplied Arvados token
+ curlArgs = append(curlArgs, "-H", "Authorization: Bearer "+auth)
}
curlArgs = append(curlArgs, args...)
curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
config.vm.define "arvados" do |arv|
arv.vm.box = "bento/debian-10"
- arv.vm.hostname = "arva2.arv.local"
+ arv.vm.hostname = "vagrant.local"
+ # CPU/RAM
+ config.vm.provider :virtualbox do |v|
+ v.memory = 2048
+ v.cpus = 2
+ end
+
# Networking
arv.vm.network "forwarded_port", guest: 8443, host: 8443
arv.vm.network "forwarded_port", guest: 25100, host: 25100
arv.vm.network "forwarded_port", guest: 8001, host: 8001
arv.vm.network "forwarded_port", guest: 8000, host: 8000
arv.vm.network "forwarded_port", guest: 3001, host: 3001
- # config.vm.network "private_network", ip: "192.168.33.10"
- # arv.vm.synced_folder "salt_pillars", "/srv/pillars",
- # create: true
arv.vm.provision "shell",
path: "provision.sh",
args: [
+ "--test",
"--vagrant",
"--ssl-port=8443"
].join(" ")
-#!/bin/bash -x
+#!/bin/bash
# Copyright (C) The Arvados Authors. All rights reserved.
#
set -o pipefail
+# capture the directory that the script is running from
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
usage() {
echo >&2
- echo >&2 "Usage: $0 [-h] [-h]"
+ echo >&2 "Usage: ${0} [-h] [-h]"
echo >&2
- echo >&2 "$0 options:"
- echo >&2 " -v, --vagrant Run in vagrant and use the /vagrant shared dir"
+ echo >&2 "${0} options:"
+ echo >&2 " -d, --debug Run salt installation in debug mode"
echo >&2 " -p <N>, --ssl-port <N> SSL port to use for the web applications"
+ echo >&2 " -t, --test Test installation running a CWL workflow"
echo >&2 " -h, --help Display this help and exit"
+ echo >&2 " -v, --vagrant Run in vagrant and use the /vagrant shared dir"
echo >&2
}
arguments() {
# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
- TEMP=`getopt -o hvp: \
- --long help,vagrant,ssl-port: \
- -n "$0" -- "$@"`
+ TEMP=$(getopt -o dhp:tv \
+ --long debug,help,ssl-port:,test,vagrant \
+ -n "${0}" -- "${@}")
- if [ $? != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi
+ if [ ${?} != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi
# Note the quotes around `$TEMP': they are essential!
eval set -- "$TEMP"
- while [ $# -ge 1 ]; do
- case $1 in
+ while [ ${#} -ge 1 ]; do
+ case ${1} in
+ -d | --debug)
+ LOG_LEVEL="debug"
+ shift
+ ;;
+ -t | --test)
+ TEST="yes"
+ shift
+ ;;
-v | --vagrant)
VAGRANT="yes"
shift
done
}
+LOG_LEVEL="info"
HOST_SSL_PORT=443
+TESTS_DIR="tests"
-arguments $@
+arguments ${@}
# Salt's dir
## states
P_DIR="/srv/pillars"
apt-get update
-apt-get install -y curl git
+apt-get install -y curl git jq
dpkg -l |grep salt-minion
if [ ${?} -eq 0 ]; then
cat > ${S_DIR}/top.sls << EOFTSLS
base:
'*':
+ - example_single_host_host_entries
- example_add_snakeoil_certs
- locale
- nginx.passenger
base:
'*':
- arvados
+ - docker
- locale
- nginx_api_configuration
- nginx_controller_configuration
done
if [ "x${BRANCH}" != "x" ]; then
- cd ${F_DIR}/arvados-formula
- git checkout -t origin/${BRANCH}
+ cd ${F_DIR}/arvados-formula || exit 1
+ git checkout -t origin/"${BRANCH}"
cd -
fi
-# sed "s/__DOMAIN__/${DOMAIN}/g; s/__CLUSTER__/${CLUSTER}/g; s/__RELEASE__/${RELEASE}/g; s/__VERSION__/${VERSION}/g" \
-# ${CONFIG_DIR}/arvados_dev.sls > ${P_DIR}/arvados.sls
-
if [ "x${VAGRANT}" = "xyes" ]; then
SOURCE_PILLARS_DIR="/vagrant/${CONFIG_DIR}"
+ TESTS_DIR="/vagrant/${TESTS_DIR}"
else
- SOURCE_PILLARS_DIR="./${CONFIG_DIR}"
+ SOURCE_PILLARS_DIR="${SCRIPT_DIR}/${CONFIG_DIR}"
+ TESTS_DIR="${SCRIPT_DIR}/${TESTS_DIR}"
fi
-# Replace cluster and domain name in the example pillars
-for f in ${SOURCE_PILLARS_DIR}/*; do
- # sed "s/example.net/${DOMAIN}/g; s/fixme/${CLUSTER}/g" \
- sed "s/__DOMAIN__/${DOMAIN}/g;
- s/__CLUSTER__/${CLUSTER}/g;
+# Replace cluster and domain name in the example pillars and test files
+for f in "${SOURCE_PILLARS_DIR}"/*; do
+ sed "s/__CLUSTER__/${CLUSTER}/g;
+ s/__DOMAIN__/${DOMAIN}/g;
s/__RELEASE__/${RELEASE}/g;
s/__HOST_SSL_PORT__/${HOST_SSL_PORT}/g;
s/__GUEST_SSL_PORT__/${GUEST_SSL_PORT}/g;
s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g;
s/__VERSION__/${VERSION}/g" \
- ${f} > ${P_DIR}/$(basename ${f})
+ "${f}" > "${P_DIR}"/$(basename "${f}")
done
-# Let's write an /etc/hosts file that points all the hosts to localhost
-
-echo "127.0.0.2 api keep keep0 collections download ws workbench workbench2 ${CLUSTER}.${DOMAIN} api.${CLUSTER}.${DOMAIN} keep.${CLUSTER}.${DOMAIN} keep0.${CLUSTER}.${DOMAIN} collections.${CLUSTER}.${DOMAIN} download.${CLUSTER}.${DOMAIN} ws.${CLUSTER}.${DOMAIN} workbench.${CLUSTER}.${DOMAIN} workbench2.${CLUSTER}.${DOMAIN}" >> /etc/hosts
+mkdir -p /tmp/cluster_tests
+# Replace cluster and domain name in the example pillars and test files
+for f in "${TESTS_DIR}"/*; do
+ sed "s/__CLUSTER__/${CLUSTER}/g;
+ s/__DOMAIN__/${DOMAIN}/g;
+ s/__HOST_SSL_PORT__/${HOST_SSL_PORT}/g;
+ s/__INITIAL_USER__/${INITIAL_USER}/g;
+ s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
+ s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g" \
+ ${f} > /tmp/cluster_tests/$(basename ${f})
+done
+chmod 755 /tmp/cluster_tests/run-test.sh
# FIXME! #16992 Temporary fix for psql call in arvados-api-server
if [ -e /root/.psqlrc ]; then
# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
# Now run the install
-salt-call --local state.apply -l debug
+salt-call --local state.apply -l ${LOG_LEVEL}
# FIXME! #16992 Temporary fix for psql call in arvados-api-server
if [ "x${DELETE_PSQL}" = "xyes" ]; then
fi
if [ "x${RESTORE_PSQL}" = "xyes" ]; then
- echo "Restroting .psql file"
+ echo "Restoring .psql file"
mv -v /root/.psqlrc.provision.backup /root/.psqlrc
fi
# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
+
+# If running in a vagrant VM, add default user to docker group
+if [ "x${VAGRANT}" = "xyes" ]; then
+ usermod -a -G docker vagrant
+fi
+
+# Test that the installation finished correctly
+if [ "x${TEST}" = "xyes" ]; then
+ cd /tmp/cluster_tests
+ ./run-test.sh
+fi
### TOKENS
tokens:
- system_root: changeme_system_root_token
- management: changeme_management_token
- rails_secret: changeme_rails_secret_token
- anonymous_user: changeme_anonymous_user_token
+ system_root: changemesystemroottoken
+ management: changememanagementtoken
+ rails_secret: changemerailssecrettoken
+ anonymous_user: changemeanonymoususertoken
### KEYS
secrets:
- blob_signing_key: changeme_blob_signing_key
- workbench_secret_key: changeme_workbench_secret_key
- dispatcher_access_key: changeme_dispatcher_access_key
- dispatcher_secret_key: changeme_dispatcher_secret_key
- keep_access_key: changeme_keep_access_key
- keep_secret_key: changeme_keep_secret_key
+ blob_signing_key: changemeblobsigningkey
+ workbench_secret_key: changemeworkbenchsecretkey
+ dispatcher_access_key: changemedispatcheraccesskey
+ dispatcher_secret_key: changeme_dispatchersecretkey
+ keep_access_key: changemekeepaccesskey
+ keep_secret_key: changemekeepsecretkey
Login:
Test:
Controller:
ExternalURL: https://__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
InternalURLs:
- http://127.0.0.2:8003: {}
+ http://controller.internal:8003: {}
DispatchCloud:
InternalURLs:
http://__CLUSTER__.__DOMAIN__:9006: {}
Keepproxy:
ExternalURL: https://keep.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
InternalURLs:
- http://127.0.0.2:25100: {}
+ http://keep.internal:25100: {}
Keepstore:
InternalURLs:
http://keep0.__CLUSTER__.__DOMAIN__:25107: {}
RailsAPI:
InternalURLs:
- http://127.0.0.2:8004: {}
+ http://api.internal:8004: {}
WebDAV:
ExternalURL: https://collections.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
InternalURLs:
- http://127.0.0.2:9002: {}
+ http://collections.internal:9002: {}
WebDAVDownload:
ExternalURL: https://download.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
WebShell:
Websocket:
ExternalURL: wss://ws.__CLUSTER__.__DOMAIN__/websocket
InternalURLs:
- http://127.0.0.2:8005: {}
+ http://ws.internal:8005: {}
Workbench1:
ExternalURL: https://workbench.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
Workbench2:
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+docker:
+ pkg:
+ docker:
+ use_upstream: package
overwrite: true
config:
- server:
- - listen: '127.0.0.2:8004'
+ - listen: 'api.internal:8004'
- server_name: api
- root: /var/www/arvados-api/current/public
- index: index.html index.htm
default: 1
'127.0.0.0/8': 0
upstream controller_upstream:
- - server: '127.0.0.2:8003 fail_timeout=10s'
+ - server: 'controller.internal:8003 fail_timeout=10s'
### SITES
servers:
### STREAMS
http:
upstream keepproxy_upstream:
- - server: '127.0.0.2:25100 fail_timeout=10s'
+ - server: 'keep.internal:25100 fail_timeout=10s'
servers:
managed:
### STREAMS
http:
upstream collections_downloads_upstream:
- - server: '127.0.0.2:9002 fail_timeout=10s'
+ - server: 'collections.internal:9002 fail_timeout=10s'
servers:
managed:
### STREAMS
http:
upstream webshell_upstream:
- - server: '127.0.0.2:4200 fail_timeout=10s'
+ - server: 'shell.internal:4200 fail_timeout=10s'
### SITES
servers:
### STREAMS
http:
upstream websocket_upstream:
- - server: '127.0.0.2:8005 fail_timeout=10s'
+ - server: 'ws.internal:8005 fail_timeout=10s'
servers:
managed:
### STREAMS
http:
upstream workbench_upstream:
- - server: '127.0.0.2:9000 fail_timeout=10s'
+ - server: 'workbench.internal:9000 fail_timeout=10s'
### SITES
servers:
overwrite: true
config:
- server:
- - listen: '127.0.0.2:9000'
+ - listen: 'workbench.internal:9000'
- server_name: workbench
- root: /var/www/arvados-workbench/current/public
- index: index.html index.htm
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+inputfile:
+ class: File
+ path: test.txt
+hasher1_outputname: hasher1.md5sum.txt
+hasher2_outputname: hasher2.md5sum.txt
+hasher3_outputname: hasher3.md5sum.txt
--- /dev/null
+#!/usr/bin/env cwl-runner
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+
+inputs:
+ inputfile: File
+ hasher1_outputname: string
+ hasher2_outputname: string
+ hasher3_outputname: string
+
+outputs:
+ hasher_out:
+ type: File
+ outputSource: hasher3/hasher_out
+
+steps:
+ hasher1:
+ run: hasher.cwl
+ in:
+ inputfile: inputfile
+ outputname: hasher1_outputname
+ out: [hasher_out]
+ hints:
+ ResourceRequirement:
+ coresMin: 1
+ arv:IntermediateOutput:
+ outputTTL: 3600
+ arv:ReuseRequirement:
+ enableReuse: false
+
+ hasher2:
+ run: hasher.cwl
+ in:
+ inputfile: hasher1/hasher_out
+ outputname: hasher2_outputname
+ out: [hasher_out]
+ hints:
+ ResourceRequirement:
+ coresMin: 1
+ arv:IntermediateOutput:
+ outputTTL: 3600
+ arv:ReuseRequirement:
+ enableReuse: false
+
+ hasher3:
+ run: hasher.cwl
+ in:
+ inputfile: hasher2/hasher_out
+ outputname: hasher3_outputname
+ out: [hasher_out]
+ hints:
+ ResourceRequirement:
+ coresMin: 1
+ arv:IntermediateOutput:
+ outputTTL: 3600
+ arv:ReuseRequirement:
+ enableReuse: false
--- /dev/null
+#!/usr/bin/env cwl-runner
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+
+baseCommand: md5sum
+inputs:
+ inputfile:
+ type: File
+ inputBinding:
+ position: 1
+ outputname:
+ type: string
+
+stdout: $(inputs.outputname)
+
+outputs:
+ hasher_out:
+ type: File
+ outputBinding:
+ glob: $(inputs.outputname)
--- /dev/null
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+export ARVADOS_API_TOKEN=changemesystemroottoken
+export ARVADOS_API_HOST=__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+export ARVADOS_API_HOST_INSECURE=true
+
+
+# https://doc.arvados.org/v2.0/install/install-jobs-image.html
+echo "Creating Arvados Standard Docker Images project"
+uuid_prefix=$(arv --format=uuid user current | cut -d- -f1)
+project_uuid=$(arv --format=uuid group list --filters '[["name", "=", "Arvados Standard Docker Images"]]')
+
+if [ "x${project_uuid}" = "x" ]; then
+ project_uuid=$(arv --format=uuid group create --group "{\"owner_uuid\": \"${uuid_prefix}-tpzed-000000000000000\", \"group_class\":\"project\", \"name\":\"Arvados Standard Docker Images\"}")
+
+ read -rd $'\000' newlink <<EOF; arv link create --link "${newlink}"
+{
+ "tail_uuid":"${uuid_prefix}-j7d0g-fffffffffffffff",
+ "head_uuid":"${project_uuid}",
+ "link_class":"permission",
+ "name":"can_read"
+}
+EOF
+fi
+
+echo "Arvados project uuid is '${project_uuid}'"
+
+echo "Uploading arvados/jobs' docker image to the project"
+VERSION="2.1.1"
+arv-keepdocker --pull arvados/jobs "${VERSION}" --project-uuid "${project_uuid}"
+
+# Create the initial user
+echo "Creating initial user '__INITIAL_USER__'"
+user_uuid=$(arv --format=uuid user list --filters '[["email", "=", "__INITIAL_USER_EMAIL__"], ["username", "=", "__INITIAL_USER__"]]')
+
+if [ "x${user_uuid}" = "x" ]; then
+ user_uuid=$(arv --format=uuid user create --user '{"email": "__INITIAL_USER_EMAIL__", "username": "__INITIAL_USER__"}')
+ echo "Setting up user '__INITIAL_USER__'"
+ arv user setup --uuid "${user_uuid}"
+fi
+
+echo "Activating user '__INITIAL_USER__'"
+arv user update --uuid "${user_uuid}" --user '{"is_active": true}'
+
+echo "Getting the user API TOKEN"
+user_api_token=$(arv api_client_authorization list --filters "[[\"owner_uuid\", \"=\", \"${user_uuid}\"],[\"kind\", \"==\", \"arvados#apiClientAuthorization\"]]" --limit=1 |jq -r .items[].api_token)
+
+if [ "x${user_api_token}" = "x" ]; then
+ user_api_token=$(arv api_client_authorization create --api-client-authorization "{\"owner_uuid\": \"${user_uuid}\"}" | jq -r .api_token)
+fi
+
+# Change to the user's token and run the workflow
+export ARVADOS_API_TOKEN="${user_api_token}"
+
+echo "Running test CWL workflow"
+cwl-runner hasher-workflow.cwl hasher-workflow-job.yml
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+test