if current_user && !profile_config.empty?
current_user_profile = current_user.prefs[:profile]
profile_config.each do |k, entry|
- if entry['Required']
+ if entry[:Required]
if !current_user_profile ||
!current_user_profile[k] ||
current_user_profile[k].empty?
def profile
params[:offer_return_to] ||= params[:return_to]
+
+ # In a federation situation, when you get a user record using
+ # "current user of token" it can fetch a stale user record from
+ # the local cluster. So even if profile settings were just written
+ # to the user record on the login cluster (because the user just
+ # filled out the profile), those profile settings may not appear
+ # in the "current user" response because it is returning a cached
+ # record from the local cluster.
+ #
+ # In this case, explicitly fetching user record forces it to get a
+ # fresh record from the login cluster.
+ Thread.current[:user] = User.find(current_user.uuid)
end
def activity
</div>
<% profile_config.kind_of?(Array) && profile_config.andand.each do |entry| %>
- <% if entry['Key'] %>
+ <% if entry[:Key] %>
<%
show_save_button = true
- label = entry['Required'] ? '* ' : ''
- label += entry['FormFieldTitle']
- value = current_user_profile[entry['Key'].to_sym] if current_user_profile
+ label = entry[:Required] ? '* ' : ''
+ label += entry[:FormFieldTitle]
+ value = current_user_profile[entry[:Key].to_sym] if current_user_profile
%>
<div class="form-group">
- <label for="<%=entry['Key']%>"
+ <label for="<%=entry[:Key]%>"
class="col-sm-3 control-label"
- style=<%="color:red" if entry['Required']&&(!value||value.empty?)%>> <%=label%>
+ style=<%="color:red" if entry[:Required]&&(!value||value.empty?)%>> <%=label%>
</label>
- <% if entry['Type'] == 'select' %>
+ <% if entry[:Type] == 'select' %>
<div class="col-sm-8">
- <select class="form-control" name="user[prefs][profile][<%=entry['Key']%>]">
- <% entry['Options'].each do |option, _| %>
+ <select class="form-control" name="user[prefs][profile][<%=entry[:Key]%>]">
+ <% entry[:Options].each do |option, _| %>
+ <% option = option.to_s %>
<option value="<%=option%>" <%='selected' if option==value%>><%=option%></option>
<% end %>
</select>
</div>
<% else %>
<div class="col-sm-8">
- <input type="text" class="form-control" name="user[prefs][profile][<%=entry['Key']%>]" placeholder="<%=entry['FormFieldDescription']%>" value="<%=value%>" ></input>
+ <input type="text" class="form-control" name="user[prefs][profile][<%=entry[:Key]%>]" placeholder="<%=entry[:FormFieldDescription]%>" value="<%=value%>" ></input>
</div>
<% end %>
</div>
. build/run-library.sh
+# This defines python_sdk_version and cwl_runner_version with python-style
+# package suffixes (.dev/rc)
calculate_python_sdk_cwl_package_versions
-cwl_runner_version=$(echo -n $cwl_runner_version | sed s/~dev/.dev/g | sed s/~rc/rc/g)
-
set -x
docker build --no-cache --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py --build-arg pipcmd=$pipcmd -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
echo arv-keepdocker arvados/jobs $cwl_runner_version
ARVADOS_BUILDING_ITERATION="1"
fi
+# This defines python_sdk_version and cwl_runner_version with python-style
+# package suffixes (.dev/rc)
calculate_python_sdk_cwl_package_versions
-echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
-
-if [[ "${python_sdk_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
- python_sdk_version="${python_sdk_version}-1"
-else
- python_sdk_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
+if [[ -z "$cwl_runner_version" ]]; then
+ echo "ERROR: cwl_runner_version is empty";
+ exit 1
fi
-# What we use to tag the Docker image. For development and release
-# candidate packages, the OS package has a "~dev" or "~rc" suffix, but
-# Python requires a ".dev" or "rc" suffix. Arvados-cwl-runner will be
-# expecting the Python-compatible version string when it tries to pull
-# the Docker image, but --build-arg is expecting the OS package
+echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
+
+# For development and release candidate packages, the OS package has a "~dev"
+# or "~rc" suffix, but Python requires a ".dev" or "rc" suffix.
+#
+# Arvados-cwl-runner will be expecting the Python-compatible version string
+# when it tries to pull the Docker image, so we use that to tag the Docker
+# image.
+#
+# The --build-arg docker invocation arguments are expecting the OS package
# version.
-cwl_runner_version_tag=$(echo -n $cwl_runner_version | sed s/~dev/.dev/g | sed s/~rc/rc/g)
+python_sdk_version_os=$(echo -n $python_sdk_version | sed s/.dev/~dev/g | sed s/rc/~rc/g)
+cwl_runner_version_os=$(echo -n $cwl_runner_version | sed s/.dev/~dev/g | sed s/rc/~rc/g)
-if [[ -z "$cwl_runner_version_tag" ]]; then
- echo "ERROR: cwl_runner_version_tag is empty";
- exit 1
+if [[ "${python_sdk_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
+ python_sdk_version_os="${python_sdk_version_os}-1"
+else
+ python_sdk_version_os="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
fi
-if [[ "${cwl_runner_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
- cwl_runner_version="${cwl_runner_version}-1"
+if [[ "${cwl_runner_version_os}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
+ cwl_runner_version_os="${cwl_runner_version_os}-1"
else
- cwl_runner_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
+ cwl_runner_version_os="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
fi
cd docker/jobs
docker build $NOCACHE \
- --build-arg python_sdk_version=${python_sdk_version} \
- --build-arg cwl_runner_version=${cwl_runner_version} \
+ --build-arg python_sdk_version=${python_sdk_version_os} \
+ --build-arg cwl_runner_version=${cwl_runner_version_os} \
--build-arg repo_version=${REPO} \
- -t arvados/jobs:$cwl_runner_version_tag .
+ -t arvados/jobs:$cwl_runner_version .
ECODE=$?
## 20150526 nico -- *sometimes* dockerhub needs re-login
## even though credentials are already in .dockercfg
docker login -u arvados
- docker_push arvados/jobs:$cwl_runner_version_tag
+ docker_push arvados/jobs:$cwl_runner_version
title "upload arvados images finished (`timer`)"
else
title "upload arvados images SKIPPED because no --upload option set (`timer`)"
COLUMNS=80
. `dirname "$(readlink -f "$0")"`/run-library.sh
-#. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh
read -rd "\000" helpmessage <<EOF
$(basename $0): Build Arvados Python packages and Ruby gems
title "End of $gem_name gem build (`timer`)"
}
+handle_python_package () {
+ # This function assumes the current working directory is the python package directory
+ if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
+ echo "This package doesn't need rebuilding."
+ return
+ fi
+ # Make sure only to use sdist - that's the only format pip can deal with (sigh)
+ python3 setup.py $DASHQ_UNLESS_DEBUG sdist
+}
+
python_wrapper() {
local package_name="$1"; shift
local package_directory="$1"; shift
cwl_runner_version=$(cd sdk/cwl && python3 arvados_version.py)
}
-handle_python_package () {
- # This function assumes the current working directory is the python package directory
- if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
- # This package doesn't need rebuilding.
- return
- fi
- # Make sure only to use sdist - that's the only format pip can deal with (sigh)
- python setup.py $DASHQ_UNLESS_DEBUG sdist
-}
-
handle_ruby_gem() {
local gem_name="$1"; shift
local gem_version="$(nohash_version_from_git)"
done
fi
- # the python-arvados-cwl-runner package comes with cwltool, expose that version
- if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool" ]]; then
- COMMAND_ARR+=("usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool=/usr/bin/")
+ # the python3-arvados-cwl-runner package comes with cwltool, expose that version
+ if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/python-arvados-cwl-runner/bin/cwltool" ]]; then
+ COMMAND_ARR+=("usr/share/$python/dist/python-arvados-cwl-runner/bin/cwltool=/usr/bin/")
fi
COMMAND_ARR+=(".")
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-The master Arvados configuration is stored at @/etc/arvados/config.yml@
+The Arvados configuration is stored at @/etc/arvados/config.yml@
See "Migrating Configuration":config-migration.html for information about migrating from legacy component-specific configuration files.
LoginCluster: clsr1
</pre>
-The @LoginCluster@ configuration redirects all user logins to the LoginCluster, and the LoginCluster will issue API tokens which will be accepted by the federation. Users are activated or deactivated across the entire federation based on their status on the master cluster.
+The @LoginCluster@ configuration redirects all user logins to the LoginCluster, and the LoginCluster will issue API tokens which will be accepted by the federation. Users are activated or deactivated across the entire federation based on their status on the login cluster.
-Note: tokens issued by the master cluster need to be periodically re-validated when used on other clusters in the federation. The period between revalidation attempts is configured with @Login.RemoteTokenRefresh@. The default is 5 minutes. A longer period reduces overhead from validating tokens, but means it may take longer for other clusters to notice when a token has been revoked or a user has changed status (being activated/deactivated, admin flag changed).
+Note: tokens issued by the login cluster need to be periodically re-validated when used on other clusters in the federation. The period between revalidation attempts is configured with @Login.RemoteTokenRefresh@. The default is 5 minutes. A longer period reduces overhead from validating tokens, but means it may take longer for other clusters to notice when a token has been revoked or a user has changed status (being activated/deactivated, admin flag changed).
To migrate users of existing clusters with separate user databases to use a single LoginCluster, use "arv-federation-migrate":merge-remote-account.html .
<div class="releasenotes">
</notextile>
-h2(#master). development master (as of 2020-10-28)
+h2(#main). development main (as of 2020-10-28)
"Upgrading from 2.1.0":#v2_1_0
The "bucket name" is an Arvados collection uuid, portable data hash, or project uuid.
-The bucket name must be encoded as the first path segment of every request. This is what the S3 documentation calls "Path-Style Requests".
+Path-style and virtual host-style requests are supported.
+* A path-style request uses the hostname indicated by @Services.WebDAVDownload.ExternalURL@, with the bucket name in the first path segment: @https://download.example.com/zzzzz-4zz18-asdfgasdfgasdfg/@.
+* A virtual host-style request uses the hostname pattern indicated by @Services.WebDAV.ExternalURL@, with a bucket name in place of the leading @*@: @https://zzzzz-4zz18-asdfgasdfgasdfg.collections.example.com/@.
+
+If you have wildcard DNS, TLS, and routing set up, an S3 client configured with endpoint @collections.example.com@ should work regardless of which request style it uses.
h3. Supported Operations
Keep-web accepts AWS Signature Version 4 (AWS4-HMAC-SHA256) as well as the older V2 AWS signature.
-* If your client uses V4 signatures exclusively: use the Arvados token's UUID part as AccessKey, and its secret part as SecretKey. This is preferred.
-* If your client uses V2 signatures, or a combination of V2 and V4, or the Arvados token UUID is unknown: use the secret part of the Arvados token for both AccessKey and SecretKey.
+If your client uses V4 signatures exclusively _and_ your Arvados token was issued by the same cluster you are connecting to, you can use the Arvados token's UUID part as your S3 Access Key, and its secret part as your S3 Secret Key. This is preferred, where applicable.
+
+Example using cluster @zzzzz@:
+* Arvados token: @v2/zzzzz-gj3su-yyyyyyyyyyyyyyy/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
+* Access Key: @zzzzz-gj3su-yyyyyyyyyyyyyyy@
+* Secret Key: @xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
+
+In all other cases, replace every @/@ character in your Arvados token with @_@, and use the resulting string as both Access Key and Secret Key.
+
+Example using a cluster other than @zzzzz@ _or_ an S3 client that uses V2 signatures:
+* Arvados token: @v2/zzzzz-gj3su-yyyyyyyyyyyyyyy/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
+* Access Key: @v2_zzzzz-gj3su-yyyyyyyyyyyyyyy_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
+* Secret Key: @v2_zzzzz-gj3su-yyyyyyyyyyyyyyy_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
|@=@, @!=@|string, number, timestamp, or null|Equality comparison|@["tail_uuid","=","xyzzy-j7d0g-fffffffffffffff"]@ @["tail_uuid","!=",null]@|
|@<@, @<=@, @>=@, @>@|string, number, or timestamp|Ordering comparison|@["script_version",">","123"]@|
|@like@, @ilike@|string|SQL pattern match. Single character match is @_@ and wildcard is @%@. The @ilike@ operator is case-insensitive|@["script_version","like","d00220fb%"]@|
-|@in@, @not in@|array of strings|Set membership|@["script_version","in",["master","d00220fb38d4b85ca8fc28a8151702a2b9d1dec5"]]@|
+|@in@, @not in@|array of strings|Set membership|@["script_version","in",["main","d00220fb38d4b85ca8fc28a8151702a2b9d1dec5"]]@|
|@is_a@|string|Arvados object type|@["head_uuid","is_a","arvados#collection"]@|
|@exists@|string|Test if a subproperty is present.|@["properties","exists","my_subproperty"]@|
h3(#script_version). Specifying Git versions
-The script_version attribute and arvados_sdk_version runtime constraint are typically given as a branch, tag, or commit hash, but there are many more ways to specify a Git commit. The "specifying revisions" section of the "gitrevisions manual page":http://git-scm.com/docs/gitrevisions.html has a definitive list. Arvados accepts Git versions in any format listed there that names a single commit (not a tree, a blob, or a range of commits). However, some kinds of names can be expected to resolve differently in Arvados than they do in your local repository. For example, <code>HEAD@{1}</code> refers to the local reflog, and @origin/master@ typically refers to a remote branch: neither is likely to work as desired if given as a Git version.
+The script_version attribute and arvados_sdk_version runtime constraint are typically given as a branch, tag, or commit hash, but there are many more ways to specify a Git commit. The "specifying revisions" section of the "gitrevisions manual page":http://git-scm.com/docs/gitrevisions.html has a definitive list. Arvados accepts Git versions in any format listed there that names a single commit (not a tree, a blob, or a range of commits). However, some kinds of names can be expected to resolve differently in Arvados than they do in your local repository. For example, <code>HEAD@{1}</code> refers to the local reflog, and @origin/main@ typically refers to a remote branch: neither is likely to work as desired if given as a Git version.
h3. Runtime constraints
h4. Examples
-Run the script "crunch_scripts/hash.py" in the repository "you" using the "master" commit. Arvados should re-use a previous job if the script_version of the previous job is the same as the current "master" commit. This works irrespective of whether the previous job was submitted using the name "master", a different branch name or tag indicating the same commit, a SHA-1 commit hash, etc.
+Run the script "crunch_scripts/hash.py" in the repository "you" using the "main" commit. Arvados should re-use a previous job if the script_version of the previous job is the same as the current "main" commit. This works irrespective of whether the previous job was submitted using the name "main", a different branch name or tag indicating the same commit, a SHA-1 commit hash, etc.
<notextile><pre>
{
"job": {
"script": "hash.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": "c1bad4b39ca5a924e481008009d94e32+210"
}
}
</pre></notextile>
-Arvados should re-use a previous job if the "script_version" of the previous job is between "earlier_version_tag" and the "master" commit (inclusive), but not the commit indicated by "blacklisted_version_tag". If there are no previous jobs matching these criteria, run the job using the "master" commit.
+Arvados should re-use a previous job if the "script_version" of the previous job is between "earlier_version_tag" and the "main" commit (inclusive), but not the commit indicated by "blacklisted_version_tag". If there are no previous jobs matching these criteria, run the job using the "main" commit.
<notextile><pre>
{
"job": {
"script": "hash.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": "c1bad4b39ca5a924e481008009d94e32+210"
}
"job": {
"script": "hash.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": "c1bad4b39ca5a924e481008009d94e32+210"
}
}
</pre></notextile>
-Run the script "crunch_scripts/monte-carlo.py" in the repository "you/you" using the current "master" commit. Because it is marked as "nondeterministic", this job will not be considered as a suitable candidate for future job submissions that use the "find_or_create" feature.
+Run the script "crunch_scripts/monte-carlo.py" in the repository "you/you" using the current "main" commit. Because it is marked as "nondeterministic", this job will not be considered as a suitable candidate for future job submissions that use the "find_or_create" feature.
<notextile><pre>
{
"job": {
"script": "monte-carlo.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"nondeterministic": true,
"script_parameters": {
"input": "c1bad4b39ca5a924e481008009d94e32+210"
"do_hash": {
"script": "hash.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": {
"required": true,
"filter": {
"script": "0-filter.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": {
"output_of": "do_hash"
"cat_in_the_hat": {
"script": "cat.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": { }
},
"thing1": {
"script": "thing1.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": {
"output_of": "cat_in_the_hat"
"thing2": {
"script": "thing2.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": {
"output_of": "cat_in_the_hat"
"thing1": {
"script": "thing1.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": { }
},
"thing2": {
"script": "thing2.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": { }
},
"cleanup": {
"script": "cleanup.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"mess1": {
"output_of": "thing1"
h2(#introduction). Introduction
-The Keep-web server provides read/write HTTP (WebDAV) access to files stored in Keep. This makes it easy to access files in Keep from a browser, or mount Keep as a network folder using WebDAV support in various operating systems. It serves public data to unauthenticated clients, and serves private data to clients that supply Arvados API tokens. It can be installed anywhere with access to Keep services, typically behind a web proxy that provides TLS support. See the "godoc page":http://godoc.org/github.com/curoverse/arvados/services/keep-web for more detail.
+The Keep-web server provides read/write access to files stored in Keep using WebDAV and S3 protocols. This makes it easy to access files in Keep from a browser, or mount Keep as a network folder using WebDAV support in various operating systems. It serves public data to unauthenticated clients, and serves private data to clients that supply Arvados API tokens. It can be installed anywhere with access to Keep services, typically behind a web proxy that provides TLS support. See the "godoc page":http://godoc.org/github.com/curoverse/arvados/services/keep-web for more detail.
h2(#dns). Configure DNS
</code></pre>
</notextile>
+This option is preferred if you plan to access Keep using third-party S3 client software, because it accommodates S3 virtual host-style requests and path-style requests without any special client configuration.
+
h4. Under the main domain
Alternately, they can go under the main domain by including @--@:
# "Install Saltstack":#saltstack
# "Single host install using the provision.sh script":#single_host
-# "Local testing Arvados in a Vagrant box":#vagrant
# "DNS configuration":#final_steps
# "Initial user and login":#initial_user
+# "Test the installed cluster running a simple workflow":#test_install
h2(#saltstack). Install Saltstack
* User: 'admin'
* Password: 'password'
* Email: 'admin@arva2.arv.local'
+
+h2(#test_install). Test the installed cluster running a simple workflow
+
+The @provision.sh@ script saves a simple example test workflow in the @/tmp/cluster_tests@. If you want to run it, just change to that directory and run:
+
+<notextile>
+<pre><code>cd /tmp/cluster_tests
+./run-test.sh
+</code></pre>
+</notextile>
+
+It will create a test user, upload a small workflow and run it. If everything goes OK, the output should similar to this (some output was shortened for clarity):
+
+<notextile>
+<pre><code>Creating Arvados Standard Docker Images project
+Arvados project uuid is 'arva2-j7d0g-0prd8cjlk6kfl7y'
+{
+ ...
+ "uuid":"arva2-o0j2j-n4zu4cak5iifq2a",
+ "owner_uuid":"arva2-tpzed-000000000000000",
+ ...
+}
+Uploading arvados/jobs' docker image to the project
+2.1.1: Pulling from arvados/jobs
+8559a31e96f4: Pulling fs layer
+...
+Status: Downloaded newer image for arvados/jobs:2.1.1
+docker.io/arvados/jobs:2.1.1
+2020-11-23 21:43:39 arvados.arv_put[32678] INFO: Creating new cache file at /home/vagrant/.cache/arvados/arv-put/c59256eda1829281424c80f588c7cc4d
+2020-11-23 21:43:46 arvados.arv_put[32678] INFO: Collection saved as 'Docker image arvados jobs:2.1.1 sha256:0dd50'
+arva2-4zz18-1u5pvbld7cvxuy2
+Creating initial user ('admin')
+Setting up user ('admin')
+{
+ "items":[
+ {
+ ...
+ "owner_uuid":"arva2-tpzed-000000000000000",
+ ...
+ "uuid":"arva2-o0j2j-1ownrdne0ok9iox"
+ },
+ {
+ ...
+ "owner_uuid":"arva2-tpzed-000000000000000",
+ ...
+ "uuid":"arva2-o0j2j-1zbeyhcwxc1tvb7"
+ },
+ {
+ ...
+ "email":"admin@arva2.arv.local",
+ ...
+ "owner_uuid":"arva2-tpzed-000000000000000",
+ ...
+ "username":"admin",
+ "uuid":"arva2-tpzed-3wrm93zmzpshrq2",
+ ...
+ }
+ ],
+ "kind":"arvados#HashList"
+}
+Activating user 'admin'
+{
+ ...
+ "email":"admin@arva2.arv.local",
+ ...
+ "username":"admin",
+ "uuid":"arva2-tpzed-3wrm93zmzpshrq2",
+ ...
+}
+Running test CWL workflow
+INFO /usr/bin/cwl-runner 2.1.1, arvados-python-client 2.1.1, cwltool 3.0.20200807132242
+INFO Resolved 'hasher-workflow.cwl' to 'file:///tmp/cluster_tests/hasher-workflow.cwl'
+...
+INFO Using cluster arva2 (https://arva2.arv.local:8443/)
+INFO Upload local files: "test.txt"
+INFO Uploaded to ea34d971b71d5536b4f6b7d6c69dc7f6+50 (arva2-4zz18-c8uvwqdry4r8jao)
+INFO Using collection cache size 256 MiB
+INFO [container hasher-workflow.cwl] submitted container_request arva2-xvhdp-v1bkywd58gyocwm
+INFO [container hasher-workflow.cwl] arva2-xvhdp-v1bkywd58gyocwm is Final
+INFO Overall process status is success
+INFO Final output collection d6c69a88147dde9d52a418d50ef788df+123
+{
+ "hasher_out": {
+ "basename": "hasher3.md5sum.txt",
+ "class": "File",
+ "location": "keep:d6c69a88147dde9d52a418d50ef788df+123/hasher3.md5sum.txt",
+ "size": 95
+ }
+}
+INFO Final process status is success
+</code></pre>
+</notextile>
# "Vagrant":#vagrant
# "DNS configuration":#final_steps
# "Initial user and login":#initial_user
+# "Test the installed cluster running a simple workflow":#test_install
h2(#vagrant). Vagrant
* User: 'admin'
* Password: 'password'
* Email: 'admin@arva2.arv.local'
+
+h2(#test_install). Test the installed cluster running a simple workflow
+
+As documented in the <a href="{{ site.baseurl }}/install/salt-single-host.html">Single Host installation</a> page, You can run a test workflow to verify the installation finished correctly. To do so, you can follow these steps:
+
+<notextile>
+<pre><code>vagrant ssh</code></pre>
+</notextile>
+
+and once in the instance:
+
+<notextile>
+<pre><code>cd /tmp/cluster_tests
+./run-test.sh
+</code></pre>
+</notextile>
** Contains about resource consumption (RAM, cpu, disk, network) on the node while it was running
This is different from the log crunchstat.txt because it includes resource consumption of Arvados components that run on the node outside the container such as crunch-run and other processes related to the Keep file system.
-For the highest level logs, the logs are tracking the container that ran the @arvados-cwl-runner@ process which you can think of as the “mastermind” behind tracking which parts of the CWL workflow need to be run when, which have been run already, what order they need to be run, which can be run simultaneously, and so forth and then sending out the related container requests. Each step then has their own logs related to containers running a CWL step of the workflow including a log of standard error that contains the standard error of the code run in that CWL step. Those logs can be found by expanding the steps and clicking on the link to the log collection.
+For the highest level logs, the logs are tracking the container that ran the @arvados-cwl-runner@ process which you can think of as the “workflow runner”. It tracks which parts of the CWL workflow need to be run when, which have been run already, what order they need to be run, which can be run simultaneously, and so forth and then creates the necessary container requests. Each step has its own logs related to containers running a CWL step of the workflow including a log of standard error that contains the standard error of the code run in that CWL step. Those logs can be found by expanding the steps and clicking on the link to the log collection.
-Let’s take a peek at a few of these logs to get you more familiar with them. First, we can look at the @stderr.txt@ of the highest level process. Again recall this should be of the “mastermind” @arvados-cwl-runner@ process. You can click on the log to download it to your local machine, and when you look at the contents - you should see something like the following...
+Let’s take a peek at a few of these logs to get you more familiar with them. First, we can look at the @stderr.txt@ of the highest level process. Again recall this should be of the “workflow runner” @arvados-cwl-runner@ process. You can click on the log to download it to your local machine, and when you look at the contents - you should see something like the following...
<pre><code>2020-06-22T20:30:04.737703197Z INFO /usr/bin/arvados-cwl-runner 2.0.3, arvados-python-client 2.0.3, cwltool 1.0.20190831161204
2020-06-22T20:30:04.743250012Z INFO Resolved '/var/lib/cwl/workflow.json#main' to 'file:///var/lib/cwl/workflow.json#main'
if err != nil {
return fmt.Errorf("user.Lookup(\"postgres\"): %s", err)
}
- postgresUid, err := strconv.Atoi(postgresUser.Uid)
+ postgresUID, err := strconv.Atoi(postgresUser.Uid)
if err != nil {
return fmt.Errorf("user.Lookup(\"postgres\"): non-numeric uid?: %q", postgresUser.Uid)
}
if err != nil {
return err
}
- err = os.Chown(datadir, postgresUid, 0)
+ err = os.Chown(datadir, postgresUID, 0)
if err != nil {
return err
}
crString, ok := request["container_request"].(string)
if ok {
- var crJson map[string]interface{}
- err := json.Unmarshal([]byte(crString), &crJson)
+ var crJSON map[string]interface{}
+ err := json.Unmarshal([]byte(crString), &crJSON)
if err != nil {
httpserver.Error(w, err.Error(), http.StatusBadRequest)
return true
}
- request["container_request"] = crJson
+ request["container_request"] = crJSON
}
containerRequest, ok := request["container_request"].(map[string]interface{})
if options.BypassFederation {
return conn.local.UserUpdate(ctx, options)
}
- return conn.chooseBackend(options.UUID).UserUpdate(ctx, options)
+ resp, err := conn.chooseBackend(options.UUID).UserUpdate(ctx, options)
+ if err != nil {
+ return resp, err
+ }
+ if !strings.HasPrefix(options.UUID, conn.cluster.ClusterID) {
+ // Copy the updated user record to the local cluster
+ err = conn.batchUpdateUsers(ctx, arvados.ListOptions{}, []arvados.User{resp})
+ if err != nil {
+ return arvados.User{}, err
+ }
+ }
+ return resp, err
}
func (conn *Conn) UserUpdateUUID(ctx context.Context, options arvados.UpdateUUIDOptions) (arvados.User, error) {
"git.arvados.org/arvados.git/sdk/go/health"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"github.com/jmoiron/sqlx"
+ // sqlx needs lib/pq to talk to PostgreSQL
_ "github.com/lib/pq"
)
"bytes"
"context"
"encoding/json"
+ "fmt"
"io"
"io/ioutil"
"math"
"net/http"
"net/url"
"os"
+ "os/exec"
"path/filepath"
+ "strconv"
+ "strings"
"git.arvados.org/arvados.git/lib/boot"
"git.arvados.org/arvados.git/lib/config"
c.Check(coll.PortableDataHash, check.Equals, pdh)
}
+func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) {
+ if _, err := exec.LookPath("s3cmd"); err != nil {
+ c.Skip("s3cmd not in PATH")
+ return
+ }
+
+ testText := "IntegrationSuite.TestS3WithFederatedToken"
+
+ conn1 := s.conn("z1111")
+ rootctx1, _, _ := s.rootClients("z1111")
+ userctx1, ac1, _, _ := s.userClients(rootctx1, c, conn1, "z1111", true)
+ conn3 := s.conn("z3333")
+
+ createColl := func(clusterID string) arvados.Collection {
+ _, ac, kc := s.clientsWithToken(clusterID, ac1.AuthToken)
+ var coll arvados.Collection
+ fs, err := coll.FileSystem(ac, kc)
+ c.Assert(err, check.IsNil)
+ f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = io.WriteString(f, testText)
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+ mtxt, err := fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ coll, err = s.conn(clusterID).CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
+ "manifest_text": mtxt,
+ }})
+ c.Assert(err, check.IsNil)
+ return coll
+ }
+
+ for _, trial := range []struct {
+ clusterID string // create the collection on this cluster (then use z3333 to access it)
+ token string
+ }{
+ // Try the hardest test first: z3333 hasn't seen
+ // z1111's token yet, and we're just passing the
+ // opaque secret part, so z3333 has to guess that it
+ // belongs to z1111.
+ {"z1111", strings.Split(ac1.AuthToken, "/")[2]},
+ {"z3333", strings.Split(ac1.AuthToken, "/")[2]},
+ {"z1111", strings.Replace(ac1.AuthToken, "/", "_", -1)},
+ {"z3333", strings.Replace(ac1.AuthToken, "/", "_", -1)},
+ } {
+ c.Logf("================ %v", trial)
+ coll := createColl(trial.clusterID)
+
+ cfgjson, err := conn3.ConfigGet(userctx1)
+ c.Assert(err, check.IsNil)
+ var cluster arvados.Cluster
+ err = json.Unmarshal(cfgjson, &cluster)
+ c.Assert(err, check.IsNil)
+
+ c.Logf("TokenV2 is %s", ac1.AuthToken)
+ host := cluster.Services.WebDAV.ExternalURL.Host
+ s3args := []string{
+ "--ssl", "--no-check-certificate",
+ "--host=" + host, "--host-bucket=" + host,
+ "--access_key=" + trial.token, "--secret_key=" + trial.token,
+ }
+ buf, err := exec.Command("s3cmd", append(s3args, "ls", "s3://"+coll.UUID)...).CombinedOutput()
+ c.Check(err, check.IsNil)
+ c.Check(string(buf), check.Matches, `.* `+fmt.Sprintf("%d", len(testText))+` +s3://`+coll.UUID+`/test.txt\n`)
+
+ buf, err = exec.Command("s3cmd", append(s3args, "get", "s3://"+coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput()
+ // Command fails because we don't return Etag header.
+ // c.Check(err, check.IsNil)
+ flen := strconv.Itoa(len(testText))
+ c.Check(string(buf), check.Matches, `(?ms).*`+flen+` of `+flen+`.*`)
+ }
+}
+
func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) {
conn1 := s.conn("z1111")
conn3 := s.conn("z3333")
var _ = check.Suite(&RPCSuite{})
-const contextKeyTestTokens = "testTokens"
+type key int
+
+const (
+ contextKeyTestTokens key = iota
+)
type RPCSuite struct {
log logrus.FieldLogger
var pi procinfo
err = json.NewDecoder(f).Decode(&pi)
if err != nil {
- return fmt.Errorf("decode %s: %s\n", path, err)
+ return fmt.Errorf("decode %s: %s", path, err)
}
if pi.UUID != uuid || pi.PID == 0 {
}
for bind := range runner.SecretMounts {
if _, ok := runner.Container.Mounts[bind]; ok {
- return fmt.Errorf("Secret mount %q conflicts with regular mount", bind)
+ return fmt.Errorf("secret mount %q conflicts with regular mount", bind)
}
if runner.SecretMounts[bind].Kind != "json" &&
runner.SecretMounts[bind].Kind != "text" {
- return fmt.Errorf("Secret mount %q type is %q but only 'json' and 'text' are permitted.",
+ return fmt.Errorf("secret mount %q type is %q but only 'json' and 'text' are permitted",
bind, runner.SecretMounts[bind].Kind)
}
binds = append(binds, bind)
if bind == "stdout" || bind == "stderr" {
// Is it a "file" mount kind?
if mnt.Kind != "file" {
- return fmt.Errorf("Unsupported mount kind '%s' for %s. Only 'file' is supported.", mnt.Kind, bind)
+ return fmt.Errorf("unsupported mount kind '%s' for %s: only 'file' is supported", mnt.Kind, bind)
}
// Does path start with OutputPath?
if bind == "stdin" {
// Is it a "collection" mount kind?
if mnt.Kind != "collection" && mnt.Kind != "json" {
- return fmt.Errorf("Unsupported mount kind '%s' for stdin. Only 'collection' or 'json' are supported.", mnt.Kind)
+ return fmt.Errorf("unsupported mount kind '%s' for stdin: only 'collection' and 'json' are supported", mnt.Kind)
}
}
if strings.HasPrefix(bind, runner.Container.OutputPath+"/") && bind != runner.Container.OutputPath+"/" {
if mnt.Kind != "collection" && mnt.Kind != "text" && mnt.Kind != "json" {
- return fmt.Errorf("Only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path for %q, was %q", bind, mnt.Kind)
+ return fmt.Errorf("only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path for %q, was %q", bind, mnt.Kind)
}
}
case mnt.Kind == "collection" && bind != "stdin":
var src string
if mnt.UUID != "" && mnt.PortableDataHash != "" {
- return fmt.Errorf("Cannot specify both 'uuid' and 'portable_data_hash' for a collection mount")
+ return fmt.Errorf("cannot specify both 'uuid' and 'portable_data_hash' for a collection mount")
}
if mnt.UUID != "" {
if mnt.Writable {
- return fmt.Errorf("Writing to existing collections currently not permitted.")
+ return fmt.Errorf("writing to existing collections currently not permitted")
}
pdhOnly = false
src = fmt.Sprintf("%s/by_id/%s", runner.ArvMountPoint, mnt.UUID)
} else if mnt.PortableDataHash != "" {
if mnt.Writable && !strings.HasPrefix(bind, runner.Container.OutputPath+"/") {
- return fmt.Errorf("Can never write to a collection specified by portable data hash")
+ return fmt.Errorf("can never write to a collection specified by portable data hash")
}
idx := strings.Index(mnt.PortableDataHash, "/")
if idx > 0 {
var tmpdir string
tmpdir, err = runner.MkTempDir(runner.parentTemp, "tmp")
if err != nil {
- return fmt.Errorf("While creating mount temp dir: %v", err)
+ return fmt.Errorf("while creating mount temp dir: %v", err)
}
st, staterr := os.Stat(tmpdir)
if staterr != nil {
- return fmt.Errorf("While Stat on temp dir: %v", staterr)
+ return fmt.Errorf("while Stat on temp dir: %v", staterr)
}
err = os.Chmod(tmpdir, st.Mode()|os.ModeSetgid|0777)
if staterr != nil {
- return fmt.Errorf("While Chmod temp dir: %v", err)
+ return fmt.Errorf("while Chmod temp dir: %v", err)
}
runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", tmpdir, bind))
if bind == runner.Container.OutputPath {
}
if runner.HostOutputDir == "" {
- return fmt.Errorf("Output path does not correspond to a writable mount point")
+ return fmt.Errorf("output path does not correspond to a writable mount point")
}
if wantAPI := runner.Container.RuntimeConstraints.API; needCertMount && wantAPI != nil && *wantAPI {
runner.ArvMount, err = runner.RunArvMount(arvMountCmd, token)
if err != nil {
- return fmt.Errorf("While trying to start arv-mount: %v", err)
+ return fmt.Errorf("while trying to start arv-mount: %v", err)
}
for _, p := range collectionPaths {
_, err = os.Stat(p)
if err != nil {
- return fmt.Errorf("While checking that input files exist: %v", err)
+ return fmt.Errorf("while checking that input files exist: %v", err)
}
}
for _, cp := range copyFiles {
st, err := os.Stat(cp.src)
if err != nil {
- return fmt.Errorf("While staging writable file from %q to %q: %v", cp.src, cp.bind, err)
+ return fmt.Errorf("while staging writable file from %q to %q: %v", cp.src, cp.bind, err)
}
if st.IsDir() {
err = filepath.Walk(cp.src, func(walkpath string, walkinfo os.FileInfo, walkerr error) error {
}
return os.Chmod(target, walkinfo.Mode()|os.ModeSetgid|0777)
} else {
- return fmt.Errorf("Source %q is not a regular file or directory", cp.src)
+ return fmt.Errorf("source %q is not a regular file or directory", cp.src)
}
})
} else if st.Mode().IsRegular() {
}
}
if err != nil {
- return fmt.Errorf("While staging writable file from %q to %q: %v", cp.src, cp.bind, err)
+ return fmt.Errorf("while staging writable file from %q to %q: %v", cp.src, cp.bind, err)
}
}
err := cr.SetupMounts()
c.Check(err, NotNil)
- c.Check(err, ErrorMatches, `Only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
+ c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
checkEmpty()
err := cr.SetupMounts()
c.Check(err, NotNil)
- c.Check(err, ErrorMatches, `Unsupported mount kind 'tmp' for stdin.*`)
+ c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
checkEmpty()
}`, func(t *TestDockerClient) {})
c.Check(err, NotNil)
- c.Check(strings.Contains(err.Error(), "Unsupported mount kind 'tmp' for stdout"), Equals, true)
+ c.Check(strings.Contains(err.Error(), "unsupported mount kind 'tmp' for stdout"), Equals, true)
}
func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
}`, func(t *TestDockerClient) {})
c.Check(err, NotNil)
- c.Check(strings.Contains(err.Error(), "Unsupported mount kind 'collection' for stdout"), Equals, true)
+ c.Check(strings.Contains(err.Error(), "unsupported mount kind 'collection' for stdout"), Equals, true)
}
func (s *TestSuite) TestFullRunWithAPI(c *C) {
"git.arvados.org/arvados.git/lib/controller/api"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"github.com/jmoiron/sqlx"
+ // sqlx needs lib/pq to talk to PostgreSQL
_ "github.com/lib/pq"
)
DataDirectory string
LogFile string
}
- if pg_lsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil {
+ if pgLsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil {
err = fmt.Errorf("pg_lsclusters: %s", err2)
return 1
- } else if pgclusters := strings.Split(strings.TrimSpace(string(pg_lsclusters)), "\n"); len(pgclusters) != 1 {
+ } else if pgclusters := strings.Split(strings.TrimSpace(string(pgLsclusters)), "\n"); len(pgclusters) != 1 {
logger.Warnf("pg_lsclusters returned %d postgresql clusters -- skipping postgresql initdb/startup, hope that's ok", len(pgclusters))
} else if _, err = fmt.Sscanf(pgclusters[0], "%s %s %d %s %s %s %s", &pgc.Version, &pgc.Cluster, &pgc.Port, &pgc.Status, &pgc.Owner, &pgc.DataDirectory, &pgc.LogFile); err != nil {
err = fmt.Errorf("error parsing pg_lsclusters output: %s", err)
"io"
"log"
"net/http"
+ // pprof is only imported to register its HTTP handlers
_ "net/http/pprof"
"os"
. /usr/src/arvados/build/run-library.sh
TMPHERE=\$(pwd)
cd /usr/src/arvados
+
+ # This defines python_sdk_version and cwl_runner_version with python-style
+ # package suffixes (.dev/rc)
calculate_python_sdk_cwl_package_versions
- cwl_runner_version=\$(echo -n \$cwl_runner_version | sed s/~dev/.dev/g | sed s/~rc/rc/g)
cd \$TMPHERE
set -u
"git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"github.com/jmoiron/sqlx"
+ // sqlx needs lib/pq to talk to PostgreSQL
_ "github.com/lib/pq"
"gopkg.in/check.v1"
)
func getStackTrace() string {
buf := make([]byte, 1000)
- bytes_written := runtime.Stack(buf, false)
- return "Stack Trace:\n" + string(buf[:bytes_written])
+ bytesWritten := runtime.Stack(buf, false)
+ return "Stack Trace:\n" + string(buf[:bytesWritten])
}
func expectEqual(t *testing.T, actual interface{}, expected interface{}) {
// Reads from the underlying reader, update the hashing function, and
// pass the results through. Returns BadChecksum (instead of EOF) on
// the last read if the checksum doesn't match.
-func (this HashCheckingReader) Read(p []byte) (n int, err error) {
- n, err = this.Reader.Read(p)
+func (hcr HashCheckingReader) Read(p []byte) (n int, err error) {
+ n, err = hcr.Reader.Read(p)
if n > 0 {
- this.Hash.Write(p[:n])
+ hcr.Hash.Write(p[:n])
}
if err == io.EOF {
- sum := this.Hash.Sum(nil)
- if fmt.Sprintf("%x", sum) != this.Check {
+ sum := hcr.Hash.Sum(nil)
+ if fmt.Sprintf("%x", sum) != hcr.Check {
err = BadChecksum
}
}
return n, err
}
-// WriteTo writes the entire contents of this.Reader to dest. Returns
+// WriteTo writes the entire contents of hcr.Reader to dest. Returns
// BadChecksum if writing is successful but the checksum doesn't
// match.
-func (this HashCheckingReader) WriteTo(dest io.Writer) (written int64, err error) {
- if writeto, ok := this.Reader.(io.WriterTo); ok {
- written, err = writeto.WriteTo(io.MultiWriter(dest, this.Hash))
+func (hcr HashCheckingReader) WriteTo(dest io.Writer) (written int64, err error) {
+ if writeto, ok := hcr.Reader.(io.WriterTo); ok {
+ written, err = writeto.WriteTo(io.MultiWriter(dest, hcr.Hash))
} else {
- written, err = io.Copy(io.MultiWriter(dest, this.Hash), this.Reader)
+ written, err = io.Copy(io.MultiWriter(dest, hcr.Hash), hcr.Reader)
}
if err != nil {
return written, err
}
- sum := this.Hash.Sum(nil)
- if fmt.Sprintf("%x", sum) != this.Check {
+ sum := hcr.Hash.Sum(nil)
+ if fmt.Sprintf("%x", sum) != hcr.Check {
return written, BadChecksum
}
// Close reads all remaining data from the underlying Reader and
// returns BadChecksum if the checksum doesn't match. It also closes
// the underlying Reader if it implements io.ReadCloser.
-func (this HashCheckingReader) Close() (err error) {
- _, err = io.Copy(this.Hash, this.Reader)
+func (hcr HashCheckingReader) Close() (err error) {
+ _, err = io.Copy(hcr.Hash, hcr.Reader)
- if closer, ok := this.Reader.(io.Closer); ok {
+ if closer, ok := hcr.Reader.(io.Closer); ok {
closeErr := closer.Close()
if err == nil {
err = closeErr
if err != nil {
return err
}
- if fmt.Sprintf("%x", this.Hash.Sum(nil)) != this.Check {
+ if fmt.Sprintf("%x", hcr.Hash.Sum(nil)) != hcr.Check {
return BadChecksum
}
return nil
type StubPutHandler struct {
c *C
expectPath string
- expectApiToken string
+ expectAPIToken string
expectBody string
expectStorageClass string
handled chan string
func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
- sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
+ sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectAPIToken))
sph.c.Check(req.Header.Get("X-Keep-Storage-Classes"), Equals, sph.expectStorageClass)
body, err := ioutil.ReadAll(req.Body)
sph.c.Check(err, Equals, nil)
func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
ks = make([]KeepServer, n)
- for i := 0; i < n; i += 1 {
+ for i := 0; i < n; i++ {
ks[i] = RunFakeKeepServer(st)
}
type StubGetHandler struct {
c *C
expectPath string
- expectApiToken string
+ expectAPIToken string
httpStatus int
body []byte
}
func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath)
- sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectApiToken))
+ sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectAPIToken))
resp.WriteHeader(sgh.httpStatus)
resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body)))
resp.Write(sgh.body)
response string
}
-func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Reader,
+func (kc *KeepClient) uploadToKeepServer(host string, hash string, body io.Reader,
uploadStatusChan chan<- uploadStatus, expectedLength int64, reqid string) {
var req *http.Request
}
req.Header.Add("X-Request-Id", reqid)
- req.Header.Add("Authorization", "OAuth2 "+this.Arvados.ApiToken)
+ req.Header.Add("Authorization", "OAuth2 "+kc.Arvados.ApiToken)
req.Header.Add("Content-Type", "application/octet-stream")
- req.Header.Add(XKeepDesiredReplicas, fmt.Sprint(this.Want_replicas))
- if len(this.StorageClasses) > 0 {
- req.Header.Add("X-Keep-Storage-Classes", strings.Join(this.StorageClasses, ", "))
+ req.Header.Add(XKeepDesiredReplicas, fmt.Sprint(kc.Want_replicas))
+ if len(kc.StorageClasses) > 0 {
+ req.Header.Add("X-Keep-Storage-Classes", strings.Join(kc.StorageClasses, ", "))
}
var resp *http.Response
- if resp, err = this.httpClient().Do(req); err != nil {
+ if resp, err = kc.httpClient().Do(req); err != nil {
DebugPrintf("DEBUG: [%s] Upload failed %v error: %v", reqid, url, err.Error())
uploadStatusChan <- uploadStatus{err, url, 0, 0, err.Error()}
return
}
}
-func (this *KeepClient) putReplicas(
+func (kc *KeepClient) putReplicas(
hash string,
getReader func() io.Reader,
expectedLength int64) (locator string, replicas int, err error) {
- reqid := this.getRequestID()
+ reqid := kc.getRequestID()
// Calculate the ordering for uploading to servers
- sv := NewRootSorter(this.WritableLocalRoots(), hash).GetSortedRoots()
+ sv := NewRootSorter(kc.WritableLocalRoots(), hash).GetSortedRoots()
// The next server to try contacting
nextServer := 0
}()
replicasDone := 0
- replicasTodo := this.Want_replicas
+ replicasTodo := kc.Want_replicas
- replicasPerThread := this.replicasPerService
+ replicasPerThread := kc.replicasPerService
if replicasPerThread < 1 {
// unlimited or unknown
replicasPerThread = replicasTodo
}
- retriesRemaining := 1 + this.Retries
+ retriesRemaining := 1 + kc.Retries
var retryServers []string
lastError := make(map[string]string)
// Start some upload requests
if nextServer < len(sv) {
DebugPrintf("DEBUG: [%s] Begin upload %s to %s", reqid, hash, sv[nextServer])
- go this.uploadToKeepServer(sv[nextServer], hash, getReader(), uploadStatusChan, expectedLength, reqid)
+ go kc.uploadToKeepServer(sv[nextServer], hash, getReader(), uploadStatusChan, expectedLength, reqid)
nextServer++
active++
} else {
token_uuid = ''
secret = token
+ stored_secret = nil # ...if different from secret
optional = nil
case token[0..2]
# below. If so, we'll stuff the database with hmac instead of
# the real OIDC token.
upstream_cluster_id = Rails.configuration.Login.LoginCluster
- token_uuid = upstream_cluster_id + generate_uuid[5..27]
- secret = hmac
+ stored_secret = hmac
else
return nil
end
remote_user_prefix = remote_user['uuid'][0..4]
+ if token_uuid == ''
+ # Use the same UUID as the remote when caching the token.
+ begin
+ remote_token = SafeJSON.load(
+ clnt.get_content('https://' + host + '/arvados/v1/api_client_authorizations/current',
+ {'remote' => Rails.configuration.ClusterID},
+ {'Authorization' => 'Bearer ' + token}))
+ token_uuid = remote_token['uuid']
+ if !token_uuid.match(HasUuid::UUID_REGEX) || token_uuid[0..4] != upstream_cluster_id
+ raise "remote cluster #{upstream_cluster_id} returned invalid token uuid #{token_uuid.inspect}"
+ end
+ rescue => e
+ Rails.logger.warn "error getting remote token details for #{token.inspect}: #{e}"
+ return nil
+ end
+ end
+
# Clusters can only authenticate for their own users.
if remote_user_prefix != upstream_cluster_id
Rails.logger.warn "remote authentication rejected: claimed remote user #{remote_user_prefix} but token was issued by #{upstream_cluster_id}"
auth.user = user
auth.api_client_id = 0
end
+ # If stored_secret is set, we save stored_secret in the database
+ # but return the real secret to the caller. This way, if we end
+ # up returning the auth record to the client, they see the same
+ # secret they supplied, instead of the HMAC we saved in the
+ # database.
+ stored_secret = stored_secret || secret
auth.update_attributes!(user: user,
- api_token: secret,
+ api_token: stored_secret,
api_client_id: 0,
expires_at: Time.now + Rails.configuration.Login.RemoteTokenRefresh)
- Rails.logger.debug "cached remote token #{token_uuid} with secret #{secret} in local db"
+ Rails.logger.debug "cached remote token #{token_uuid} with secret #{stored_secret} in local db"
+ auth.api_token = secret
return auth
end
// Cancelled or Complete. See https://dev.arvados.org/issues/10979
func (disp *Dispatcher) checkSqueueForOrphans() {
for _, uuid := range disp.sqCheck.All() {
- if !containerUuidPattern.MatchString(uuid) {
+ if !containerUuidPattern.MatchString(uuid) || !strings.HasPrefix(uuid, disp.cluster.ClusterID) {
continue
}
err := disp.TrackContainer(uuid)
"net/url"
"os"
"path/filepath"
+ "regexp"
"sort"
"strconv"
"strings"
return hashdigest(hmac.New(sha256.New, key), stringToSign), nil
}
+var v2tokenUnderscore = regexp.MustCompile(`^v2_[a-z0-9]{5}-gj3su-[a-z0-9]{15}_`)
+
+func unescapeKey(key string) string {
+ if v2tokenUnderscore.MatchString(key) {
+ // Entire Arvados token, with "/" replaced by "_" to
+ // avoid colliding with the Authorization header
+ // format.
+ return strings.Replace(key, "_", "/", -1)
+ } else if s, err := url.PathUnescape(key); err == nil {
+ return s
+ } else {
+ return key
+ }
+}
+
// checks3signature verifies the given S3 V4 signature and returns the
// Arvados token that corresponds to the given accessKey. An error is
// returned if accessKey is not a valid token UUID or the signature
} else {
// Access key and secret key are both an entire
// Arvados token or OIDC access token.
- ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+key)
+ ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+unescapeKey(key))
err = client.RequestAndDecodeContext(ctx, &aca, "GET", "arvados/v1/api_client_authorizations/current", nil, nil)
secret = key
}
} else if expect != signature {
return "", fmt.Errorf("signature does not match (scope %q signedHeaders %q stringToSign %q)", scope, signedHeaders, stringToSign)
}
- return secret, nil
+ return aca.TokenV2(), nil
}
func s3ErrorResponse(w http.ResponseWriter, s3code string, message string, resource string, code int) {
s3ErrorResponse(w, InvalidRequest, "malformed Authorization header", r.URL.Path, http.StatusUnauthorized)
return true
}
- token = split[0]
+ token = unescapeKey(split[0])
} else if strings.HasPrefix(auth, s3SignAlgorithm+" ") {
t, err := h.checks3signature(r)
if err != nil {
fs := client.SiteFileSystem(kc)
fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
- objectNameGiven := strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
+ var objectNameGiven bool
+ fspath := "/by_id"
+ if id := parseCollectionIDFromDNSName(r.Host); id != "" {
+ fspath += "/" + id
+ objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 0
+ } else {
+ objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
+ }
+ fspath += r.URL.Path
switch {
case r.Method == http.MethodGet && !objectNameGiven:
}
return true
case r.Method == http.MethodGet || r.Method == http.MethodHead:
- fspath := "/by_id" + r.URL.Path
fi, err := fs.Stat(fspath)
if r.Method == "HEAD" && !objectNameGiven {
// HeadBucket
s3ErrorResponse(w, InvalidArgument, "Missing object name in PUT request.", r.URL.Path, http.StatusBadRequest)
return true
}
- fspath := "by_id" + r.URL.Path
var objectIsDir bool
if strings.HasSuffix(fspath, "/") {
if !h.Config.cluster.Collections.S3FolderObjects {
s3ErrorResponse(w, InvalidArgument, "missing object name in DELETE request", r.URL.Path, http.StatusBadRequest)
return true
}
- fspath := "by_id" + r.URL.Path
if strings.HasSuffix(fspath, "/") {
fspath = strings.TrimSuffix(fspath, "/")
fi, err := fs.Stat(fspath)
"fmt"
"io/ioutil"
"net/http"
+ "net/url"
"os"
"os/exec"
"strings"
secretkey string
}{
{true, aws.V2Signature, arvadostest.ActiveToken, "none"},
+ {true, aws.V2Signature, url.QueryEscape(arvadostest.ActiveTokenV2), "none"},
+ {true, aws.V2Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), "none"},
{false, aws.V2Signature, "none", "none"},
{false, aws.V2Signature, "none", arvadostest.ActiveToken},
{true, aws.V4Signature, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken},
{true, aws.V4Signature, arvadostest.ActiveToken, arvadostest.ActiveToken},
+ {true, aws.V4Signature, url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2)},
+ {true, aws.V4Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1)},
{false, aws.V4Signature, arvadostest.ActiveToken, ""},
{false, aws.V4Signature, arvadostest.ActiveToken, "none"},
{false, aws.V4Signature, "none", arvadostest.ActiveToken},
c.Check(err, check.IsNil)
c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
}
+
+func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) {
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+
+ hdr, body, _ := s.runCurl(c, "AWS "+arvadostest.ActiveTokenV2+":none", stage.coll.UUID+".collections.example.com", "/sailboat.txt")
+ c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
+ c.Check(body, check.Equals, "⛵\n")
+}
}
// Return header block and body.
-func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
+func (s *IntegrationSuite) runCurl(c *check.C, auth, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
curlArgs := []string{"--silent", "--show-error", "--include"}
testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
- if token != "" {
- curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token)
+ if strings.Contains(auth, " ") {
+ // caller supplied entire Authorization header value
+ curlArgs = append(curlArgs, "-H", "Authorization: "+auth)
+ } else if auth != "" {
+ // caller supplied Arvados token
+ curlArgs = append(curlArgs, "-H", "Authorization: Bearer "+auth)
}
curlArgs = append(curlArgs, args...)
curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
config.vm.define "arvados" do |arv|
arv.vm.box = "bento/debian-10"
- arv.vm.hostname = "arva2.arv.local"
+ arv.vm.hostname = "vagrant.local"
+ # CPU/RAM
+ config.vm.provider :virtualbox do |v|
+ v.memory = 2048
+ v.cpus = 2
+ end
+
# Networking
arv.vm.network "forwarded_port", guest: 8443, host: 8443
arv.vm.network "forwarded_port", guest: 25100, host: 25100
arv.vm.network "forwarded_port", guest: 8001, host: 8001
arv.vm.network "forwarded_port", guest: 8000, host: 8000
arv.vm.network "forwarded_port", guest: 3001, host: 3001
- # config.vm.network "private_network", ip: "192.168.33.10"
- # arv.vm.synced_folder "salt_pillars", "/srv/pillars",
- # create: true
arv.vm.provision "shell",
path: "provision.sh",
args: [
+ "--test",
"--vagrant",
"--ssl-port=8443"
].join(" ")
-#!/bin/bash -x
+#!/bin/bash
# Copyright (C) The Arvados Authors. All rights reserved.
#
set -o pipefail
+# capture the directory that the script is running from
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
usage() {
echo >&2
- echo >&2 "Usage: $0 [-h] [-h]"
+ echo >&2 "Usage: ${0} [-h] [-h]"
echo >&2
- echo >&2 "$0 options:"
- echo >&2 " -v, --vagrant Run in vagrant and use the /vagrant shared dir"
+ echo >&2 "${0} options:"
+ echo >&2 " -d, --debug Run salt installation in debug mode"
echo >&2 " -p <N>, --ssl-port <N> SSL port to use for the web applications"
+ echo >&2 " -t, --test Test installation running a CWL workflow"
echo >&2 " -h, --help Display this help and exit"
+ echo >&2 " -v, --vagrant Run in vagrant and use the /vagrant shared dir"
echo >&2
}
arguments() {
# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
- TEMP=`getopt -o hvp: \
- --long help,vagrant,ssl-port: \
- -n "$0" -- "$@"`
+ TEMP=$(getopt -o dhp:tv \
+ --long debug,help,ssl-port:,test,vagrant \
+ -n "${0}" -- "${@}")
- if [ $? != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi
+ if [ ${?} != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi
# Note the quotes around `$TEMP': they are essential!
eval set -- "$TEMP"
- while [ $# -ge 1 ]; do
- case $1 in
+ while [ ${#} -ge 1 ]; do
+ case ${1} in
+ -d | --debug)
+ LOG_LEVEL="debug"
+ shift
+ ;;
+ -t | --test)
+ TEST="yes"
+ shift
+ ;;
-v | --vagrant)
VAGRANT="yes"
shift
done
}
+LOG_LEVEL="info"
HOST_SSL_PORT=443
+TESTS_DIR="tests"
-arguments $@
+arguments ${@}
# Salt's dir
## states
P_DIR="/srv/pillars"
apt-get update
-apt-get install -y curl git
+apt-get install -y curl git jq
dpkg -l |grep salt-minion
if [ ${?} -eq 0 ]; then
cat > ${S_DIR}/top.sls << EOFTSLS
base:
'*':
+ - example_single_host_host_entries
- example_add_snakeoil_certs
- locale
- nginx.passenger
base:
'*':
- arvados
+ - docker
- locale
- nginx_api_configuration
- nginx_controller_configuration
done
if [ "x${BRANCH}" != "x" ]; then
- cd ${F_DIR}/arvados-formula
- git checkout -t origin/${BRANCH}
+ cd ${F_DIR}/arvados-formula || exit 1
+ git checkout -t origin/"${BRANCH}"
cd -
fi
-# sed "s/__DOMAIN__/${DOMAIN}/g; s/__CLUSTER__/${CLUSTER}/g; s/__RELEASE__/${RELEASE}/g; s/__VERSION__/${VERSION}/g" \
-# ${CONFIG_DIR}/arvados_dev.sls > ${P_DIR}/arvados.sls
-
if [ "x${VAGRANT}" = "xyes" ]; then
SOURCE_PILLARS_DIR="/vagrant/${CONFIG_DIR}"
+ TESTS_DIR="/vagrant/${TESTS_DIR}"
else
- SOURCE_PILLARS_DIR="./${CONFIG_DIR}"
+ SOURCE_PILLARS_DIR="${SCRIPT_DIR}/${CONFIG_DIR}"
+ TESTS_DIR="${SCRIPT_DIR}/${TESTS_DIR}"
fi
-# Replace cluster and domain name in the example pillars
-for f in ${SOURCE_PILLARS_DIR}/*; do
- # sed "s/example.net/${DOMAIN}/g; s/fixme/${CLUSTER}/g" \
- sed "s/__DOMAIN__/${DOMAIN}/g;
- s/__CLUSTER__/${CLUSTER}/g;
+# Replace cluster and domain name in the example pillars and test files
+for f in "${SOURCE_PILLARS_DIR}"/*; do
+ sed "s/__CLUSTER__/${CLUSTER}/g;
+ s/__DOMAIN__/${DOMAIN}/g;
s/__RELEASE__/${RELEASE}/g;
s/__HOST_SSL_PORT__/${HOST_SSL_PORT}/g;
s/__GUEST_SSL_PORT__/${GUEST_SSL_PORT}/g;
s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g;
s/__VERSION__/${VERSION}/g" \
- ${f} > ${P_DIR}/$(basename ${f})
+ "${f}" > "${P_DIR}"/$(basename "${f}")
done
-# Let's write an /etc/hosts file that points all the hosts to localhost
-
-echo "127.0.0.2 api keep keep0 collections download ws workbench workbench2 ${CLUSTER}.${DOMAIN} api.${CLUSTER}.${DOMAIN} keep.${CLUSTER}.${DOMAIN} keep0.${CLUSTER}.${DOMAIN} collections.${CLUSTER}.${DOMAIN} download.${CLUSTER}.${DOMAIN} ws.${CLUSTER}.${DOMAIN} workbench.${CLUSTER}.${DOMAIN} workbench2.${CLUSTER}.${DOMAIN}" >> /etc/hosts
+mkdir -p /tmp/cluster_tests
+# Replace cluster and domain name in the example pillars and test files
+for f in "${TESTS_DIR}"/*; do
+ sed "s/__CLUSTER__/${CLUSTER}/g;
+ s/__DOMAIN__/${DOMAIN}/g;
+ s/__HOST_SSL_PORT__/${HOST_SSL_PORT}/g;
+ s/__INITIAL_USER__/${INITIAL_USER}/g;
+ s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
+ s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g" \
+ ${f} > /tmp/cluster_tests/$(basename ${f})
+done
+chmod 755 /tmp/cluster_tests/run-test.sh
# FIXME! #16992 Temporary fix for psql call in arvados-api-server
if [ -e /root/.psqlrc ]; then
# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
# Now run the install
-salt-call --local state.apply -l debug
+salt-call --local state.apply -l ${LOG_LEVEL}
# FIXME! #16992 Temporary fix for psql call in arvados-api-server
if [ "x${DELETE_PSQL}" = "xyes" ]; then
fi
if [ "x${RESTORE_PSQL}" = "xyes" ]; then
- echo "Restroting .psql file"
+ echo "Restoring .psql file"
mv -v /root/.psqlrc.provision.backup /root/.psqlrc
fi
# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
+
+# If running in a vagrant VM, add default user to docker group
+if [ "x${VAGRANT}" = "xyes" ]; then
+ usermod -a -G docker vagrant
+fi
+
+# Test that the installation finished correctly
+if [ "x${TEST}" = "xyes" ]; then
+ cd /tmp/cluster_tests
+ ./run-test.sh
+fi
### TOKENS
tokens:
- system_root: changeme_system_root_token
- management: changeme_management_token
- rails_secret: changeme_rails_secret_token
- anonymous_user: changeme_anonymous_user_token
+ system_root: changemesystemroottoken
+ management: changememanagementtoken
+ rails_secret: changemerailssecrettoken
+ anonymous_user: changemeanonymoususertoken
### KEYS
secrets:
- blob_signing_key: changeme_blob_signing_key
- workbench_secret_key: changeme_workbench_secret_key
- dispatcher_access_key: changeme_dispatcher_access_key
- dispatcher_secret_key: changeme_dispatcher_secret_key
- keep_access_key: changeme_keep_access_key
- keep_secret_key: changeme_keep_secret_key
+ blob_signing_key: changemeblobsigningkey
+ workbench_secret_key: changemeworkbenchsecretkey
+ dispatcher_access_key: changemedispatcheraccesskey
+ dispatcher_secret_key: changeme_dispatchersecretkey
+ keep_access_key: changemekeepaccesskey
+ keep_secret_key: changemekeepsecretkey
Login:
Test:
Controller:
ExternalURL: https://__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
InternalURLs:
- http://127.0.0.2:8003: {}
+ http://controller.internal:8003: {}
DispatchCloud:
InternalURLs:
http://__CLUSTER__.__DOMAIN__:9006: {}
Keepproxy:
ExternalURL: https://keep.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
InternalURLs:
- http://127.0.0.2:25100: {}
+ http://keep.internal:25100: {}
Keepstore:
InternalURLs:
http://keep0.__CLUSTER__.__DOMAIN__:25107: {}
RailsAPI:
InternalURLs:
- http://127.0.0.2:8004: {}
+ http://api.internal:8004: {}
WebDAV:
ExternalURL: https://collections.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
InternalURLs:
- http://127.0.0.2:9002: {}
+ http://collections.internal:9002: {}
WebDAVDownload:
ExternalURL: https://download.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
WebShell:
Websocket:
ExternalURL: wss://ws.__CLUSTER__.__DOMAIN__/websocket
InternalURLs:
- http://127.0.0.2:8005: {}
+ http://ws.internal:8005: {}
Workbench1:
ExternalURL: https://workbench.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
Workbench2:
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+docker:
+ pkg:
+ docker:
+ use_upstream: package
overwrite: true
config:
- server:
- - listen: '127.0.0.2:8004'
+ - listen: 'api.internal:8004'
- server_name: api
- root: /var/www/arvados-api/current/public
- index: index.html index.htm
default: 1
'127.0.0.0/8': 0
upstream controller_upstream:
- - server: '127.0.0.2:8003 fail_timeout=10s'
+ - server: 'controller.internal:8003 fail_timeout=10s'
### SITES
servers:
### STREAMS
http:
upstream keepproxy_upstream:
- - server: '127.0.0.2:25100 fail_timeout=10s'
+ - server: 'keep.internal:25100 fail_timeout=10s'
servers:
managed:
### STREAMS
http:
upstream collections_downloads_upstream:
- - server: '127.0.0.2:9002 fail_timeout=10s'
+ - server: 'collections.internal:9002 fail_timeout=10s'
servers:
managed:
### STREAMS
http:
upstream webshell_upstream:
- - server: '127.0.0.2:4200 fail_timeout=10s'
+ - server: 'shell.internal:4200 fail_timeout=10s'
### SITES
servers:
### STREAMS
http:
upstream websocket_upstream:
- - server: '127.0.0.2:8005 fail_timeout=10s'
+ - server: 'ws.internal:8005 fail_timeout=10s'
servers:
managed:
### STREAMS
http:
upstream workbench_upstream:
- - server: '127.0.0.2:9000 fail_timeout=10s'
+ - server: 'workbench.internal:9000 fail_timeout=10s'
### SITES
servers:
overwrite: true
config:
- server:
- - listen: '127.0.0.2:9000'
+ - listen: 'workbench.internal:9000'
- server_name: workbench
- root: /var/www/arvados-workbench/current/public
- index: index.html index.htm
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+inputfile:
+ class: File
+ path: test.txt
+hasher1_outputname: hasher1.md5sum.txt
+hasher2_outputname: hasher2.md5sum.txt
+hasher3_outputname: hasher3.md5sum.txt
--- /dev/null
+#!/usr/bin/env cwl-runner
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+
+inputs:
+ inputfile: File
+ hasher1_outputname: string
+ hasher2_outputname: string
+ hasher3_outputname: string
+
+outputs:
+ hasher_out:
+ type: File
+ outputSource: hasher3/hasher_out
+
+steps:
+ hasher1:
+ run: hasher.cwl
+ in:
+ inputfile: inputfile
+ outputname: hasher1_outputname
+ out: [hasher_out]
+ hints:
+ ResourceRequirement:
+ coresMin: 1
+ arv:IntermediateOutput:
+ outputTTL: 3600
+ arv:ReuseRequirement:
+ enableReuse: false
+
+ hasher2:
+ run: hasher.cwl
+ in:
+ inputfile: hasher1/hasher_out
+ outputname: hasher2_outputname
+ out: [hasher_out]
+ hints:
+ ResourceRequirement:
+ coresMin: 1
+ arv:IntermediateOutput:
+ outputTTL: 3600
+ arv:ReuseRequirement:
+ enableReuse: false
+
+ hasher3:
+ run: hasher.cwl
+ in:
+ inputfile: hasher2/hasher_out
+ outputname: hasher3_outputname
+ out: [hasher_out]
+ hints:
+ ResourceRequirement:
+ coresMin: 1
+ arv:IntermediateOutput:
+ outputTTL: 3600
+ arv:ReuseRequirement:
+ enableReuse: false
--- /dev/null
+#!/usr/bin/env cwl-runner
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+
+baseCommand: md5sum
+inputs:
+ inputfile:
+ type: File
+ inputBinding:
+ position: 1
+ outputname:
+ type: string
+
+stdout: $(inputs.outputname)
+
+outputs:
+ hasher_out:
+ type: File
+ outputBinding:
+ glob: $(inputs.outputname)
--- /dev/null
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+export ARVADOS_API_TOKEN=changemesystemroottoken
+export ARVADOS_API_HOST=__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+export ARVADOS_API_HOST_INSECURE=true
+
+
+# https://doc.arvados.org/v2.0/install/install-jobs-image.html
+echo "Creating Arvados Standard Docker Images project"
+uuid_prefix=$(arv --format=uuid user current | cut -d- -f1)
+project_uuid=$(arv --format=uuid group list --filters '[["name", "=", "Arvados Standard Docker Images"]]')
+
+if [ "x${project_uuid}" = "x" ]; then
+ project_uuid=$(arv --format=uuid group create --group "{\"owner_uuid\": \"${uuid_prefix}-tpzed-000000000000000\", \"group_class\":\"project\", \"name\":\"Arvados Standard Docker Images\"}")
+
+ read -rd $'\000' newlink <<EOF; arv link create --link "${newlink}"
+{
+ "tail_uuid":"${uuid_prefix}-j7d0g-fffffffffffffff",
+ "head_uuid":"${project_uuid}",
+ "link_class":"permission",
+ "name":"can_read"
+}
+EOF
+fi
+
+echo "Arvados project uuid is '${project_uuid}'"
+
+echo "Uploading arvados/jobs' docker image to the project"
+VERSION="2.1.1"
+arv-keepdocker --pull arvados/jobs "${VERSION}" --project-uuid "${project_uuid}"
+
+# Create the initial user
+echo "Creating initial user '__INITIAL_USER__'"
+user_uuid=$(arv --format=uuid user list --filters '[["email", "=", "__INITIAL_USER_EMAIL__"], ["username", "=", "__INITIAL_USER__"]]')
+
+if [ "x${user_uuid}" = "x" ]; then
+ user_uuid=$(arv --format=uuid user create --user '{"email": "__INITIAL_USER_EMAIL__", "username": "__INITIAL_USER__"}')
+ echo "Setting up user '__INITIAL_USER__'"
+ arv user setup --uuid "${user_uuid}"
+fi
+
+echo "Activating user '__INITIAL_USER__'"
+arv user update --uuid "${user_uuid}" --user '{"is_active": true}'
+
+echo "Getting the user API TOKEN"
+user_api_token=$(arv api_client_authorization list --filters "[[\"owner_uuid\", \"=\", \"${user_uuid}\"],[\"kind\", \"==\", \"arvados#apiClientAuthorization\"]]" --limit=1 |jq -r .items[].api_token)
+
+if [ "x${user_api_token}" = "x" ]; then
+ user_api_token=$(arv api_client_authorization create --api-client-authorization "{\"owner_uuid\": \"${user_uuid}\"}" | jq -r .api_token)
+fi
+
+# Change to the user's token and run the workflow
+export ARVADOS_API_TOKEN="${user_api_token}"
+
+echo "Running test CWL workflow"
+cwl-runner hasher-workflow.cwl hasher-workflow-job.yml
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+test
projects / arvados.git / commitdiff