# star / unstar the current project
def star
- links = Link.where(tail_uuid: current_user.uuid,
+ links = Link.where(owner_uuid: current_user.uuid,
head_uuid: @object.uuid,
link_class: 'star')
helper_method :my_starred_projects
def my_starred_projects user
return if defined?(@starred_projects) && @starred_projects
- links = Link.filter([['tail_uuid', '=', user.uuid],
+ links = Link.filter([['owner_uuid', 'in', ["#{Rails.configuration.ClusterID}-j7d0g-fffffffffffffff", user.uuid]],
['link_class', '=', 'star'],
['head_uuid', 'is_a', 'arvados#group']]).with_count("none").select(%w(head_uuid))
uuids = links.collect { |x| x.head_uuid }
my_child_containers = my_children.map(&:container_uuid).compact.uniq
grandchildren = {}
my_child_containers.each { |c| grandchildren[c] = []} if my_child_containers.any?
- reqs = ContainerRequest.select(cols).where(requesting_container_uuid: my_child_containers).with_count("none").results if my_child_containers.any?
+ reqs = ContainerRequest.select(cols).where(requesting_container_uuid: my_child_containers).order(["requesting_container_uuid", "uuid"]).with_count("none").results if my_child_containers.any?
reqs.each {|cr| grandchildren[cr.requesting_container_uuid] << cr} if reqs
my_children.each do |cr|
<td style="word-break:break-all;">
<% if @my_vm_logins[vm[:uuid]] %>
<% @my_vm_logins[vm[:uuid]].each do |login| %>
- <code>ssh <%= login %>@<%= vm[:hostname] %>.<%= current_uuid_prefix || 'xyzzy' %></code>
+ <code>ssh <%= login %>@<%= vm[:hostname] %></code>
<% end %>
<% end %>
</td>
<% end %>
</div>
</div>
- <p>In order to access virtual machines using SSH, <%= link_to ssh_keys_user_path(current_user) do%> add an SSH key to your account<%end%> and add a section like this to your SSH configuration file ( <i>~/.ssh/config</i>):</p>
- <pre>Host *.<%= current_uuid_prefix || 'xyzzy' %>
- TCPKeepAlive yes
- ServerAliveInterval 60
- ProxyCommand ssh -p2222 turnout@switchyard.<%= current_api_host || 'xyzzy.arvadosapi.com' %> -x -a $SSH_PROXY_FLAGS %h
- </pre>
+
+<p>In order to access virtual machines using SSH, <%= link_to ssh_keys_user_path(current_user) do%>add an SSH key to your account<%end%>.</p>
+
+<%= raw(Rails.configuration.Workbench.SSHHelpPageHTML) %>
SPDX-License-Identifier: AGPL-3.0 %>
-<p>
-Sample <code>~/.ssh/config</code> section:
-</p>
-
-<pre>
-Host *.arvados
- ProxyCommand ssh -p2222 turnout@switchyard.<%= current_api_host.sub(/:.*$/,"") || 'xyzzy.arvadosapi.com' %> -x -a $SSH_PROXY_FLAGS %h
-<% if @objects.first.andand.current_user_logins.andand.first %>
- User <%= @objects.first.current_user_logins.andand.first %>
-<% end %>
-</pre>
-
-<p>
-Sample login command:
-</p>
-
-<pre>
-ssh <%= @objects.first.andand.hostname.andand.sub('.'+current_api_host.sub(/:.*$/,""),'') or 'vm-hostname' %>.arvados
-</pre>
-
-<p>
- See also:
- <%= link_to raw('Arvados Docs → User Guide → SSH access'),
- "#{Rails.configuration.Workbench.ArvadosDocsite}/user/getting_started/ssh-access-unix.html",
- target: "_blank"%>.
-</p>
+<%= raw(Rails.configuration.Workbench.SSHHelpPageHTML) %>
case "$TARGET" in
centos*)
- fpm_depends+=(git bison make automake gcc gcc-c++)
+ fpm_depends+=(git bison make automake gcc gcc-c++ graphviz)
;;
debian* | ubuntu*)
- fpm_depends+=(git g++ bison zlib1g-dev make)
+ fpm_depends+=(git g++ bison zlib1g-dev make graphviz)
;;
esac
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
# The version of setuptools that comes with CentOS is way too old
-RUN pip install --upgrade setuptools
+RUN pip install --upgrade 'setuptools<45'
ENV WORKSPACE /arvados
CMD ["scl", "enable", "rh-python36", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos7"]
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools python3-pip libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip python3-venv python3-dev
# Install virtualenv
-RUN /usr/bin/pip install virtualenv
+RUN /usr/bin/pip install 'virtualenv<20'
# Install RVM
ADD generated/mpapis.asc /tmp/
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools python3-pip libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip python3-venv python3-dev
# Install virtualenv
-RUN /usr/bin/pip install virtualenv
+RUN /usr/bin/pip install 'virtualenv<20'
# Install RVM
ADD generated/mpapis.asc /tmp/
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools python3-pip libcurl4-gnutls-dev libgnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata python3-venv python3-dev
# Install virtualenv
-RUN /usr/bin/pip install virtualenv
+RUN /usr/bin/pip install 'virtualenv<20'
# Install RVM
ADD generated/mpapis.asc /tmp/
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata python3-venv python3-dev
# Install virtualenv
-RUN /usr/bin/pip install virtualenv
+RUN /usr/bin/pip install 'virtualenv<20'
# Install RVM
ADD generated/mpapis.asc /tmp/
export RAILS_ENV=production
if ! $COMMAND_PREFIX bundle --version >/dev/null; then
- run_and_report "Installing bundle" $COMMAND_PREFIX gem install bundle
+ run_and_report "Installing bundler" $COMMAND_PREFIX gem install bundler --version 1.17.3
fi
run_and_report "Running bundle install" \
# set to --no-cache-dir to disable pip caching
CACHE_FLAG=
-MAINTAINER="Ward Vandewege <wvandewege@veritasgenetics.com>"
-VENDOR="Veritas Genetics, Inc."
+MAINTAINER="Arvados Package Maintainers <packaging@arvados.org>"
+VENDOR="The Arvados Project"
# End of user configuration
# Arvados cwl runner
fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl"
+# Arvados cwl runner - Python3 package
+fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "python3"
+
# The PAM module
fpm_build_virtualenv "libpam-arvados" "sdk/pam"
if [ ${#failures[@]} -eq 0 ]; then
if [[ "$RC" != 0 ]]; then
- echo "/usr/local/arvados-dev/jenkins/run_upload_packages_testing.py -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET"
- /usr/local/arvados-dev/jenkins/run_upload_packages_testing.py -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET
+ echo "/usr/local/arvados-dev/jenkins/run_upload_packages.py --repo testing -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET"
+ /usr/local/arvados-dev/jenkins/run_upload_packages.py --repo testing -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET
else
- echo "/usr/local/arvados-dev/jenkins/run_upload_packages.py -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET"
- /usr/local/arvados-dev/jenkins/run_upload_packages.py -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET
+ echo "/usr/local/arvados-dev/jenkins/run_upload_packages.py --repo dev -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET"
+ /usr/local/arvados-dev/jenkins/run_upload_packages.py --repo dev -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET
fi
else
echo "Skipping package upload, there were errors building and/or testing the packages"
# older packages.
LICENSE_PACKAGE_TS=20151208015500
-librarydir=`dirname "$(readlink -f "$0")"`
-
if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
RAILS_PACKAGE_ITERATION=1
else
local git_ts git_hash
declare $(format_last_commit_here "git_ts=%ct git_hash=%h" "$subdir")
- ARVADOS_BUILDING_VERSION="$($librarydir/version-at-commit.sh $git_hash)"
+ ARVADOS_BUILDING_VERSION="$($WORKSPACE/build/version-at-commit.sh $git_hash)"
echo "$ARVADOS_BUILDING_VERSION"
}
rm -rf dist/*
# Get the latest setuptools
- if ! $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools; then
+ if ! $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then
echo "Error, unable to upgrade setuptools with"
- echo " $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools"
+ echo " $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'"
exit 1
fi
# filter a useless warning (when building the cwltest package) from the stderr output
fi
echo "pip version: `build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip --version`"
- if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools; then
+ if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then
echo "Error, unable to upgrade setuptools with"
- echo " build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools"
+ echo " build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'"
exit 1
fi
echo "setuptools version: `build/usr/share/$python/dist/$PYTHON_PKG/bin/$python -c 'import setuptools; print(setuptools.__version__)'`"
versionglob="[0-9].[0-9]*.[0-9]*"
devsuffix=".dev"
-if ! git describe --exact-match --match "$versionglob" "$commit" 2>/dev/null; then
- if git merge-base --is-ancestor "$commit" origin/master; then
- # x.(y+1).0.preTIMESTAMP, where x.y.z is the newest version that does not contain $commit
+# automatically assign version
+#
+# handles the following cases:
+#
+# 1. commit is directly tagged. print that.
+#
+# 2. commit is on master or a development branch, the nearest tag is older
+# than commit where this branch joins master.
+# -> take greatest version tag in repo X.Y.Z and assign X.(Y+1).0
+#
+# 3. commit is on a release branch, the nearest tag is newer
+# than the commit where this branch joins master.
+# -> take nearest tag X.Y.Z and assign X.Y.(Z+1)
+
+tagged=$(git tag --points-at "$commit")
+
+if [[ -n "$tagged" ]] ; then
+ echo $tagged
+else
+ # 1. get the nearest tag with 'git describe'
+ # 2. get the merge base between this commit and master
+ # 3. if the tag is an ancestor of the merge base,
+ # (tag is older than merge base) increment minor version
+ # else, tag is newer than merge base, so increment point version
+
+ nearest_tag=$(git describe --tags --abbrev=0 --match "$versionglob" "$commit")
+ merge_base=$(git merge-base origin/master "$commit")
+
+ if git merge-base --is-ancestor "$nearest_tag" "$merge_base" ; then
+ # x.(y+1).0.devTIMESTAMP, where x.y.z is the newest version that does not contain $commit
+ # grep reads the list of tags (-f) that contain $commit and filters them out (-v)
+ # this prevents a newer tag from retroactively changing the versions of everything before it
v=$(git tag | grep -vFf <(git tag --contains "$commit") | sort -Vr | head -n1 | perl -pe 's/\.(\d+)\.\d+/".".($1+1).".0"/e')
else
- # x.y.(z+1).preTIMESTAMP, where x.y.z is the latest released ancestor of $commit
- v=$(git describe --abbrev=0 --match "$versionglob" "$commit" | perl -pe 's/(\d+)$/$1+1/e')
+ # x.y.(z+1).devTIMESTAMP, where x.y.z is the latest released ancestor of $commit
+ v=$(echo $nearest_tag | perl -pe 's/(\d+)$/$1+1/e')
fi
isodate=$(TZ=UTC git log -n1 --format=%cd --date=iso "$commit")
ts=$(TZ=UTC date --date="$isodate" "+%Y%m%d%H%M%S")
Type=notify
EnvironmentFile=-/etc/arvados/environment
ExecStart=/usr/bin/arvados-controller
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=1
Type=notify
EnvironmentFile=-/etc/arvados/environment
ExecStart=/usr/bin/arvados-dispatch-cloud
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=1
- admin/metrics.html.textile.liquid
- admin/health-checks.html.textile.liquid
- admin/management-token.html.textile.liquid
- - Cloud:
- - admin/storage-classes.html.textile.liquid
- - admin/spot-instances.html.textile.liquid
- - admin/cloudtest.html.textile.liquid
- Data Management:
- admin/collection-versioning.html.textile.liquid
- admin/collection-managed-properties.html.textile.liquid
- admin/keep-balance.html.textile.liquid
- admin/controlling-container-reuse.html.textile.liquid
- admin/logs-table-management.html.textile.liquid
+ - admin/workbench2-vocabulary.html.textile.liquid
+ - admin/storage-classes.html.textile.liquid
+ - Cloud:
+ - admin/spot-instances.html.textile.liquid
+ - admin/cloudtest.html.textile.liquid
- Other:
- install/migrate-docker19.html.textile.liquid
- admin/upgrade-crunch2.html.textile.liquid
- - admin/workbench2-vocabulary.html.textile.liquid
installguide:
- Overview:
- install/index.html.textile.liquid
<script>
function dismissAnnotateNotify() {
- window.sessionStorage.setItem("dismiss-annotate-notify", "true");
+ window.localStorage.setItem("dismiss-annotate-notify", "true");
$('#annotate-notify').attr('style', "display: none;");
}
- if (window.sessionStorage.getItem("dismiss-annotate-notify") === "true") {
+ if (window.localStorage.getItem("dismiss-annotate-notify") === "true") {
dismissAnnotateNotify();
} else {
$('#annotate-notify').attr('style', "display: inline-block;");
<link href="{{ site.baseurl }}/css/button-override.css" rel="stylesheet">
<link href="{{ site.baseurl }}/css/images.css" rel="stylesheet">
<script src="{{ site.baseurl }}/js/jquery.min.js"></script>
+ <script src="{{ site.baseurl }}/js/bootstrap.min.js"></script>
<script src="https://hypothes.is/embed.js" async></script>
<style>
html {
This page describes how to enable and configure the collection versioning feature on the API server.
-h3. API Server configuration
+h3. Configuration
-There are 2 configuration settings that control this feature, both go on the @application.yml@ file.
+There are 2 configuration settings in the @Collections@ section of @config.yml@ that control this feature.
-h4. Setting: @collection_versioning@ (Boolean. Default: false)
+<pre>
+ Collections:
+ # If true, enable collection versioning.
+ # When a collection's preserve_version field is true or the current version
+ # is older than the amount of seconds defined on PreserveVersionIfIdle,
+ # a snapshot of the collection's previous state is created and linked to
+ # the current collection.
+ CollectionVersioning: false
-If @true@, collection versioning is enabled, meaning that new version records can be created. Note that if you set @collection_versioning@ to @false@ after being enabled, old versions will still be accessible, but further changes will not be versioned.
+ # This setting control the auto-save aspect of collection versioning, and can be set to:
+ # 0s = auto-create a new version on every update.
+ # -1s = never auto-create new versions.
+ # > 0s = auto-create a new version when older than the specified number of seconds.
+ PreserveVersionIfIdle: -1s
+</pre>
-h4. Setting: @preserve_version_if_idle@ (Numeric. Default: -1)
-
-This setting control the auto-save aspect of collection versioning, and can be set to:
-* @-1@: Never auto-save versions. Only save versions when the client ask for it by setting @preserve_version@ to @true@ on any given collection.
-* @0@: Preserve all versions every time a collection gets a versionable update.
-* @N@ (being N > 0): Preserve version when a collection gets a versionable update after a period of at least N seconds since the last time it was modified.
+Note that if you set @collection_versioning@ to @false@ after being enabled, old versions will still be accessible, but further changes will not be versioned.
h3. Using collection versioning
-"Discussed in the user guide":{{site.baseurl}}/user/topics/collection-versioning.html
\ No newline at end of file
+"Discussed in the user guide":{{site.baseurl}}/user/topics/collection-versioning.html
---
layout: default
navsection: installguide
-title: Migrating Configuration from v1.4 to v1.5
+title: Migrating Configuration from v1.4 to v2.0
...
{% comment %}
{% endcomment %}
{% include 'notebox_begin_warning' %}
-_New installations of Arvados 1.5+ can skip this section_
+_New installations of Arvados 2.0+ can skip this section_
{% include 'notebox_end' %}
-Arvados 1.5 migrates to a centralized configuration file for all components. The centralized Arvados configuration is @/etc/arvados/config.yml@. Components that support the new centralized configuration are listed below. During the migration period, legacy configuration files are still loaded and take precedence over the centralized configuration file.
+Arvados 2.0 migrates to a centralized configuration file for all components. The centralized Arvados configuration is @/etc/arvados/config.yml@. Components that support the new centralized configuration are listed below. During the migration period, legacy configuration files are still loaded and take precedence over the centralized configuration file.
h2. API server
---
layout: default
navsection: admin
-title: Controlling container reuse
+title: Preventing container reuse
...
{% comment %}
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-This page describes how an admin can control container reuse using the @arv@ command. This can be utilized to avoid reusing a completed container without disabling reuse for the corresponding steps in affected workflows. For example, if a container exited successfully but produced bad output, it may not be feasible to update the workflow immediately. Meanwhile, changing the state of the container from @Complete@ to @Cancelled@ will prevent it from being used in subsequent workflows.
+Sometimes a container exited successfully but produced bad output, and re-running the workflow will cause it to re-use the bad container instead of running a new container. One way to deal with this is to re-run the entire workflow with reuse disable. Another way is for the workflow author to tweak the input data or workflow so that on re-run it produces a distinct container request. However, for large or complex workflows both these options may be impractical.
-If a container is in the @Complete@ state, the following @arv@ command will change its state to @Cancelled@, where @xxxxx-xxxxx-xxxxxxxxxxxxxxx@ is the @UUID@ of the container:
+To prevent an individual container from being reused in later workflows, an admin can manually change the state of the bad container record from @Complete@ to @Cancelled@. The following @arv@ command demonstrates how change a container state to @Cancelled@, where @xxxxx-xxxxx-xxxxxxxxxxxxxxx@ is the @UUID@ of the container:
<pre>arv container update -u xxxxx-xxxxx-xxxxxxxxxxxxxxx -c '{"state":"Cancelled"}'</pre>
-
-Use the following command to list all containers that exited with 0 and were then cancelled:
-
-<pre>arv container list --filters='[["state", "=", "Cancelled"], ["exit_code", "=", 0]]'</pre>See the "arv CLI tool overview":{{site.baseurl}}/sdk/cli/index.html for more details about using the @arv@ command.
As a result, this table grows indefinitely, even on sites where policy does not require an audit log; making backups, migrations, and upgrades unnecessarily slow and painful.
-h3. API Server configuration
+h3. Configuration
-To solve the problem mentioned above, the API server offers the possibility to limit the amount of log information stored on the table:
+To solve the problem mentioned above, the @AuditLogs@ section of @config.yml@ offers several options to limit the amount of log information stored on the table:
<pre>
-# Attributes to suppress in events and audit logs. Notably,
-# specifying ["manifest_text"] here typically makes the database
-# smaller and faster.
-#
-# Warning: Using any non-empty value here can have undesirable side
-# effects for any client or component that relies on event logs.
-# Use at your own risk.
-unlogged_attributes: []
+ AuditLogs:
+ # Time to keep audit logs. (An audit log is a row added
+ # to the "logs" table in the PostgreSQL database each time an
+ # Arvados object is created, modified, or deleted.)
+ #
+ # Currently, websocket event notifications rely on audit logs, so
+ # this should not be set lower than 5 minutes.
+ MaxAge: 336h
+
+ # Maximum number of log rows to delete in a single SQL transaction,
+ # to prevent surprises and avoid bad database behavior
+ # (especially the first time the cleanup job runs on an existing
+ # cluster with a huge backlog) a maximum number of rows to
+ # delete in a single transaction.
+ #
+ # If MaxDeleteBatch is 0, log entries will never be
+ # deleted by Arvados. Cleanup can be done by an external process
+ # without affecting any Arvados system processes, as long as very
+ # recent (<5 minutes old) logs are not deleted.
+ #
+ # 100000 is a reasonable batch size for most sites.
+ MaxDeleteBatch: 0
+
+ # Attributes to suppress in events and audit logs. Notably,
+ # specifying {"manifest_text": {}} here typically makes the database
+ # smaller and faster.
+ #
+ # Warning: Using any non-empty value here can have undesirable side
+ # effects for any client or component that relies on event logs.
+ # Use at your own risk.
+ UnloggedAttributes: {}
</pre>
-The above setting affects all events being logged, independently of how much time they will be kept on the database.
-
-<pre>
-# Time to keep audit logs (a row in the log table added each time an
-# Arvados object is created, modified, or deleted) in the PostgreSQL
-# database. Currently, websocket event notifications rely on audit
-# logs, so this should not be set lower than 300 (5 minutes).
-max_audit_log_age: 1209600
-</pre>
-
-...and to prevent surprises and avoid bad database behavior (especially the first time the cleanup job runs on an existing cluster with a huge backlog) a maximum number of rows to delete in a single transaction.
-
-<pre>
-# Maximum number of log rows to delete in a single SQL transaction.
-#
-# If max_audit_log_delete_batch is 0, log entries will never be
-# deleted by Arvados. Cleanup can be done by an external process
-# without affecting any Arvados system processes, as long as very
-# recent (<5 minutes old) logs are not deleted.
-#
-# 100000 is a reasonable batch size for most sites.
-max_audit_log_delete_batch: 0
-</pre>
-
-This feature works when both settings are non-zero, periodically dispatching a background task that deletes all log rows older than @max_audit_log_age@.
-The events being cleaned up by this process don't include job/container stderr logs (they're handled by the existing @delete job/container logs@ rake tasks)
h3. Additional consideration
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Some Arvados services publish Prometheus/OpenMetrics-compatible metrics at @/metrics@, and some provide additional runtime status at @/status.json@. Metrics can help you understand how components perform under load, find performance bottlenecks, and detect and diagnose problems.
+Some Arvados services publish Prometheus/OpenMetrics-compatible metrics at @/metrics@. Metrics can help you understand how components perform under load, find performance bottlenecks, and detect and diagnose problems.
To access metrics endpoints, services must be configured with a "management token":management-token.html. When accessing a metrics endpoint, prefix the management token with @"Bearer "@ and supply it in the @Authorization@ request header.
-<pre>curl -sfH "Authorization: Bearer your_management_token_goes_here" "https://0.0.0.0:25107/status.json"
+<pre>curl -sfH "Authorization: Bearer your_management_token_goes_here" "https://0.0.0.0:25107/metrics"
</pre>
-h2. Keep-web
+The plain text export format includes "help" messages with a description of each reported metric.
-Keep-web exports metrics at @/metrics@ -- e.g., @https://collections.zzzzz.arvadosapi.com/metrics@.
+When configuring Prometheus, use a @bearer_token@ or @bearer_token_file@ option to authenticate requests.
-table(table table-bordered table-condensed).
-|_. Name|_. Type|_. Description|
-|request_duration_seconds|summary|elapsed time between receiving a request and sending the last byte of the response body (segmented by HTTP request method and response status code)|
-|time_to_status_seconds|summary|elapsed time between receiving a request and sending the HTTP response status code (segmented by HTTP request method and response status code)|
-
-Metrics in the @arvados_keepweb_collectioncache@ namespace report keep-web's internal cache of Arvados collection metadata.
-
-table(table table-bordered table-condensed).
-|_. Name|_. Type|_. Description|
-|arvados_keepweb_collectioncache_requests|counter|cache lookups|
-|arvados_keepweb_collectioncache_api_calls|counter|outgoing API calls|
-|arvados_keepweb_collectioncache_permission_hits|counter|collection-to-permission cache hits|
-|arvados_keepweb_collectioncache_pdh_hits|counter|UUID-to-PDH cache hits|
-|arvados_keepweb_collectioncache_hits|counter|PDH-to-manifest cache hits|
-|arvados_keepweb_collectioncache_cached_manifests|gauge|number of collections in the cache|
-|arvados_keepweb_collectioncache_cached_manifest_bytes|gauge|memory consumed by cached collection manifests|
-
-h2. Keepstore
-
-Keepstore exports metrics at @/status.json@ -- e.g., @http://keep0.zzzzz.arvadosapi.com:25107/status.json@.
-
-h3. Root
-
-table(table table-bordered table-condensed).
-|_. Attribute|_. Type|_. Description|
-|Volumes| array of "volumeStatusEnt":#volumeStatusEnt ||
-|BufferPool| "PoolStatus":#PoolStatus ||
-|PullQueue| "WorkQueueStatus":#WorkQueueStatus ||
-|TrashQueue| "WorkQueueStatus":#WorkQueueStatus ||
-|RequestsCurrent| int ||
-|RequestsMax| int ||
-|Version| string ||
-
-h3(#volumeStatusEnt). volumeStatusEnt
-
-table(table table-bordered table-condensed).
-|_. Attribute|_. Type|_. Description|
-|Label| string||
-|Status| "VolumeStatus":#VolumeStatus ||
-|VolumeStats| "ioStats":#ioStats ||
-
-h3(#VolumeStatus). VolumeStatus
-
-table(table table-bordered table-condensed).
-|_. Attribute|_. Type|_. Description|
-|MountPoint| string||
-|DeviceNum| uint64||
-|BytesFree| uint64||
-|BytesUsed| uint64||
-
-h3(#ioStats). ioStats
-
-table(table table-bordered table-condensed).
-|_. Attribute|_. Type|_. Description|
-|Errors| uint64||
-|Ops| uint64||
-|CompareOps| uint64||
-|GetOps| uint64||
-|PutOps| uint64||
-|TouchOps| uint64||
-|InBytes| uint64||
-|OutBytes| uint64||
-
-h3(#PoolStatus). PoolStatus
-
-table(table table-bordered table-condensed).
-|_. Attribute|_. Type|_. Description|
-|BytesAllocatedCumulative| uint64||
-|BuffersMax| int||
-|BuffersInUse| int||
-
-h3(#WorkQueueStatus). WorkQueueStatus
-
-table(table table-bordered table-condensed).
-|_. Attribute|_. Type|_. Description|
-|InProgress| int||
-|Queued| int||
-
-h3. Example response
-
-<pre>
-{
- "Volumes": [
- {
- "Label": "[UnixVolume /var/lib/arvados/keep0]",
- "Status": {
- "MountPoint": "/var/lib/arvados/keep0",
- "DeviceNum": 65029,
- "BytesFree": 222532972544,
- "BytesUsed": 435456679936
- },
- "InternalStats": {
- "Errors": 0,
- "InBytes": 1111,
- "OutBytes": 0,
- "OpenOps": 1,
- "StatOps": 4,
- "FlockOps": 0,
- "UtimesOps": 0,
- "CreateOps": 0,
- "RenameOps": 0,
- "UnlinkOps": 0,
- "ReaddirOps": 0
- }
- }
- ],
- "BufferPool": {
- "BytesAllocatedCumulative": 67108864,
- "BuffersMax": 20,
- "BuffersInUse": 0
- },
- "PullQueue": {
- "InProgress": 0,
- "Queued": 0
- },
- "TrashQueue": {
- "InProgress": 0,
- "Queued": 0
- },
- "RequestsCurrent": 1,
- "RequestsMax": 40,
- "Version": "dev"
-}
+<pre>scrape_configs:
+ - job_name: keepstore
+ bearer_token: your_management_token_goes_here
+ static_configs:
+ - targets:
+ - "keep0.ClusterID.example.com:25107"
</pre>
-h2. Keep-balance
-
-Keep-balance exports metrics at @/metrics@ -- e.g., @http://keep.zzzzz.arvadosapi.com:9005/metrics@.
-
-table(table table-bordered table-condensed).
-|_. Name|_. Type|_. Description|
-|arvados_keep_total_{replicas,blocks,bytes}|gauge|stored data (stored in backend volumes, whether referenced or not)|
-|arvados_keep_garbage_{replicas,blocks,bytes}|gauge|garbage data (unreferenced, and old enough to trash)|
-|arvados_keep_transient_{replicas,blocks,bytes}|gauge|transient data (unreferenced, but too new to trash)|
-|arvados_keep_overreplicated_{replicas,blocks,bytes}|gauge|overreplicated data (more replicas exist than are needed)|
-|arvados_keep_underreplicated_{replicas,blocks,bytes}|gauge|underreplicated data (fewer replicas exist than are needed)|
-|arvados_keep_lost_{replicas,blocks,bytes}|gauge|lost data (referenced by collections, but not found on any backend volume)|
-|arvados_keep_dedup_block_ratio|gauge|deduplication ratio (block references in collections ÷ distinct blocks referenced)|
-|arvados_keep_dedup_byte_ratio|gauge|deduplication ratio (block references in collections ÷ distinct blocks referenced, weighted by block size)|
-|arvados_keepbalance_get_state_seconds|summary|time to get all collections and keepstore volume indexes for one iteration|
-|arvados_keepbalance_changeset_compute_seconds|summary|time to compute changesets for one iteration|
-|arvados_keepbalance_send_pull_list_seconds|summary|time to send pull lists to all keepstore servers for one iteration|
-|arvados_keepbalance_send_trash_list_seconds|summary|time to send trash lists to all keepstore servers for one iteration|
-|arvados_keepbalance_sweep_seconds|summary|time to complete one iteration|
-
-Each @arvados_keep_@ storage state statistic above is presented as a set of three metrics:
-
-table(table table-bordered table-condensed).
-|*_blocks|distinct block hashes|
-|*_bytes|bytes stored on backend volumes|
-|*_replicas|objects/files stored on backend volumes|
+table(table table-bordered table-condensed table-hover).
+|_. Component|_. Metrics endpoint|
+|arvados-api-server||
+|arvados-controller|✓|
+|arvados-dispatch-cloud|✓|
+|arvados-git-httpd||
+|arvados-node-manager||
+|arvados-ws||
+|composer||
+|keepproxy||
+|keepstore|✓|
+|keep-balance|✓|
+|keep-web|✓|
+|sso-provider||
+|workbench1||
+|workbench2||
h2. Node manager
-The node manager status end point provides a snapshot of internal status at the time of the most recent wishlist update.
+The node manager does not export prometheus-style metrics, but its @/status.json@ endpoint provides a snapshot of internal status at the time of the most recent wishlist update.
+
+<pre>curl -sfH "Authorization: Bearer your_management_token_goes_here" "http://0.0.0.0:8989/status.json"
+</pre>
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|
The storage classes for each volume are set in the per-volume "keepstore configuration":{{site.baseurl}}/install/install-keepstore.html
<pre>
-Volumes:
- - ... Volume configuration ...
- #
- # If no storage classes are specified, will use [default]
- #
- StorageClasses: null
-
- - ... Volume configuration ...
- #
- # Specify this volume is in the "archival" storage class.
- #
- StorageClasses: [archival]
-
+ Volumes:
+ ClusterID-nyw5e-000000000000000:
+ # This volume is in the "default" storage class.
+ StorageClasses:
+ default: true
+ ClusterID-nyw5e-000000000000001:
+ # Specify this volume is in the "archival" storage class.
+ StorageClasses:
+ archival: true
</pre>
Names of storage classes are internal to the cluster and decided by the administrator. Aside from "default", Arvados currently does not define any standard storage class names.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-The "containers" API is the recommended way to submit compute work to Arvados. It supersedes the "jobs" API, which is end-of-life as of Arvados 1.5.
+The "containers" API is the recommended way to submit compute work to Arvados. It supersedes the "jobs" API, which is end-of-life in Arvados 2.0.
h2. Benefits over the "jobs" API
TODO: extract this information based on git commit messages and generate changelogs / release notes automatically.
{% endcomment %}
-h3(#master). development master (as of 2020-01-16)
+<notextile>
+<div class="releasenotes">
+</notextile>
-h4. "/" prohibited in collection and project names
+h2(#master). development master (as of 2020-02-07)
-(issue "#15836":https://dev.arvados.org/issues/15836) By default, Arvados now rejects new names containing the @/@ character when creating or renaming collections and projects. Previously, these names were permitted, but the resulting objects were invisible in the WebDAV "home" tree. If you prefer, you can restore the previous behavior, and optionally configure a substitution string to make the affected objects accessible via WebDAV. See @ForwardSlashNameSubstitution@ in the "configuration reference":config.html.
+"Upgrading from 2.0.0":#v2_0_0
-h4. Migrating to centralized config.yml
+None in current development master.
-See "Migrating Configuration":config-migration.html for notes on migrating legacy per-component configuration files to the new centralized @/etc/arvados/config.yml@. To ensure a smooth transition, the per-component config files continue to be read, and take precedence over the centralized configuration.
+h2(#v2_0_0). v2.0.0 (2020-02-07)
-h4. Cloud installations only: node manager replaced by arvados-dispatch-cloud
-
-Node manager is deprecated and replaced by @arvados-dispatch-cloud@. No automated config migration is available. Follow the instructions to "install the cloud dispatcher":../install/install-dispatch-cloud.html
+"Upgrading from 1.4":#v1_4_1
-*Only one dispatch process should be running at a time.* If you are migrating a system that currently runs Node manager and @crunch-dispatch-slurm@, it is safest to remove the @crunch-dispatch-slurm@ service entirely before installing @arvados-dispatch-cloud@.
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo systemctl --now disable crunch-dispatch-slurm</span>
-~$ <span class="userinput">sudo apt-get remove crunch-dispatch-slurm</span>
-</code></pre>
-</notextile>
+Arvados 2.0 is a major upgrade, with many changes. Please read these upgrade notes carefully before you begin.
-h4. Delete "keep_services" records
+h3. Migrating to centralized config.yml
-After all keepproxy and keepstore configurations have been migrated to the centralized configuration file (see below), all keep_services records you added manually during installation should be removed. System logs from keepstore and keepproxy at startup, as well as the output of @arvados-server config-check@, will remind you to do this.
+See "Migrating Configuration":config-migration.html for notes on migrating legacy per-component configuration files to the new centralized @/etc/arvados/config.yml@.
-<notextile><pre><code>$ export ARVADOS_API_HOST=...
-$ export ARVADOS_API_TOKEN=...
-$ arv --format=uuid keep_service list | xargs -n1 arv keep_service delete --uuid
-</code></pre></notextile>
+To ensure a smooth transition, the per-component config files continue to be read, and take precedence over the centralized configuration. Your cluster should continue to function after upgrade but before doing the full configuration migration. However, several services (keepstore, keep-web, keepproxy) require a minimal `/etc/arvados/config.yml` to start:
-Once these old records are removed, @arv keep_service list@ will instead return the services listed under Services/Keepstore/InternalURLs and Services/Keepproxy/ExternalURL in your centralized configuration file.
+<pre>
+Clusters:
+ zzzzz:
+ Services:
+ Controller:
+ ExternalURL: "https://zzzzz.example.com"
+</pre>
-h4. Keep-balance configuration migration
+h3. Keep-balance configuration migration
(feature "#14714":https://dev.arvados.org/issues/14714 ) The keep-balance service can now be configured using the centralized configuration file at @/etc/arvados/config.yml@. The following command line and configuration options have changed.
Please see the "config migration guide":{{site.baseurl}}/admin/config-migration.html and "keep-balance install guide":{{site.baseurl}}/install/install-keep-balance.html for more details.
-h4. Arv-git-httpd configuration migration
+h3. Arv-git-httpd configuration migration
(feature "#14712":https://dev.arvados.org/issues/14712 ) The arv-git-httpd package can now be configured using the centralized configuration file at @/etc/arvados/config.yml@. Configuration via individual command line arguments is no longer available. Please see "arv-git-httpd's config migration guide":{{site.baseurl}}/admin/config-migration.html#arv-git-httpd for more details.
-h4. Keepstore and keep-web configuration migration
+h3. Keepstore and keep-web configuration migration
keepstore and keep-web no longer support configuration via (previously deprecated) command line configuration flags and environment variables.
keepstore now supports the legacy @keepstore.yml@ config format (used by Arvados 1.4) and the new cluster config file format. Please check the "keepstore config migration notes":{{site.baseurl}}/admin/config-migration.html#keepstore and "keepstore install guide":{{site.baseurl}}/install/install-keepstore.html for more details.
-h4. Keepproxy configuration migration
+h3. Keepproxy configuration migration
(feature "#14715":https://dev.arvados.org/issues/14715 ) Keepproxy can now be configured using the centralized config at @/etc/arvados/config.yml@. Configuration via individual command line arguments is no longer available and the @DisableGet@, @DisablePut@, and @PIDFile@ configuration options are no longer supported. If you are still using the legacy config and @DisableGet@ or @DisablePut@ are set to true or @PIDFile@ has a value, keepproxy will produce an error and fail to start. Please see "keepproxy's config migration guide":{{site.baseurl}}/admin/config-migration.html#keepproxy for more details.
-h4. Jobs API is read-only
-
-(task "#15133":https://dev.arvados.org/issues/15133 ) The legacy 'jobs' API is now read-only. It has been superceded since Arvados 1.1 by containers / container_requests (aka crunch v2). Arvados installations since the end of 2017 (v1.1.0) have probably only used containers, and are unaffected by this change.
-
-So that older Arvados sites don't lose access to legacy records, the API has been converted to read-only. Creating and updating jobs (and related types job_task, pipeline_template and pipeline_instance) is disabled and much of the business logic related has been removed, along with various other code specific to the jobs API. Specifically, the following programs associated with the jobs API have been removed: @crunch-dispatch.rb@, @crunch-job@, @crunchrunner@, @arv-run-pipeline-instance@, @arv-run@.
+h3. Delete "keep_services" records
-h4. No longer stripping ':' from strings in serialized database columns
+After all keepproxy and keepstore configurations have been migrated to the centralized configuration file, all keep_services records you added manually during installation should be removed. System logs from keepstore and keepproxy at startup, as well as the output of @arvados-server config-check@, will remind you to do this.
-(bug "#15311":https://dev.arvados.org/issues/15311 ) Strings read from serialized columns in the database with a leading ':' would have the ':' stripped after loading the record. This behavior existed due to legacy serialization behavior which stored Ruby symbols with a leading ':'. Unfortunately this corrupted fields where the leading ":" was intentional. This behavior has been removed.
+<notextile><pre><code>$ export ARVADOS_API_HOST=...
+$ export ARVADOS_API_TOKEN=...
+$ arv --format=uuid keep_service list | xargs -n1 arv keep_service delete --uuid
+</code></pre></notextile>
-You can test if any records in your database are affected by going to the API server directory and running @bundle exec rake symbols:check@. This will report which records contain fields with a leading ':' that would previously have been stripped. If there are records to be updated, you can update the database using @bundle exec rake symbols:stringify@.
+Once these old records are removed, @arv keep_service list@ will instead return the services listed under Services/Keepstore/InternalURLs and Services/Keepproxy/ExternalURL in your centralized configuration file.
-h4. Enabling Postgres trigram indexes
+h3. Enabling Postgres trigram indexes
Feature "#15106":https://dev.arvados.org/issues/15106 improves the speed and functionality of full text search by introducing trigram indexes on text searchable database columns via a migration. Prior to updating, you must first install the postgresql-contrib package on your system and subsequently run the <code class="userprint">CREATE EXTENSION pg_trgm</code> SQL command on the arvados_production database as a postgres superuser.
Subsequently, the <code class="userinput">psql -d 'arvados_production' -c '\dx'</code> command will display the installed extensions for the arvados_production database. This list should now contain @pg_trgm@.
+h3. New Workbench 2
+
+Workbench 2 is now ready for regular use. Follow the instructions to "install workbench 2":../install/install-workbench2-app.html
+
+h3. New property vocabulary format for Workbench2
-h3(#v1_4_1). v1.4.1 (2019-09-20)
+(feature "#14151":https://dev.arvados.org/issues/14151) Workbench2 supports a new vocabulary format and it isn't compatible with the previous one, please read the "workbench2 vocabulary format admin page":{{site.baseurl}}/admin/workbench2-vocabulary.html for more information.
-h4. Centos7 Python 3 dependency upgraded to rh-python36
+h3. Cloud installations only: node manager replaced by arvados-dispatch-cloud
+
+Node manager is deprecated and replaced by @arvados-dispatch-cloud@. No automated config migration is available. Follow the instructions to "install the cloud dispatcher":../install/install-dispatch-cloud.html
+
+*Only one dispatch process should be running at a time.* If you are migrating a system that currently runs Node manager and @crunch-dispatch-slurm@, it is safest to remove the @crunch-dispatch-slurm@ service entirely before installing @arvados-dispatch-cloud@.
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo systemctl --now disable crunch-dispatch-slurm</span>
+~$ <span class="userinput">sudo apt-get remove crunch-dispatch-slurm</span>
+</code></pre>
+</notextile>
+
+h3. Jobs API is read-only
+
+(task "#15133":https://dev.arvados.org/issues/15133 ) The legacy 'jobs' API is now read-only. It has been superceded since Arvados 1.1 by containers / container_requests (aka crunch v2). Arvados installations since the end of 2017 (v1.1.0) have probably only used containers, and are unaffected by this change.
+
+So that older Arvados sites don't lose access to legacy records, the API has been converted to read-only. Creating and updating jobs (and related types job_task, pipeline_template and pipeline_instance) is disabled and much of the business logic related has been removed, along with various other code specific to the jobs API. Specifically, the following programs associated with the jobs API have been removed: @crunch-dispatch.rb@, @crunch-job@, @crunchrunner@, @arv-run-pipeline-instance@, @arv-run@.
+
+h3. "/" prohibited in collection and project names
+
+(issue "#15836":https://dev.arvados.org/issues/15836) By default, Arvados now rejects new names containing the @/@ character when creating or renaming collections and projects. Previously, these names were permitted, but the resulting objects were invisible in the WebDAV "home" tree. If you prefer, you can restore the previous behavior, and optionally configure a substitution string to make the affected objects accessible via WebDAV. See @ForwardSlashNameSubstitution@ in the "configuration reference":config.html.
+
+h3. No longer stripping ':' from strings in serialized database columns
+
+(bug "#15311":https://dev.arvados.org/issues/15311 ) Strings read from serialized columns in the database with a leading ':' would have the ':' stripped after loading the record. This behavior existed due to legacy serialization behavior which stored Ruby symbols with a leading ':'. Unfortunately this corrupted fields where the leading ":" was intentional. This behavior has been removed.
+
+You can test if any records in your database are affected by going to the API server directory and running @bundle exec rake symbols:check@. This will report which records contain fields with a leading ':' that would previously have been stripped. If there are records to be updated, you can update the database using @bundle exec rake symbols:stringify@.
+
+h3. Scoped tokens should use PATCH for updates
+
+The API server accepts both PUT and PATCH for updates, but they will be normalized to PATCH by arvados-controller. Scoped tokens should be updated accordingly.
+
+
+
+h2(#v1_4_1). v1.4.1 (2019-09-20)
+
+"Upgrading from 1.4.0":#v1_4_0
+
+h3. Centos7 Python 3 dependency upgraded to rh-python36
The Python 3 dependency for Centos7 Arvados packages was upgraded from rh-python35 to rh-python36.
-h3(#v1_4_0). v1.4.0 (2019-06-05)
+h2(#v1_4_0). v1.4.0 (2019-06-05)
+
+"Upgrading from 1.3.3":#v1_3_3
-h4. Populating the new file_count and file_size_total columns on the collections table
+h3. Populating the new file_count and file_size_total columns on the collections table
As part of story "#14484":https://dev.arvados.org/issues/14484, two new columns were added to the collections table in a database migration. If your installation has a large collections table, this migration may take some time. We've seen it take ~5 minutes on an installation with 250k collections, but your mileage may vary.
The new columns are initialized with a zero value. In order to populate them, it is necessary to run a script called <code class="userinput">populate-file-info-columns-in-collections.rb</code> from the scripts directory of the API server. This can be done out of band, ideally directly after the API server has been upgraded to v1.4.0.
-h4. Stricter collection manifest validation on the API server
+h3. Stricter collection manifest validation on the API server
As a consequence of "#14482":https://dev.arvados.org/issues/14482, the Ruby SDK does a more rigorous collection manifest validation. Collections created after 2015-05 are unlikely to be invalid, however you may check for invalid manifests using the script below.
The script will return a final report enumerating any invalid collection by UUID, with its creation date and error message so you can take the proper correction measures, if needed.
-h4. Python packaging change
+h3. Python packaging change
As part of story "#9945":https://dev.arvados.org/issues/9945, the distribution packaging (deb/rpm) of our Python packages has changed. These packages now include a built-in virtualenv to reduce dependencies on system packages. We have also stopped packaging and publishing backports for all the Python dependencies of our packages, as they are no longer needed.
</pre>
</notextile>
-h4. python-arvados-cwl-runner deb/rpm package now conflicts with python-cwltool deb/rpm package
+h3. python-arvados-cwl-runner deb/rpm package now conflicts with python-cwltool deb/rpm package
As part of story "#9945":https://dev.arvados.org/issues/9945, the distribution packaging (deb/rpm) of our Python packages has changed. The python-arvados-cwl-runner package now includes a version of cwltool. If present, the python-cwltool and cwltool distribution packages will need to be uninstalled before the python-arvados-cwl-runner deb or rpm package can be installed.
-h4. Centos7 Python 3 dependency upgraded to rh-python35
+h3. Centos7 Python 3 dependency upgraded to rh-python35
As part of story "#9945":https://dev.arvados.org/issues/9945, the Python 3 dependency for Centos7 Arvados packages was upgraded from SCL python33 to rh-python35.
-h4. Centos7 package for libpam-arvados depends on the python-pam package, which is available from EPEL
+h3. Centos7 package for libpam-arvados depends on the python-pam package, which is available from EPEL
As part of story "#9945":https://dev.arvados.org/issues/9945, it was discovered that the Centos7 package for libpam-arvados was missing a dependency on the python-pam package, which is available from the EPEL repository. The dependency has been added to the libpam-arvados package. This means that going forward, the EPEL repository will need to be enabled to install libpam-arvados on Centos7.
-h4. New configuration
+h3. New configuration
Arvados is migrating to a centralized configuration file for all components. During the migration, legacy configuration files will continue to be loaded. See "Migrating Configuration":config-migration.html for details.
-h3(#v1_3_3). v1.3.3 (2019-05-14)
+h2(#v1_3_3). v1.3.3 (2019-05-14)
+
+"Upgrading from 1.3.0":#v1_3_0
This release corrects a potential data loss issue, if you are running Arvados 1.3.0 or 1.3.1 we strongly recommended disabling @keep-balance@ until you can upgrade to 1.3.3 or 1.4.0. With keep-balance disabled, there is no chance of data loss.
We've put together a "wiki page":https://dev.arvados.org/projects/arvados/wiki/Recovering_lost_data which outlines how to recover blocks which have been put in the trash, but not yet deleted, as well as how to identify any collections which have missing blocks so that they can be regenerated. The keep-balance component has been enhanced to provide a list of missing blocks and affected collections and we've provided a "utility script":https://github.com/arvados/arvados/blob/master/tools/keep-xref/keep-xref.py which can be used to identify the workflows that generated those collections and who ran those workflows, so that they can be rerun.
-h3(#v1_3_0). v1.3.0 (2018-12-05)
+h2(#v1_3_0). v1.3.0 (2018-12-05)
+
+"Upgrading from 1.2":#v1_2_0
This release includes several database migrations, which will be executed automatically as part of the API server upgrade. On large Arvados installations, these migrations will take a while. We've seen the upgrade take 30 minutes or more on installations with a lot of collections.
Support for the deprecated "jobs" API is broken in this release. Users who rely on it should not upgrade. This will be fixed in an upcoming 1.3.1 patch release, however users are "encouraged to migrate":upgrade-crunch2.html as support for the "jobs" API will be dropped in an upcoming release. Users who are already using the "containers" API are not affected.
-h3(#v1_2_1). v1.2.1 (2018-11-26)
+h2(#v1_2_1). v1.2.1 (2018-11-26)
There are no special upgrade notes for this release.
-h3(#v1_2_0). v1.2.0 (2018-09-05)
+h2(#v1_2_0). v1.2.0 (2018-09-05)
-h4. Regenerate Postgres table statistics
+"Upgrading from 1.1.2 or 1.1.3":#v1_1_2
+
+h3. Regenerate Postgres table statistics
It is recommended to regenerate the table statistics for Postgres after upgrading to v1.2.0. If autovacuum is enabled on your installation, this script would do the trick:
If you also need to do the vacuum, you could adapt the script to run 'vacuum analyze' instead of 'analyze'.
-h4. New component: arvados-controller
+h3. New component: arvados-controller
Commit "db5107dca":https://dev.arvados.org/projects/arvados/repository/revisions/db5107dca adds a new system service, arvados-controller. More detail is available in story "#13496":https://dev.arvados.org/issues/13497.
Verify your setup by confirming that API calls appear in the controller's logs (_e.g._, @journalctl -fu arvados-controller@) while loading a workbench page.
-h3(#v1_1_4). v1.1.4 (2018-04-10)
+h2(#v1_1_4). v1.1.4 (2018-04-10)
+
+"Upgrading from 1.1.3":#v1_1_3
-h4. arvados-cwl-runner regressions (2018-04-05)
+h3. arvados-cwl-runner regressions (2018-04-05)
<strong>Secondary files missing from toplevel workflow inputs</strong>
This bug has been fixed in Arvados release v1.2.0.
-h3(#v1_1_3). v1.1.3 (2018-02-08)
+h2(#v1_1_3). v1.1.3 (2018-02-08)
There are no special upgrade notes for this release.
-h3(#v1_1_2). v1.1.2 (2017-12-22)
+h2(#v1_1_2). v1.1.2 (2017-12-22)
-h4. The minimum version for Postgres is now 9.4 (2017-12-08)
+"Upgrading from 1.1.0 or 1.1.1":#v1_1_0
+
+h3. The minimum version for Postgres is now 9.4 (2017-12-08)
As part of story "#11908":https://dev.arvados.org/issues/11908, commit "8f987a9271":https://dev.arvados.org/projects/arvados/repository/revisions/8f987a9271 introduces a dependency on Postgres 9.4. Previously, Arvados required Postgres 9.3.
*# Install the @rh-postgresql94@ backport package from either Software Collections: http://doc.arvados.org/install/install-postgresql.html or the Postgres developers: https://www.postgresql.org/download/linux/redhat/
*# Restore from the backup using @psql@
-h3(#v1_1_1). v1.1.1 (2017-11-30)
+h2(#v1_1_1). v1.1.1 (2017-11-30)
There are no special upgrade notes for this release.
-h3(#v1_1_0). v1.1.0 (2017-10-24)
+h2(#v1_1_0). v1.1.0 (2017-10-24)
-h4. The minimum version for Postgres is now 9.3 (2017-09-25)
+h3. The minimum version for Postgres is now 9.3 (2017-09-25)
As part of story "#12032":https://dev.arvados.org/issues/12032, commit "68bdf4cbb1":https://dev.arvados.org/projects/arvados/repository/revisions/68bdf4cbb1 introduces a dependency on Postgres 9.3. Previously, Arvados required Postgres 9.1.
*# Install the @rh-postgresql94@ backport package from either Software Collections: http://doc.arvados.org/install/install-postgresql.html or the Postgres developers: https://www.postgresql.org/download/linux/redhat/
*# Restore from the backup using @psql@
-h3(#older). Older versions
+h2(#older). Older versions
-h4. Upgrade slower than usual (2017-06-30)
+h3. Upgrade slower than usual (2017-06-30)
As part of story "#11807":https://dev.arvados.org/issues/11807, commit "55aafbb":https://dev.arvados.org/projects/arvados/repository/revisions/55aafbb converts old "jobs" database records from YAML to JSON, making the upgrade process slower than usual.
* The conversion runs as a database migration, i.e., during the deb/rpm package upgrade process, while your API server is unavailable.
* Expect it to take about 1 minute per 20K jobs that have ever been created/run.
-h4. Service discovery overhead change in keep-web (2017-06-05)
+h3. Service discovery overhead change in keep-web (2017-06-05)
As part of story "#9005":https://dev.arvados.org/issues/9005, commit "cb230b0":https://dev.arvados.org/projects/arvados/repository/revisions/cb230b0 reduces service discovery overhead in keep-web requests.
* When upgrading keep-web _or keepproxy_ to/past this version, make sure to update API server as well. Otherwise, a bad token in a request can cause keep-web to fail future requests until either keep-web restarts or API server gets upgraded.
-h4. Node manager now has an http endpoint for management (2017-04-12)
+h3. Node manager now has an http endpoint for management (2017-04-12)
As part of story "#11349":https://dev.arvados.org/issues/11349, commit "2c094e2":https://dev.arvados.org/projects/arvados/repository/revisions/2c094e2 adds a "management" http server to nodemanager.
port = 8989</pre> (see example configuration files in source:services/nodemanager/doc or https://doc.arvados.org/install/install-nodemanager.html for more info)
* The server responds to @http://{address}:{port}/status.json@ with a summary of how many nodes are in each state (booting, busy, shutdown, etc.)
-h4. New websockets component (2017-03-23)
+h3. New websockets component (2017-03-23)
As part of story "#10766":https://dev.arvados.org/issues/10766, commit "e8cc0d7":https://dev.arvados.org/projects/arvados/repository/revisions/e8cc0d7 replaces puma with arvados-ws as the recommended websocket server.
* See http://doc.arvados.org/install/install-ws.html for install/upgrade instructions.
$ systemctl stop puma
</pre>
-h4. Change of database encoding for hashes and arrays (2017-03-06)
+h3. Change of database encoding for hashes and arrays (2017-03-06)
As part of story "#11168":https://dev.arvados.org/issues/11168, commit "660a614":https://dev.arvados.org/projects/arvados/repository/revisions/660a614 uses JSON instead of YAML to encode hashes and arrays in the database.
* Downgrading past this version is not supported, and is likely to cause errors. If this happens, the solution is to upgrade past this version.
* After upgrading, make sure to restart puma and crunch-dispatch-* processes.
-h4. Docker image format compatibility check (2017-02-03)
+h3. Docker image format compatibility check (2017-02-03)
As part of story "#10969":https://dev.arvados.org/issues/10969, commit "74a9dec":https://dev.arvados.org/projects/arvados/repository/revisions/74a9dec introduces a Docker image format compatibility check: the @arv keep docker@ command prevents users from inadvertently saving docker images that compute nodes won't be able to run.
* If your compute nodes run a version of *docker older than 1.10* you must override the default by adding to your API server configuration (@/etc/arvados/api/application.yml@): <pre><code class="yaml">docker_image_formats: ["v1"]</code></pre>
* Refer to the comments above @docker_image_formats@ in @/var/www/arvados-api/current/config/application.default.yml@ or source:services/api/config/application.default.yml or issue "#10969":https://dev.arvados.org/issues/10969 for more detail.
* *NOTE:* This does *not* include any support for migrating existing Docker images from v1 to v2 format. This will come later: for now, sites running Docker 1.9 or earlier should still *avoid upgrading Docker further than 1.9.*
-h4. Debian and RPM packages now have systemd unit files (2016-09-27)
+h3. Debian and RPM packages now have systemd unit files (2016-09-27)
Several Debian and RPM packages -- keep-balance ("d9eec0b":https://dev.arvados.org/projects/arvados/repository/revisions/d9eec0b), keep-web ("3399e63":https://dev.arvados.org/projects/arvados/repository/revisions/3399e63), keepproxy ("6de67b6":https://dev.arvados.org/projects/arvados/repository/revisions/6de67b6), and arvados-git-httpd ("9e27ddf":https://dev.arvados.org/projects/arvados/repository/revisions/9e27ddf) -- now enable their respective components using systemd. These components prefer YAML configuration files over command line flags ("3bbe1cd":https://dev.arvados.org/projects/arvados/repository/revisions/3bbe1cd).
** keepproxy - /etc/arvados/keepproxy/keepproxy.yml
** arvados-git-httpd - /etc/arvados/arv-git-httpd/arv-git-httpd.yml
-h4. Installation paths for Python modules and script changed (2016-05-31)
+h3. Installation paths for Python modules and script changed (2016-05-31)
Commits "ae72b172c8":https://dev.arvados.org/projects/arvados/repository/revisions/ae72b172c8 and "3aae316c25":https://dev.arvados.org/projects/arvados/repository/revisions/3aae316c25 change the filesystem location where Python modules and scripts are installed.
* Previous packages installed these files to the distribution's preferred path under @/usr/local@ (or the equivalent location in a Software Collection). Now they get installed to a path under @/usr@. This improves compatibility with other Python packages provided by the distribution. See "#9242":https://dev.arvados.org/issues/9242 for more background.
* If you simply import Python modules from scripts, or call Python tools relying on $PATH, you don't need to make any changes. If you have hardcoded full paths to some of these files (e.g., in symbolic links or configuration files), you will need to update those paths after this upgrade.
-h4. Crunchrunner package is required on compute and shell nodes (2016-04-25)
+h3. Crunchrunner package is required on compute and shell nodes (2016-04-25)
Commit "eebcb5e":https://dev.arvados.org/projects/arvados/repository/revisions/eebcb5e requires the crunchrunner package to be installed on compute nodes and shell nodes in order to run CWL workflows.
* On each Debian-based compute node and shell node, run: @sudo apt-get install crunchrunner@
* On each Red Hat-based compute node and shell node, run: @sudo yum install crunchrunner@
-h4. Keep permission signature algorithm change (2016-04-21)
+h3. Keep permission signature algorithm change (2016-04-21)
Commit "3c88abd":https://dev.arvados.org/projects/arvados/repository/revisions/3c88abd changes the Keep permission signature algorithm.
* All software components that generate signatures must be upgraded together. These are: keepstore, API server, keep-block-check, and keep-rsync. For example, if keepstore < 0.1.20160421183420 but API server >= 0.1.20160421183420, clients will not be able to read or write data in Keep.
* Jobs and client operations that are in progress during the upgrade (including arv-put's "resume cache") will fail.
-h4. Workbench's "Getting Started" popup disabled by default (2015-01-05)
+h3. Workbench's "Getting Started" popup disabled by default (2015-01-05)
Commit "e1276d6e":https://dev.arvados.org/projects/arvados/repository/revisions/e1276d6e disables Workbench's "Getting Started" popup by default.
* If you want new users to continue seeing this popup, set @enable_getting_started_popup: true@ in Workbench's @application.yml@ configuration.
-h4. Crunch jobs now have access to Keep-backed writable scratch storage (2015-12-03)
+h3. Crunch jobs now have access to Keep-backed writable scratch storage (2015-12-03)
Commit "5590c9ac":https://dev.arvados.org/projects/arvados/repository/revisions/5590c9ac makes a Keep-backed writable scratch directory available in crunch jobs (see "#7751":https://dev.arvados.org/issues/7751)
* All compute nodes must be upgraded to arvados-fuse >= 0.1.2015112518060 because crunch-job uses some new arv-mount flags (--mount-tmp, --mount-by-pdh) introduced in merge "346a558":https://dev.arvados.org/projects/arvados/repository/revisions/346a558
* Jobs will fail if the API server (in particular crunch-job from the arvados-cli gem) is upgraded without upgrading arvados-fuse on compute nodes.
-h4. Recommended configuration change for keep-web (2015-11-11)
+h3. Recommended configuration change for keep-web (2015-11-11)
Commit "1e2ace5":https://dev.arvados.org/projects/arvados/repository/revisions/1e2ace5 changes recommended config for keep-web (see "#5824":https://dev.arvados.org/issues/5824)
* Workbench config adds @keep_web_download_url@
* More info on the (still beta/non-TOC-linked) "keep-web doc page":http://doc.arvados.org/install/install-keep-web.html
-h4. Stopped containers are now automatically removed on compute nodes (2015-11-04)
+h3. Stopped containers are now automatically removed on compute nodes (2015-11-04)
Commit "1d1c6de":https://dev.arvados.org/projects/arvados/repository/revisions/1d1c6de removes stopped containers (see "#7444":https://dev.arvados.org/issues/7444)
* arvados-docker-cleaner removes _all_ docker containers as soon as they exit, effectively making @docker run@ default to @--rm@. If you run arvados-docker-cleaner on a host that does anything other than run crunch-jobs, and you still want to be able to use @docker start@, read the "new doc page":http://doc.arvados.org/install/install-compute-node.html to learn how to turn this off before upgrading.
-h4. New keep-web service (2015-11-04)
+h3. New keep-web service (2015-11-04)
Commit "21006cf":https://dev.arvados.org/projects/arvados/repository/revisions/21006cf adds a new keep-web service (see "#5824":https://dev.arvados.org/issues/5824).
* Nothing relies on keep-web yet, but early adopters can install it now by following http://doc.arvados.org/install/install-keep-web.html (it is not yet linked in the TOC).
+
+<notextile>
+</div>
+</notextile>
---
layout: default
navsection: admin
-title: Workbench2 Vocabulary Format
+title: User properties vocabulary
...
{% comment %}
{% codeblock as python %}
{% include 'vocabulary_migrate_py' %}
-{% endcodeblock %}
\ No newline at end of file
+{% endcodeblock %}
* Have filters only matching @[["uuid", "in", [...]]@ or @["uuid", "=", "..."]@
* Specify @count=none@
-* If @select@ is specified, it must include @uuid@
* Not specify @limit@, @offset@ or @order@
* Not request more items than the maximum response size
See "permission links":{{site.baseurl}}/api/permission-model.html#links section of the permission model.
+h3. star
+
+A **star** link is a shortcut to a project that is displayed in the user interface (Workbench) as "favorites". Users can mark their own favorites (implemented by creating or deleting **star** links).
+
+An admin can also create **star** links owned by the "All Users" group, these will be displayed to all users that have permission to read the project that has been favorited.
+
+The schema for a star link is:
+
+table(table table-bordered table-condensed).
+|_. Field|_. Value|_. Description|
+|owner_uuid|user or group uuid|Either the user that owns the favorite, or the "All Users" group for public favorites.|
+|head_uuid|project uuid|The project being favorited|
+|link_class|string of value "star"|Indicates this represents a link to a user favorite|
+
+h4. Creating a favorite
+
+@owner_uuid@ is either an individual user, or the "All Users" group. The @head_uuid@ is the project being favorited.
+
+<pre>
+$ arv link create --link '{
+ "owner_uuid": "zzzzz-j7d0g-fffffffffffffff",
+ "head_uuid": "zzzzz-j7d0g-theprojectuuid",
+ "link_class": "star"}'
+</pre>
+
+h4. Deleting a favorite
+
+<pre>
+$ arv link delete --uuid zzzzz-o0j2j-thestarlinkuuid
+</pre>
+
+h4. Listing favorites
+
+To list all 'star' links that will be displayed for a user:
+
+<pre>
+$ arv link list --filters '[
+ ["link_class", "=", "star"],
+ ["owner_uuid", "in", ["zzzzz-j7d0g-fffffffffffffff", "zzzzz-tpzed-currentuseruuid"]]]'
+</pre>
+
h3. tag
-A **tag** link describes an object using an unparsed plain text string. Tags can be used to annotate objects that are not editable, like collections and objects shared as read-only.
+A **tag** link describes an object using an unparsed plain text string. Tags can be used to annotate objects that are not directly editable by the user, like collections and objects shared as read-only.
table(table table-bordered table-condensed).
|_. tail_type→head_type|_. name→head_uuid {properties}|
td.line-numbers {
width: 2em;
}
+
+.releasenotes h2 { margin-top: 1.5em; text-decoration: underline; }
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+{% include 'notebox_begin' %}
+This section is about installing an Arvados cluster. If you are just looking to install Arvados client tools and libraries, "go to the SDK section.":{{site.baseurl}}/sdk
+{% include 'notebox_end' %}
+
Arvados components run on GNU/Linux systems, and supports AWS, GCP and Azure cloud platforms as well as on-premises installs. Arvados supports Debian and derivatives such as Ubuntu, as well as Red Hat and derivatives such as CentOS. "Arvados is Free Software":{{site.baseurl}}/user/copying/copying.html and self-install installations are not limited in any way. Commercial support and development are also available from "Curii Corporation.":mailto:info@curii.com
Arvados components can be installed and configured in a number of different ways.
The cloud dispatch service can run on any node that can connect to the Arvados API service, the cloud provider's API, and the SSH service on cloud VMs. It is not resource-intensive, so you can run it on the API server node.
-h2(#create-image). Create compute node VM image
+h2(#create-image). Create compute node VM image and configure resolver
-Create a VM image following the steps "to set up a compute node":crunch2-slurm/install-compute-node.html
+Set up a VM following the steps "to set up a compute node":crunch2-slurm/install-compute-node.html
-On the compute VM image, add the API server's internal IP address to @/etc/hosts@, this will ensure that it contacts the API server on the private network and not through the public interface. For example:
+Compute nodes must be able to resolve the hostnames of the API server and any keepstore servers to your internal IP addresses. You can do this by running an internal DNS resolver and configuring the compute VMs to use that resolver, or by hardcoding the services in the @/etc/hosts@ file. For example:
<notextile><pre><code>10.20.30.40 <span class="userinput">ClusterID.example.com</span>
+10.20.30.41 <span class="userinput">keep1.ClusterID.example.com</span>
+10.20.30.42 <span class="userinput">keep2.ClusterID.example.com</span>
</code></pre></notextile>
+Once the VM is fully configured, create a reusable VM image from it and make note of the image id.
+
h2(#update-config). Update config.yml
h3. Create a private key
---
layout: default
navsection: installguide
-title: Install Workbench2 (beta)
+title: Install Workbench 2
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
Workbench2 is the web-based user interface for Arvados.
{% include 'notebox_begin' %}
-Workbench2 is the replacement for Arvados Workbench. Workbench2 is currently in <i>beta</i>, it is not yet feature complete.
+Workbench2 is the replacement for Arvados Workbench. Workbench2 is suitable for day-to-day use, but does not yet implement every feature of the traditional Workbench.
{% include 'notebox_end' %}
h2(#configure). Update config.yml
This section documents language bindings for the "Arvados API":{{site.baseurl}}/api and Keep that are available for various programming languages. Not all features are available in every SDK. The most complete SDK is the Python SDK. Note that this section only gives a high level overview of each SDK. Consult the "Arvados API":{{site.baseurl}}/api section for detailed documentation about Arvados API calls available on each resource.
-* "Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html
+* "Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html (also includes essential command line tools such as "arv-put" and "arv-get")
* "Command line SDK":{{site.baseurl}}/sdk/cli/install.html ("arv")
* "Go SDK":{{site.baseurl}}/sdk/go/index.html
* "R SDK":{{site.baseurl}}/sdk/R/index.html
# Based on Debian Stretch
FROM debian:stretch-slim
-MAINTAINER Ward Vandewege <wvandewege@veritasgenetics.com>
+MAINTAINER Peter Amstutz <peter.amstutz@curii.com>
ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update -q
RUN apt-get install -yq --no-install-recommends nodejs \
python-arvados-python-client=$python_sdk_version \
- python-arvados-cwl-runner=$cwl_runner_version
+ python3-arvados-cwl-runner=$cwl_runner_version
# use the Python executable from the python-arvados-cwl-runner package
-RUN rm -f /usr/bin/python && ln -s /usr/share/python2.7/dist/python-arvados-cwl-runner/bin/python /usr/bin/python
+RUN rm -f /usr/bin/python && ln -s /usr/share/python2.7/dist/python-arvados-python-client/bin/python /usr/bin/python
+RUN rm -f /usr/bin/python3 && ln -s /usr/share/python3/dist/python3-arvados-cwl-runner/bin/python /usr/bin/python3
# Install dependencies and set up system.
RUN /usr/sbin/adduser --disabled-password \
identification, and does not retrieve any other personal
information.</i>
+ # Workbench screen displayed to inactive users. This is HTML
+ # text that will be incorporated directly onto the page.
InactivePageHTML: |
<img src="/arvados-logo-big.png" style="width: 20%; float: right; padding: 1em;" />
<h3>Hi! You're logged in, but...</h3>
<p>An administrator must activate your account before you can get
any further.</p>
+ # Connecting to Arvados shell VMs tends to be site-specific.
+ # Put any special instructions here. This is HTML text that will
+ # be incorporated directly onto the Workbench page.
+ SSHHelpPageHTML: |
+ <a href="https://doc.arvados.org/user/getting_started/ssh-access-unix.html">Accessing an Arvados VM with SSH</a> (generic instructions).
+ Site configurations vary. Contact your local cluster administrator if you have difficulty accessing an Arvados shell node.
+
# Bypass new (Arvados 1.5) API implementations, and hand off
# requests directly to Rails instead. This can provide a temporary
# workaround for clients that are incompatible with the new API
"Workbench.VocabularyURL": true,
"Workbench.WelcomePageHTML": true,
"Workbench.InactivePageHTML": true,
+ "Workbench.SSHHelpPageHTML": true,
}
func redactUnsafe(m map[string]interface{}, mPrefix, lookupPrefix string) error {
identification, and does not retrieve any other personal
information.</i>
+ # Workbench screen displayed to inactive users. This is HTML
+ # text that will be incorporated directly onto the page.
InactivePageHTML: |
<img src="/arvados-logo-big.png" style="width: 20%; float: right; padding: 1em;" />
<h3>Hi! You're logged in, but...</h3>
<p>An administrator must activate your account before you can get
any further.</p>
+ # Connecting to Arvados shell VMs tends to be site-specific.
+ # Put any special instructions here. This is HTML text that will
+ # be incorporated directly onto the Workbench page.
+ SSHHelpPageHTML: |
+ <a href="https://doc.arvados.org/user/getting_started/ssh-access-unix.html">Accessing an Arvados VM with SSH</a> (generic instructions).
+ Site configurations vary. Contact your local cluster administrator if you have difficulty accessing an Arvados shell node.
+
# Bypass new (Arvados 1.5) API implementations, and hand off
# requests directly to Rails instead. This can provide a temporary
# workaround for clients that are incompatible with the new API
if !remote.Proxy {
continue
}
- remotes[id] = rpc.NewConn(id, &url.URL{Scheme: remote.Scheme, Host: remote.Host}, remote.Insecure, saltedTokenProvider(local, id))
+ conn := rpc.NewConn(id, &url.URL{Scheme: remote.Scheme, Host: remote.Host}, remote.Insecure, saltedTokenProvider(local, id))
+ // Older versions of controller rely on the Via header
+ // to detect loops.
+ conn.SendHeader = http.Header{"Via": {"HTTP/1.1 arvados-controller"}}
+ remotes[id] = conn
}
return &Conn{
// or "" for the local backend.
//
// A non-nil error means all backends failed.
-func (conn *Conn) tryLocalThenRemotes(ctx context.Context, fn func(context.Context, string, backend) error) error {
- if err := fn(ctx, "", conn.local); err == nil || errStatus(err) != http.StatusNotFound {
+func (conn *Conn) tryLocalThenRemotes(ctx context.Context, forwardedFor string, fn func(context.Context, string, backend) error) error {
+ if err := fn(ctx, "", conn.local); err == nil || errStatus(err) != http.StatusNotFound || forwardedFor != "" {
+ // Note: forwardedFor != "" means this request came
+ // from a remote cluster, so we don't take a second
+ // hop. This avoids cycles, redundant calls to a
+ // mutually reachable remote, and use of double-salted
+ // tokens.
return err
}
} else {
// UUID is a PDH
first := make(chan arvados.Collection, 1)
- err := conn.tryLocalThenRemotes(ctx, func(ctx context.Context, remoteID string, be backend) error {
- c, err := be.CollectionGet(ctx, options)
+ err := conn.tryLocalThenRemotes(ctx, options.ForwardedFor, func(ctx context.Context, remoteID string, be backend) error {
+ remoteOpts := options
+ remoteOpts.ForwardedFor = conn.cluster.ClusterID + "-" + options.ForwardedFor
+ c, err := be.CollectionGet(ctx, remoteOpts)
if err != nil {
return err
}
//
// * len(Order)==0
//
-// * Each filter must be either "uuid = ..." or "uuid in [...]".
+// * Each filter is either "uuid = ..." or "uuid in [...]".
//
// * The maximum possible response size (total number of objects that
// could potentially be matched by all of the specified filters)
}
}
- if len(todoByRemote) > 1 {
- if cannotSplit {
- return httpErrorf(http.StatusBadRequest, "cannot execute federated list query: each filter must be either 'uuid = ...' or 'uuid in [...]'")
- }
- if opts.Count != "none" {
- return httpErrorf(http.StatusBadRequest, "cannot execute federated list query unless count==\"none\"")
- }
- if opts.Limit >= 0 || opts.Offset != 0 || len(opts.Order) > 0 {
- return httpErrorf(http.StatusBadRequest, "cannot execute federated list query with limit, offset, or order parameter")
- }
- if max := conn.cluster.API.MaxItemsPerResponse; nUUIDs > max {
- return httpErrorf(http.StatusBadRequest, "cannot execute federated list query because number of UUIDs (%d) exceeds page size limit %d", nUUIDs, max)
- }
- selectingUUID := false
- for _, attr := range opts.Select {
- if attr == "uuid" {
- selectingUUID = true
- }
- }
- if opts.Select != nil && !selectingUUID {
- return httpErrorf(http.StatusBadRequest, "cannot execute federated list query with a select parameter that does not include uuid")
- }
+ if len(todoByRemote) == 0 {
+ return nil
+ }
+ if len(todoByRemote) == 1 && todoByRemote[conn.cluster.ClusterID] != nil {
+ // All UUIDs are local, so proxy a single request. The
+ // generic case has some limitations (see below) which
+ // we don't want to impose on local requests.
+ _, err := fn(ctx, conn.cluster.ClusterID, conn.local, opts)
+ return err
+ }
+ if cannotSplit {
+ return httpErrorf(http.StatusBadRequest, "cannot execute federated list query: each filter must be either 'uuid = ...' or 'uuid in [...]'")
+ }
+ if opts.Count != "none" {
+ return httpErrorf(http.StatusBadRequest, "cannot execute federated list query unless count==\"none\"")
+ }
+ if opts.Limit >= 0 || opts.Offset != 0 || len(opts.Order) > 0 {
+ return httpErrorf(http.StatusBadRequest, "cannot execute federated list query with limit, offset, or order parameter")
+ }
+ if max := conn.cluster.API.MaxItemsPerResponse; nUUIDs > max {
+ return httpErrorf(http.StatusBadRequest, "cannot execute federated list query because number of UUIDs (%d) exceeds page size limit %d", nUUIDs, max)
}
ctx, cancel := context.WithCancel(ctx)
return
}
remoteOpts := opts
+ if remoteOpts.Select != nil {
+ // We always need to select UUIDs to
+ // use the response, even if our
+ // caller doesn't.
+ remoteOpts.Select = append([]string{"uuid"}, remoteOpts.Select...)
+ }
for len(todo) > 0 {
if len(batch) > len(todo) {
// Reduce batch to just the todo's
"context"
"fmt"
"net/http"
+ "reflect"
"sort"
"git.arvados.org/arvados.git/sdk/go/arvados"
break
}
if cl.matchFilters(c, options.Filters) {
+ if reflect.DeepEqual(options.Select, []string{"uuid", "name"}) {
+ c = arvados.Collection{UUID: c.UUID, Name: c.Name}
+ } else if reflect.DeepEqual(options.Select, []string{"name"}) {
+ c = arvados.Collection{Name: c.Name}
+ } else if len(options.Select) > 0 {
+ panic(fmt.Sprintf("not implemented: options=%#v", options))
+ }
resp.Items = append(resp.Items, c)
}
}
offset int
order []string
filters []arvados.Filter
+ selectfields []string
expectUUIDs []string
expectCalls []int // number of API calls to backends
expectStatus int
})
}
+func (s *CollectionListSuite) TestCollectionListOneLocalDeselectingUUID(c *check.C) {
+ s.test(c, listTrial{
+ count: "none",
+ limit: -1,
+ filters: []arvados.Filter{{"uuid", "=", s.uuids[0][0]}},
+ selectfields: []string{"name"},
+ expectUUIDs: []string{""}, // select=name is honored
+ expectCalls: []int{1, 0, 0},
+ })
+}
+
func (s *CollectionListSuite) TestCollectionListOneLocalUsingInOperator(c *check.C) {
s.test(c, listTrial{
count: "none",
})
}
+func (s *CollectionListSuite) TestCollectionListOneRemoteDeselectingUUID(c *check.C) {
+ s.test(c, listTrial{
+ count: "none",
+ limit: -1,
+ filters: []arvados.Filter{{"uuid", "=", s.uuids[1][0]}},
+ selectfields: []string{"name"},
+ expectUUIDs: []string{s.uuids[1][0]}, // uuid is returned, despite not being selected
+ expectCalls: []int{0, 1, 0},
+ })
+}
+
func (s *CollectionListSuite) TestCollectionListOneLocalOneRemote(c *check.C) {
s.test(c, listTrial{
count: "none",
})
}
+func (s *CollectionListSuite) TestCollectionListOneLocalOneRemoteDeselectingUUID(c *check.C) {
+ s.test(c, listTrial{
+ count: "none",
+ limit: -1,
+ filters: []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}}},
+ selectfields: []string{"name"},
+ expectUUIDs: []string{s.uuids[0][0], s.uuids[1][0]}, // uuid is returned, despite not being selected
+ expectCalls: []int{1, 1, 0},
+ })
+}
+
func (s *CollectionListSuite) TestCollectionListTwoRemotes(c *check.C) {
s.test(c, listTrial{
count: "none",
Offset: trial.offset,
Order: trial.order,
Filters: trial.filters,
+ Select: trial.selectfields,
})
if trial.expectStatus != 0 {
c.Assert(err, check.NotNil)
from .fsaccess import CollectionFetcher
from .pathmapper import NoFollowPathMapper, trim_listing
from .perf import Perf
+from ._version import __version__
logger = logging.getLogger('arvados.cwl-runner')
metrics = logging.getLogger('arvados.cwl-runner.metrics')
(docker_req, docker_is_req) = self.get_requirement("DockerRequirement")
if not docker_req:
- docker_req = {"dockerImageId": "arvados/jobs"}
+ docker_req = {"dockerImageId": "arvados/jobs:"+__version__}
container_request["container_image"] = arv_docker_get_image(self.arvrunner.api,
docker_req,
for req in job_reqs:
tool.requirements.append(req)
- def arv_executor(self, tool, job_order, runtimeContext, logger=None):
+ def arv_executor(self, updated_tool, job_order, runtimeContext, logger=None):
self.debug = runtimeContext.debug
- tool.visit(self.check_features)
+ updated_tool.visit(self.check_features)
self.project_uuid = runtimeContext.project_uuid
self.pipeline = None
raise Exception("--submit-request-uuid requires containers API, but using '{}' api".format(self.work_api))
if not runtimeContext.name:
- runtimeContext.name = self.name = tool.tool.get("label") or tool.metadata.get("label") or os.path.basename(tool.tool["id"])
+ runtimeContext.name = self.name = updated_tool.tool.get("label") or updated_tool.metadata.get("label") or os.path.basename(updated_tool.tool["id"])
# Upload local file references in the job order.
job_order = upload_job_order(self, "%s input" % runtimeContext.name,
- tool, job_order)
+ updated_tool, job_order)
+
+ # the last clause means: if it is a command line tool, and we
+ # are going to wait for the result, and always_submit_runner
+ # is false, then we don't submit a runner process.
submitting = (runtimeContext.update_workflow or
runtimeContext.create_workflow or
(runtimeContext.submit and not
- (tool.tool["class"] == "CommandLineTool" and
+ (updated_tool.tool["class"] == "CommandLineTool" and
runtimeContext.wait and
not runtimeContext.always_submit_runner)))
if submitting:
# Document may have been auto-updated. Reload the original
# document with updating disabled because we want to
- # submit the original document, not the auto-updated one.
- tool = load_tool(tool.tool["id"], loadingContext)
+ # submit the document with its original CWL version, not
+ # the auto-updated one.
+ tool = load_tool(updated_tool.tool["id"], loadingContext)
+ else:
+ tool = updated_tool
# Upload direct dependencies of workflow steps, get back mapping of files to keep references.
# Also uploads docker images.
if runtimeContext.submit:
# Submit a runner job to run the workflow for us.
if self.work_api == "containers":
- if tool.tool["class"] == "CommandLineTool" and runtimeContext.wait and (not runtimeContext.always_submit_runner):
- runtimeContext.runnerjob = tool.tool["id"]
+ if submitting:
+ tool = RunnerContainer(self, updated_tool,
+ tool, loadingContext, runtimeContext.enable_reuse,
+ self.output_name,
+ self.output_tags,
+ submit_runner_ram=runtimeContext.submit_runner_ram,
+ name=runtimeContext.name,
+ on_error=runtimeContext.on_error,
+ submit_runner_image=runtimeContext.submit_runner_image,
+ intermediate_output_ttl=runtimeContext.intermediate_output_ttl,
+ merged_map=merged_map,
+ priority=runtimeContext.priority,
+ secret_store=self.secret_store,
+ collection_cache_size=runtimeContext.collection_cache_size,
+ collection_cache_is_default=self.should_estimate_cache_size)
else:
- tool = RunnerContainer(self, tool, loadingContext, runtimeContext.enable_reuse,
- self.output_name,
- self.output_tags,
- submit_runner_ram=runtimeContext.submit_runner_ram,
- name=runtimeContext.name,
- on_error=runtimeContext.on_error,
- submit_runner_image=runtimeContext.submit_runner_image,
- intermediate_output_ttl=runtimeContext.intermediate_output_ttl,
- merged_map=merged_map,
- priority=runtimeContext.priority,
- secret_store=self.secret_store,
- collection_cache_size=runtimeContext.collection_cache_size,
- collection_cache_is_default=self.should_estimate_cache_size)
+ runtimeContext.runnerjob = tool.tool["id"]
if runtimeContext.cwl_runner_job is not None:
self.uuid = runtimeContext.cwl_runner_job.get('uuid')
return f.read()
if url.startswith("arvwf:"):
record = self.api_client.workflows().get(uuid=url[6:]).execute(num_retries=self.num_retries)
- definition = record["definition"] + ('\nlabel: "%s"\n' % record["name"].replace('"', '\\"'))
- return definition
+ definition = yaml.round_trip_load(record["definition"])
+ definition["label"] = record["name"]
+ return yaml.round_trip_dump(definition)
return super(CollectionFetcher, self).fetch_text(url)
def check_exists(self, url):
"""Base class for runner processes, which submit an instance of
arvados-cwl-runner and wait for the final result."""
- def __init__(self, runner, tool, loadingContext, enable_reuse,
+ def __init__(self, runner, updated_tool,
+ tool, loadingContext, enable_reuse,
output_name, output_tags, submit_runner_ram=0,
name=None, on_error=None, submit_runner_image=None,
intermediate_output_ttl=0, merged_map=None,
collection_cache_is_default=True):
loadingContext = loadingContext.copy()
- loadingContext.metadata = loadingContext.metadata.copy()
- loadingContext.metadata["cwlVersion"] = INTERNAL_VERSION
+ loadingContext.metadata = updated_tool.metadata.copy()
- super(Runner, self).__init__(tool.tool, loadingContext)
+ super(Runner, self).__init__(updated_tool.tool, loadingContext)
self.arvrunner = runner
self.embedded_tool = tool
],
test_suite='tests',
tests_require=[
- 'mock>=1.0',
+ 'mock>=1.0,<4',
'subprocess32>=3.5.1',
],
zip_safe=True
stubs.expect_container_request_uuid + '\n')
self.assertEqual(exited, 0)
+
+ @stubs
+ def test_submit_container_tool(self, stubs):
+ # test for issue #16139
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug",
+ "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
+ stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+
+ self.assertEqual(stubs.capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+ self.assertEqual(exited, 0)
+
@stubs
def test_submit_container_no_reuse(self, stubs):
exited = arvados_cwl.main(
--- /dev/null
+clipper clupper
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Test case for arvados-cwl-runner
+#
+# Used to test whether scanning a tool file for dependencies (e.g. default
+# value blub.txt) and uploading to Keep works as intended.
+
+class: CommandLineTool
+cwlVersion: v1.0
+requirements:
+ - class: DockerRequirement
+ dockerPull: debian:8
+inputs:
+ - id: x
+ type: File
+ secondaryFiles:
+ - .cat
+ inputBinding:
+ valueFrom: $(self.path).cat
+ position: 1
+outputs: []
+baseCommand: cat
--- /dev/null
+x:
+ class: File
+ location: blub.txt
UUID string `json:"uuid"`
Select []string `json:"select"`
IncludeTrash bool `json:"include_trash"`
+ ForwardedFor string `json:"forwarded_for"`
}
type UntrashOptions struct {
VocabularyURL string
WelcomePageHTML string
InactivePageHTML string
+ SSHHelpPageHTML string
}
ForceLegacyAPI14 bool
'Programming Language :: Python :: 3',
],
test_suite='tests',
- tests_require=['pbr<1.7.0', 'mock>=1.0', 'PyYAML'],
+ tests_require=['pbr<1.7.0', 'mock>=1.0,<4', 'PyYAML'],
zip_safe=False
)
# setup succeeded. send email to user
if params[:send_notification_email]
- UserNotifier.account_is_setup(@object).deliver_now
+ begin
+ UserNotifier.account_is_setup(@object).deliver_now
+ rescue => e
+ logger.warn "Failed to send email to #{@object.email}: #{e}"
+ end
end
send_json kind: "arvados#HashList", items: @response.as_api_response(nil)
def account_is_setup(user)
@user = user
- mail(to: user.email, subject: 'Welcome to Arvados - shell account enabled')
+ mail(to: user.email, subject: 'Welcome to Arvados - account enabled')
end
end
return
end
(managed_props.keys - self.properties.keys).each do |key|
- if managed_props[key].has_key?('Value')
- self.properties[key] = managed_props[key]['Value']
- elsif managed_props[key]['Function'].andand == 'original_owner'
+ if managed_props[key]['Function'] == 'original_owner'
self.properties[key] = self.user_owner_uuid
+ elsif managed_props[key]['Value']
+ self.properties[key] = managed_props[key]['Value']
else
logger.warn "Unidentified default property definition '#{key}': #{managed_props[key].inspect}"
end
assert_equal Rails.configuration.Users.UserNotifierEmailFrom, setup_email.from[0]
assert_equal 'foo@example.com', setup_email.to[0]
- assert_equal 'Welcome to Arvados - shell account enabled', setup_email.subject
+ assert_equal 'Welcome to Arvados - account enabled', setup_email.subject
assert (setup_email.body.to_s.include? 'Your Arvados shell account has been set up'),
'Expected Your Arvados shell account has been set up in email body'
assert (setup_email.body.to_s.include? "#{Rails.configuration.Services.Workbench1.ExternalURL}users/#{created['uuid']}/virtual_machines"), 'Expected virtual machines url in email body'
# Test the body of the sent email contains what we expect it to
assert_equal Rails.configuration.Users.UserNotifierEmailFrom, email.from.first
assert_equal user.email, email.to.first
- assert_equal 'Welcome to Arvados - shell account enabled', email.subject
+ assert_equal 'Welcome to Arvados - account enabled', email.subject
assert (email.body.to_s.include? 'Your Arvados shell account has been set up'),
'Expected Your Arvados shell account has been set up in email body'
assert (email.body.to_s.include? Rails.configuration.Services.Workbench1.ExternalURL.to_s),
[Service]
Type=notify
ExecStart=/usr/bin/crunch-dispatch-slurm
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=1
LimitNOFILE=1000000
import os
import re
+SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+
+def choose_version_from():
+ sdk_ts = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', os.path.join(SETUP_DIR, "../../sdk/python")]).strip()
+ cwl_ts = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', SETUP_DIR]).strip()
+ if int(sdk_ts) > int(cwl_ts):
+ getver = os.path.join(SETUP_DIR, "../../sdk/python")
+ else:
+ getver = SETUP_DIR
+ return getver
+
def git_version_at_commit():
- curdir = os.path.dirname(os.path.abspath(__file__))
+ curdir = choose_version_from()
myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
'--format=%H', curdir]).strip()
myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
[Service]
Type=simple
ExecStart=/usr/bin/arvados-health
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=1
[Service]
Type=simple
ExecStart=/usr/bin/keep-balance -commit-pulls -commit-trash
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=10s
Nice=19
[Service]
Type=notify
ExecStart=/usr/bin/keep-web
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=1
[Service]
Type=notify
ExecStart=/usr/bin/keepproxy
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=1
Environment=GOGC=10
Type=notify
ExecStart=/usr/bin/keepstore
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=1
[Service]
Type=notify
ExecStart=/usr/bin/arvados-ws
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=1
--publish=3001:3001
--publish=8000:8000
--publish=8900:8900
- --publish=9001:9001
+ --publish=9000:9000
--publish=9002:9002
- --publish=25100:25100
- --publish=25107:25107
- --publish=25108:25108
+ --publish=25101:25101
--publish=8001:8001
--publish=8002:8002"
else
export HOME=$(getent passwd arvbox | cut -d: -f6)
defaultdev=$(/sbin/ip route|awk '/default/ { print $5 }')
+dockerip=$(/sbin/ip route | grep default | awk '{ print $3 }')
containerip=$(ip addr show $defaultdev | grep 'inet ' | sed 's/ *inet \(.*\)\/.*/\1/')
if test -s /var/run/localip_override ; then
localip=$(cat /var/run/localip_override)
}
http {
- access_log off;
- include /etc/nginx/mime.types;
- default_type application/octet-stream;
- client_max_body_size 128M;
-
- geo \$external_client {
- default 1;
- 127.0.0.0/8 0;
- $containerip/32 0;
- }
-
- server {
- listen ${services[doc]} default_server;
- listen [::]:${services[doc]} default_server;
- root /usr/src/arvados/doc/.site;
- index index.html;
- server_name _;
- }
+ access_log off;
+ include /etc/nginx/mime.types;
+ default_type application/octet-stream;
+ client_max_body_size 128M;
+
+ geo \$external_client {
+ default 1;
+ 127.0.0.0/8 0;
+ $containerip/32 0;
+ $dockerip/32 0;
+ }
+
+ server {
+ listen ${services[doc]} default_server;
+ listen [::]:${services[doc]} default_server;
+ root /usr/src/arvados/doc/.site;
+ index index.html;
+ server_name _;
+ }
server {
listen 80 default_server;
proxy_set_header X-Forwarded-Proto https;
proxy_set_header X-External-Client \$external_client;
proxy_redirect off;
+ # This turns off response caching
+ proxy_buffering off;
}
}
-upstream arvados-ws {
- server localhost:${services[websockets]};
-}
-server {
- listen *:${services[websockets-ssl]} ssl default_server;
- server_name websockets;
-
- proxy_connect_timeout 90s;
- proxy_read_timeout 300s;
-
- ssl on;
- ssl_certificate "${server_cert}";
- ssl_certificate_key "${server_cert_key}";
-
- location / {
- proxy_pass http://arvados-ws;
- proxy_set_header Upgrade \$http_upgrade;
- proxy_set_header Connection "upgrade";
- proxy_set_header Host \$http_host;
- proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+ upstream arvados-ws {
+ server localhost:${services[websockets]};
+ }
+ server {
+ listen *:${services[websockets-ssl]} ssl default_server;
+ server_name websockets;
+
+ proxy_connect_timeout 90s;
+ proxy_read_timeout 300s;
+
+ ssl on;
+ ssl_certificate "${server_cert}";
+ ssl_certificate_key "${server_cert_key}";
+
+ location / {
+ proxy_pass http://arvados-ws;
+ proxy_set_header Upgrade \$http_upgrade;
+ proxy_set_header Connection "upgrade";
+ proxy_set_header Host \$http_host;
+ proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+ }
}
-}
upstream workbench2 {
server localhost:${services[workbench2]};
}
}
-
upstream keepproxy {
server localhost:${services[keepproxy]};
}
projects / arvados.git / commitdiff