i18n (>= 0.7, < 2)
minitest (~> 5.1)
tzinfo (~> 1.1)
- addressable (2.7.0)
+ addressable (2.8.0)
public_suffix (>= 2.0.2, < 5.0)
andand (1.3.3)
angularjs-rails (1.3.15)
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.5 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Install Bash 4.4.12 // see https://dev.arvados.org/issues/15612
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.5 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Install golang binary
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.5 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Install golang binary
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.5 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Install golang binary
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.5 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Install golang binary
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.5 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2
+ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.9
# Install Bash 4.4.12 // see https://dev.arvados.org/issues/15612
RUN cd /usr/local/src \
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.5 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2
+ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
# udev daemon can't start in a container, so don't try.
RUN mkdir -p /etc/udev/disabled
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.5 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2
+ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
# udev daemon can't start in a container, so don't try.
RUN mkdir -p /etc/udev/disabled
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.5 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2
+ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
# udev daemon can't start in a container, so don't try.
RUN mkdir -p /etc/udev/disabled
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.5 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2
+ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
# udev daemon can't start in a container, so don't try.
RUN mkdir -p /etc/udev/disabled
# initialize git_internal_dir
# usually /var/lib/arvados/internal.git (set in application.default.yml )
if [ "$APPLICATION_READY" = "1" ]; then
- GIT_INTERNAL_DIR=$($COMMAND_PREFIX bundle exec rake config:dump 2>&1 | grep GitInternalDir | awk '{ print $2 }' |tr -d '"')
+ GIT_INTERNAL_DIR=$($COMMAND_PREFIX bin/rake config:dump 2>&1 | grep GitInternalDir | awk '{ print $2 }' |tr -d '"')
if [ ! -e "$GIT_INTERNAL_DIR" ]; then
run_and_report "Creating git_internal_dir '$GIT_INTERNAL_DIR'" \
mkdir -p "$GIT_INTERNAL_DIR"
}
prepare_database() {
- DB_MIGRATE_STATUS=`$COMMAND_PREFIX bundle exec rake db:migrate:status 2>&1 || true`
+ DB_MIGRATE_STATUS=`$COMMAND_PREFIX bin/rake db:migrate:status 2>&1 || true`
if echo "$DB_MIGRATE_STATUS" | grep -qF 'Schema migrations table does not exist yet.'; then
# The database exists, but the migrations table doesn't.
- run_and_report "Setting up database" $COMMAND_PREFIX bundle exec \
- rake "$RAILSPKG_DATABASE_LOAD_TASK" db:seed
+ run_and_report "Setting up database" $COMMAND_PREFIX bin/rake \
+ "$RAILSPKG_DATABASE_LOAD_TASK" db:seed
elif echo "$DB_MIGRATE_STATUS" | grep -q '^database: '; then
run_and_report "Running db:migrate" \
- $COMMAND_PREFIX bundle exec rake db:migrate
+ $COMMAND_PREFIX bin/rake db:migrate
elif echo "$DB_MIGRATE_STATUS" | grep -q 'database .* does not exist'; then
if ! run_and_report "Running db:setup" \
- $COMMAND_PREFIX bundle exec rake db:setup 2>/dev/null; then
+ $COMMAND_PREFIX bin/rake db:setup 2>/dev/null; then
echo "Warning: unable to set up database." >&2
DATABASE_READY=0
fi
cd "$RELEASE_PATH"
export RAILS_ENV=production
- if ! $COMMAND_PREFIX bundle --version >/dev/null; then
- run_and_report "Installing bundler" $COMMAND_PREFIX gem install bundler --version 1.17.3
+ if ! $COMMAND_PREFIX bundle --version >/dev/null 2>&1; then
+ run_and_report "Installing bundler" $COMMAND_PREFIX gem install bundler --version 2.2.19 --no-document
fi
+ run_and_report "Running bundle config set --local path $SHARED_PATH/vendor_bundle" \
+ $COMMAND_PREFIX bin/bundle config set --local path $SHARED_PATH/vendor_bundle
+
run_and_report "Running bundle install" \
- $COMMAND_PREFIX bundle install --path $SHARED_PATH/vendor_bundle --local --quiet
+ $COMMAND_PREFIX bin/bundle install --local --quiet
echo -n "Ensuring directory and file permissions ..."
# Ensure correct ownership of a few files
# warn about config errors (deprecated/removed keys from
# previous version, etc)
run_and_report "Checking configuration for completeness" \
- $COMMAND_PREFIX bundle exec rake config:check || APPLICATION_READY=0
+ $COMMAND_PREFIX bin/rake config:check || APPLICATION_READY=0
else
APPLICATION_READY=0
fi
mv /tmp/x /etc/arvados/config.yml
perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml
- ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake npm:install >"$STDOUT_IF_DEBUG"
- ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >"$STDOUT_IF_DEBUG"
+ ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake npm:install >"$STDOUT_IF_DEBUG"
+ ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake assets:precompile >"$STDOUT_IF_DEBUG"
# Remove generated configuration files so they don't go in the package.
rm -rf /etc/arvados/
|| fatal 'rvm gemset setup'
rvm env
- (bundle version | grep -q 2.0.2) || gem install bundler -v 2.0.2
+ (bundle version | grep -q 2.2.19) || gem install bundler -v 2.2.19
bundle="$(which bundle)"
echo "$bundle"
- "$bundle" version | grep 2.0.2 || fatal 'install bundler'
+ "$bundle" version | grep 2.2.19 || fatal 'install bundler'
else
# When our "bundle install"s need to install new gems to
# satisfy dependencies, we want them to go where "gem install
(
export HOME=$GEMHOME
bundlers="$(gem list --details bundler)"
- versions=(1.16.6 1.17.3 2.0.2)
+ versions=(2.2.19)
for v in ${versions[@]}; do
if ! echo "$bundlers" | fgrep -q "($v)"; then
gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done)
- architecture/manifest-format.html.textile.liquid
- Computation with Crunch:
- api/execution.html.textile.liquid
+ - architecture/dispatchcloud.html.textile.liquid
- Other:
- api/permission-model.html.textile.liquid
- architecture/federation.html.textile.liquid
- Data Management:
- admin/collection-versioning.html.textile.liquid
- admin/collection-managed-properties.html.textile.liquid
+ - admin/restricting-upload-download.html.textile.liquid
- admin/keep-balance.html.textile.liquid
- admin/controlling-container-reuse.html.textile.liquid
- admin/logs-table-management.html.textile.liquid
<notextile>
<pre><code>~$ <span class="userinput">cd /var/www/arvados-api/current</span>
-$ <span class="userinput">sudo -u <b>webserver-user</b> RAILS_ENV=production bundle exec script/create_superuser_token.rb</span>
+$ <span class="userinput">sudo -u <b>webserver-user</b> RAILS_ENV=production bin/bundle exec script/create_superuser_token.rb</span>
zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
</code></pre>
</notextile>
This template recognizes four variables:
* railshost: The hostname included in the prompt, to let the user know where to run the command. If this is the empty string, no hostname will be displayed. Default "apiserver".
* railsdir: The directory included in the prompt, to let the user know where to run the command. Default "/var/www/arvados-api/current".
-* railscmd: The full command to run. Default "bundle exec rails console".
+* railscmd: The full command to run. Default "bin/rails console".
* railsout: The expected output of the command, if any.
{% endcomment %} Change *@webserver-user@* to the user that runs your web server process. If you install Phusion Passenger as we recommend, this is *@www-data@* on Debian-based systems, and *@nginx@* on Red Hat-based systems.
{% endunless %}
{% unless railscmd %}
- {% assign railscmd = "bundle exec rails console" %}
+ {% assign railscmd = "bin/rails console" %}
{% endunless %}
<notextile>
Change to the API server directory and use the following commands:
<pre>
-$ RAILS_ENV=production bundle exec rake config:migrate > config.yml
+$ RAILS_ENV=production bin/rake config:migrate > config.yml
$ cp config.yml /etc/arvados/config.yml
</pre>
If you wish to update @config.yml@ configuration by hand, or check that everything has been migrated, use @config:diff@ to print configuration items that differ between @application.yml@ and the system @config.yml@.
<pre>
-$ RAILS_ENV=production bundle exec rake config:diff
+$ RAILS_ENV=production bin/rake config:diff
</pre>
This command will also report if no migrations are required.
Change to the workbench server directory and use the following commands:
<pre>
-$ RAILS_ENV=production bundle exec rake config:migrate > config.yml
+$ RAILS_ENV=production bin/rake config:migrate > config.yml
$ cp config.yml /etc/arvados/config.yml
</pre>
If you wish to update @config.yml@ configuration by hand, or check that everything has been migrated, use @config:diff@ to print configuration items that differ between @application.yml@ and the system @config.yml@.
<pre>
-$ RAILS_ENV=production bundle exec rake config:diff
+$ RAILS_ENV=production bin/rake config:diff
</pre>
This command will also report if no migrations are required.
h2. Keepproxy Permissions
-Permitting @keeproxy@ makes it possible to use @arv-put@ and @arv-get@, and upload from Workbench 1. It works in terms of individual 64 MiB keep blocks. It prints a log each time a user uploads or downloads an individual block.
+Permitting @keeproxy@ makes it possible to use @arv-put@ and @arv-get@, and upload from Workbench 1. It works in terms of individual 64 MiB keep blocks. It prints a log line each time a user uploads or downloads an individual block. Those logs are usually stored by @journald@ or @syslog@.
The default policy allows anyone to upload or download.
Upload: true
</pre>
-If you create a sharing link as an admin user, and then give someone the token from the sharing link to download a file using @arv-get@, because the downloader is anonymous, the download permission will be restricted based on the "User" role and not the "Admin" role.
-
h2. WebDAV and S3 API Permissions
Permitting @WebDAV@ makes it possible to use WebDAV, S3 API, download from Workbench 1, and upload/download with Workbench 2. It works in terms of individual files. It prints a log each time a user uploads or downloads a file. When @WebDAVLogEvents@ (default true) is enabled, it also adds an entry into the API server @logs@ table.
<pre>
Collections:
- WebDAVPermisison:
+ WebDAVPermission:
User:
Download: true
Upload: true
Download: true
Upload: true
WebDAVLogEvents: true
-</pre>
+ </pre>
-If you create a sharing link as an admin user, and then give someone the token from the sharing link to download a file over HTTP (WebDAV or S3 API), because the downloader is anonymous, the download permission will be restricted based on the "User" role and not the "Admin" role.
+When a user or admin creates a sharing link, a custom scoped token is embedded in that link. This effectively allows anonymous user access to the associated data via that link. These custom scoped tokens are always treated as user tokens for the purposes of restricting download access, even when created by an admin user. In other words, these custom scoped tokens, when used in a sharing link, are always subject to the value of the @WebDAVPermission/User/Download@ configuration setting.
+
+If that custom scoped token is used with @arv-get@, its use will be subject to the value of the @KeepproxyPermission/User/Download@ configuration setting.
h2. Shell node and container permissions
h2. Choosing a policy
-This distinction between WebDAV and Keepproxy is important for auditing. WebDAV records 'upload' and 'download' events on the API server that are included in the "User Activity Report":user-activity.html , whereas @keepproxy@ only logs upload and download of individual blocks, which require a reverse lookup to determine the collection(s) and file(s) a block is associated with.
+This distinction between WebDAV and Keepproxy is important for auditing. WebDAV records 'upload' and 'download' events on the API server that are included in the "User Activity Report":user-activity.html, whereas @keepproxy@ only logs upload and download of individual blocks, which require a reverse lookup to determine the collection(s) and file(s) a block is associated with.
You set separate permissions for @WebDAV@ and @Keepproxy@, with separate policies for regular users and admin users.
<pre>
Collections:
- WebDAVPermisison:
+ WebDAVPermission:
User:
Download: true
Upload: true
<pre>
Collections:
- WebDAVPermisison:
+ WebDAVPermission:
User:
Download: false
Upload: true
<pre>
Collections:
- WebDAVPermisison:
+ WebDAVPermission:
User:
Download: true
Upload: false
Upload: true
WebDAVLogEvents: true
</pre>
+
+
+h2. Accessing the audit log
+
+When @WebDAVLogEvents@ is enabled, uploads and downloads of files are logged in the Arvados audit log. These events are included in the "User Activity Report":user-activity.html. The audit log can also be accessed via the API, SDKs or command line. For example, to show the 100 most recent file downloads:
+
+<pre>
+arv log list --filters '[["event_type","=","file_download"]]' -o 'created_at desc' -l 100
+</pre>
+
+For uploads, use the @file_upload@ event type.
+
+Note that this only covers upload and download activity via WebDAV, S3, Workbench 1 (download only) and Workbench 2.
+
+File upload in Workbench 1 and the @arv-get@ and @arv-put@ tools use @Keepproxy@, which does not log activity to the audit log because it operates at the block level, not the file level. @Keepproxy@ records the uuid of the user that owns the token used in the request in its system logs. Those logs are usually stored by @journald@ or @syslog@. A typical log line for such a block download looks like this:
+
+<pre>
+Jul 20 15:03:38 workbench.xxxx1.arvadosapi.com keepproxy[63828]: {"level":"info","locator":"abcdefghijklmnopqrstuvwxyz012345+53251584","msg":"Block download","time":"2021-07-20T15:03:38.458792300Z","user_full_name":"Albert User","user_uuid":"ce8i5-tpzed-abcdefghijklmno"}
+</pre>
+
+It is possible to do a reverse lookup from the locator to find all matching collections: the @manifest_text@ field of a collection lists all the block locators that are part of the collection. The @manifest_text@ field also provides the relevant filename in the collection. Because this lookup is rather involved and there is no automated tool to do it, we recommend disabling @KeepproxyPermission/User/Download@ and @KeepproxyPermission/User/Upload@ for sites where the audit log is important and @arv-get@ and @arv-put@ are not essential.
The @db:check_long_lived_tokens@ task will list which users have tokens with no expiration date.
<notextile>
-<pre><code># <span class="userinput">bundle exec rake db:check_long_lived_tokens</span>
+<pre><code># <span class="userinput">bin/rake db:check_long_lived_tokens</span>
Found 6 long-lived tokens from users:
user2,user2@example.com,zzzzz-tpzed-5vzt5wc62k46p6r
admin,admin@example.com,zzzzz-tpzed-6drplgwq9nm5cox
To apply the new policy to existing tokens, use the @db:fix_long_lived_tokens@ task.
<notextile>
-<pre><code># <span class="userinput">bundle exec rake db:fix_long_lived_tokens</span>
+<pre><code># <span class="userinput">bin/rake db:fix_long_lived_tokens</span>
Setting token expiration to: 2020-08-25 03:30:50 +0000
6 tokens updated.
</code></pre>
<div class="releasenotes">
</notextile>
-h2(#main). development main (as of 2021-06-03)
+h2(#main). development main (as of 2021-07-15)
"Upgrading from 2.2.0":#v2_2_0
+h3. crunch-dispatch-local now requires config.yml
+
+The @crunch-dispatch-local@ dispatcher now reads the API host and token from the system wide @/etc/arvados/config.yml@ . It will fail to start that file is not found or not readable.
+
h2(#v2_2_0). v2.2.0 (2021-06-03)
"Upgrading from 2.1.0":#v2_1_0
--- /dev/null
+---
+layout: default
+navsection: architecture
+title: Dispatching containers to cloud VMs
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+The arvados-dispatch-cloud component runs Arvados user containers on generic public cloud infrastructure by automatically creating and destroying VMs (“instances”) of various sizes according to demand, preparing the instances’ runtime environments, and running containers on them.
+
+This does not use a cloud provider’s container-execution service.
+
+h2. Overview
+
+In this diagram, the black edges show interactions involved in starting a VM instance and running a container. The blue edges show the “container shell” communication channel.
+
+!{max-width:40em}{{site.baseurl}}/architecture/dispatchcloud.svg!
+
+{% comment %}
+# svg generated using https://graphviz.it/
+digraph {
+ subgraph cluster_cloudvm {
+ node [color=black] [fillcolor=white] [style=filled];
+ style = filled;
+ color = lightgrey;
+ label = "cloud instance (VM)";
+ "SSH server" -> "crunch-run" [label = "start crunch-run"];
+ "crunch-run" -> docker [label = "create container"];
+ "crunch-run" -> docker [label = "shell"] [color = blue] [fontcolor = blue];
+ "crunch-run" -> container [label = "tcp/http"] [color = blue] [fontcolor = blue];
+ docker -> container;
+ }
+ "cloud provider" [shape=box] [style=dashed];
+ dispatcher -> controller [label = "get container queue"];
+ dispatcher -> "cloud provider" [label = "create/destroy/list VMs"];
+ "cloud provider" -> "SSH server" [label = "add authorized_keys"];
+ "crunch-run" -> controller [label = "update\ngateway ip:port,\ncontainer state,\noutput, ..."];
+ client -> controller [label = "shell/tcp/http (https tunnel)"] [color = blue] [fontcolor = blue];
+ controller -> "crunch-run" [label = "shell/tcp/http (https tunnel)"] [color = blue] [fontcolor = blue];
+ dispatcher -> "SSH server" [label = "start crunch-run"];
+}
+{% endcomment %}
+
+h2. Scheduling
+
+The dispatcher periodically polls arvados-controller to get a list of containers that are ready to run. Whenever this list changes, the dispatcher runs a scheduling loop that selects a suitable instance type for each container, allocates the highest priority containers to idle instances, requests new instances if needed, and shuts down instances that have been idle for longer than the configured idle timeout. Currently the dispatcher only runs one container at a time on an instance, even if the instance has enough RAM and CPUs to accommodate more.
+
+h2. Creating instances
+
+When creating a new instance, the dispatcher uses the cloud provider’s metadata feature to add a tag with key “InstanceSetID” and a value derived from its Arvados authentication token. This enables the dispatcher to recognize and reconnect to existing instances that belong to it, and continue monitoring existing containers, after a restart or upgrade.
+
+When using the Azure cloud service, the dispatcher needs to first create a new network interface, then attach it to a new instance. The network interface is also tagged with “InstanceSetID”.
+
+If the cloud provider returns a rate-limiting error when creating a new instance, the dispatcher avoids requesting new instances for a short period, and shuts down idle nodes more aggressively (i.e., without waiting for the usual idle timeout to elapse) until a new instance is successfully created.
+
+h2. Recovering state after a restart
+
+Restarting the dispatcher does not interrupt containers that are already running. When the dispatcher starts up, it gets the cloud provider’s current list of instances that have the expected InstanceSetID tag value. It ignores instances without that tag, so it won’t interfere with other VM instances in the same cloud account. It runs the boot probe command on each instance, checks for containers that were started by a previous invocation and are still running, and resumes monitoring. Before dispatching any new containers to a previously existing instance, it ensures the crunch-run program is updated if needed.
+
+h2. Instance boot process
+
+When the cloud provider indicates that a new instance has been created, the dispatcher connects to the instance’s SSH service (see “instance control channel” below) and executes the configured boot probe command. If this fails, the dispatcher retries until the configured boot timeout is reached, then shuts down the instance. When the boot probe succeeds, the dispatcher copies the crunch-run program to the instance, and runs it to check for running containers before reporting the instance’s state as “idle” or “busy”. (Normally of course a freshly booted instance has no containers running, but this covers the case where the dispatcher itself has restarted and containers submitted by the previous dispatcher process are still running.)
+
+The dispatcher and crunch-run programs are both packaged in a single executable file: when dispatcher copies crunch-run to an instance, it is really copying itself. This ensures the dispatcher is always using the version of crunch-run that it expects.
+
+h2. Boot probe command
+
+The purpose of the boot probe command is to ensure the dispatcher does not try to schedule containers on an instance before the instance is ready, even if its SSH daemon comes up early in the boot process. The default boot probe command, @docker ps -q@, verifies that the docker daemon is running. It is also common to use a custom startup script in the VM image that writes a file when it finishes, and a boot probe command that checks for that file, such as @cat /var/run/boot.complete@.
+
+h2. Automatic instance shutdown
+
+Normally, the dispatcher shuts down any instance that has remained idle for 1 minute (see TimeoutIdle configuration) but there are some exceptions to this rule. If the cloud provider returns a quota error when trying to create a new instance, the dispatcher shuts down idle nodes right away, in case the idle nodes are contributing to the quota. Also, the operator can use the management API to set an instance’s idle behavior to “drain” or “hold”. “Drain” shuts down the instance as soon as it becomes idle, which can be used to recycle a suspect node without interrupting a running container. “Hold” keeps the instance alive indefinitely without scheduling additional containers on it, which can be used to investigate problems like a failed startup script.
+
+Each instance is tagged with its current idle behavior (using the tag name “IdleBehavior”), which makes it visible in the cloud provider’s console and ensures the behavior is retained if dispatcher restarts.
+
+h2. Management API
+
+The dispatcher provides an HTTP management interface, which provides the operator with more visibility and control for purposes of troubleshooting and monitoring. APIs are provided to return details of current VM instances and running/scheduled containers as seen by the dispatcher, immediately terminate containers and instances, and control the on-idle behavior of instances. This interface also provides Prometheus metrics. See the "cloud dispatcher management API":{{site.baseurl}}/api/dispatch.html documentation for details.
+
+h2. Instance control channel (SSH)
+
+The dispatcher uses a multiplexed SSH connection to monitor instance boot progress, install the crunch-run supervisor program, start and stop containers, and detect crashed containers and failing instances. It establishes a persistent SSH connection to each cloud instance when the instance first appears, retrying/reconnecting as needed.
+
+Cloud VMs typically generate a random SSH host key at boot time, making host key verification impossible. To provide some assurance the dispatcher is connecting to the intended instance, when it creates a new instance the dispatcher generates a random “instance secret”, uses the cloud provider’s bootstrap command feature to save it in @/var/run/arvados-instance-secret@ on the new instance, and executes @cat /var/run/arvados-instance-secret@ to verify the instance’ identity when first connecting to its SSH server. Each instance is also tagged with its instance secret, so it can still be verified after a dispatcher restart.
+
+h2. Container communication channel (https tunnel)
+
+The crunch-run program runs a gateway server which facilitates the “container shell” feature without sending traffic through the dispatcher process. The gateway server accepts TLS connections from arvados-controller on a dynamic TCP port (typically in the range 32768-60999, see @sysctl net.ipv4.ip_local_port_range@). Crunch-run saves the selected port, along with the external IP address of the VM instance as seen by the dispatcher, in the @gateway_address@ field in the container record so arvados-controller can connect to it.
+
+On the client host (typically a shell node or a user’s workstation) the @arvados-client shell@ command sends an https “connect” request to arvados-controller, which sends an https “connect” request to the gateway server. These tunnels convey SSH protocol traffic between the user’s SSH client and crunch-run’s built-in SSH server, which uses @docker exec@ to run commands inside the container.
+
+Arvados-controller and crunch-run gateway server authenticate each other using a self-signed certificate and a shared secret based on the cluster-wide @SystemRootToken@. If that token changes (and the dispatcher restarts to load the new token) while a container is running, the container will stop accepting container shell traffic.
+
+h2. Scaling
+
+Architecturally, the dispatcher is _designed_ to accommodate multiple concurrent dispatcher processes on multiple hosts, each using a different authorization token, but such a configuration is not yet supported. Currently, each cluster should run a single dispatcher process. A single process can support thousands of concurrent VM instances.
--- /dev/null
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="542.87pt" height="561.4pt" viewBox="0 0 542.87 561.4"><style type="text/css">.dashed {stroke-dasharray: 5,5} .dotted {stroke-dasharray: 1,5} .overlay {fill: none; pointer-events: all}</style><g><g transform="translate(4, 557.4000244140625) scale(1,1)"><polygon stroke="#fffffe" stroke-opacity="0" fill="#ffffff" points="-4,4 -4,-557.4 538.87,-557.4 538.87,4"></polygon><g class="subgraph"><title>cluster_cloudvm</title><path stroke="#d3d3d3" fill="#d3d3d3" d="M 30.22,-8 L 30.22,-385.8,209.22,-385.8,209.22,-8 Z"></path><text x="119.72" y="-369.2" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">cloud instance (VM)</text></g><g class="node"><title>SSH server</title><path stroke="#000000" fill="#ffffff" d="M 93.22,-335 m -54.99,0 a 54.99,18 0 1,0 109.98,0 a 54.99,18 0 1,0 -109.98,0"></path><text x="93.22" y="-330.8" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">SSH server</text></g><g class="node"><title>crunch-run</title><path stroke="#000000" fill="#ffffff" d="M 148.22,-195.8 m -53.29,0 a 53.29,18 0 1,0 106.58,0 a 53.29,18 0 1,0 -106.58,0"></path><text x="148.22" y="-191.6" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">crunch-run</text></g><g class="relation" style="opacity: 1;"><title>SSH server->crunch-run</title><path stroke="#000000" fill="none" d="M 82.52,-317.19 C 79.55,-311.62,76.72,-305.24,75.2,-299,68.17,-269.97,59.29,-257.07,75.2,-231.8,80.6,-223.24,88.73,-216.72,97.64,-211.78"></path><path class="solid" stroke="#000000" fill="#000000" d="M 99.28,-214.87 L 106.73,-207.34,96.21,-208.58 Z"></path><text x="119.72" y="-261.2" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">start crunch-run</text></g><g class="node"><title>docker</title><path stroke="#000000" fill="#ffffff" d="M 85.22,-107 m -37.12,0 a 37.12,18 0 1,0 74.24,0 a 37.12,18 0 1,0 -74.24,0"></path><text x="85.22" y="-102.8" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">docker</text></g><g class="relation" style="opacity: 1;"><title>crunch-run->docker</title><path stroke="#000000" fill="none" d="M 100.13,-187.64 C 83.75,-182.64,67.09,-174.17,57.22,-159.8,51.11,-150.9,54.81,-140.52,61.26,-131.4"></path><path class="solid" stroke="#000000" fill="#000000" d="M 64.14,-133.41 L 67.76,-123.46,58.72,-128.98 Z"></path><path stroke="#0000ff" fill="none" d="M 151.54,-177.75 C 152.71,-167.03,152.46,-153.32,146.22,-143,141.05,-134.47,132.98,-127.82,124.38,-122.72"></path><path class="solid" stroke="#0000ff" fill="#0000ff" d="M 125.8,-119.52 L 115.32,-117.96,122.54,-125.71 Z"></path><text x="102.71" y="-147.2" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">create container</text><text x="165.44" y="-147.2" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#0000ff">shell</text></g><g class="node"><title>container</title><path stroke="#000000" fill="#ffffff" d="M 119.22,-34 m -46.93,0 a 46.93,18 0 1,0 93.86,0 a 46.93,18 0 1,0 -93.86,0"></path><text x="119.22" y="-29.8" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">container</text></g><g class="relation" style="opacity: 1;"><title>crunch-run->container</title><path stroke="#0000ff" fill="none" d="M 168.59,-179 C 174.07,-173.57,179.29,-167.02,182.22,-159.8,185.02,-152.88,184.16,-150.21,182.22,-143,173.79,-111.83,153.88,-80.42,138.69,-59.57"></path><path class="solid" stroke="#0000ff" fill="#0000ff" d="M 141.3,-57.22 L 132.51,-51.32,135.7,-61.42 Z"></path><text x="197.6" y="-102.8" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#0000ff">tcp/http</text></g><g class="relation" style="opacity: 1;"><title>docker->container</title><path stroke="#000000" fill="none" d="M 93.27,-89.17 C 97.24,-80.9,102.11,-70.72,106.56,-61.44"></path><path class="solid" stroke="#000000" fill="#000000" d="M 109.82,-62.73 L 110.98,-52.2,103.5,-59.71 Z"></path></g><g class="node"><title>controller</title><path stroke="#000000" fill="none" d="M 292.22,-335 m -48.65,0 a 48.65,18 0 1,0 97.3,0 a 48.65,18 0 1,0 -97.3,0"></path><text x="292.22" y="-330.8" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">controller</text></g><g class="relation" style="opacity: 1;"><title>crunch-run->controller</title><path stroke="#000000" fill="none" d="M 156.37,-213.74 C 159,-219.43,161.84,-225.84,164.22,-231.8,175.91,-261.13,164.83,-276.77,187.24,-299,200.71,-312.35,219.4,-320.61,237.28,-325.72"></path><path class="solid" stroke="#000000" fill="#000000" d="M 236.54,-329.14 L 247.09,-328.25,238.28,-322.37 Z"></path><text x="233.7" y="-286.4" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">update</text><text x="233.7" y="-269.6" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">gateway ip:port,</text><text x="233.7" y="-252.8" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">container state,</text><text x="233.7" y="-236" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">output, ...</text></g><g class="node"><title>cloud provider</title><path class="dashed" stroke="#000000" fill="none" d="M 202.25,-464.6 L 104.18,-464.6,104.18,-428.6,202.25,-428.6 Z"></path><text x="153.22" y="-442.4" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">cloud provider</text></g><g class="relation" style="opacity: 1;"><title>cloud provider->SSH server</title><path stroke="#000000" fill="none" d="M 143.84,-428.47 C 134.07,-410.62,118.63,-382.42,107.37,-361.85"></path><path class="solid" stroke="#000000" fill="#000000" d="M 110.4,-360.1 L 102.52,-353,104.26,-363.46 Z"></path><text x="191.95" y="-398" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">add authorized_keys</text></g><g class="node"><title>dispatcher</title><path stroke="#000000" fill="none" d="M 121.22,-535.4 m -50.94,0 a 50.94,18 0 1,0 101.88,0 a 50.94,18 0 1,0 -101.88,0"></path><text x="121.22" y="-531.2" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">dispatcher</text></g><g class="relation" style="opacity: 1;"><title>dispatcher->SSH server</title><path stroke="#000000" fill="none" d="M 84.19,-522.95 C 57.27,-512.46,22.76,-494.01,6.2,-464.6,-15.98,-425.19,28.29,-382.2,61.44,-357.25"></path><path class="solid" stroke="#000000" fill="#000000" d="M 63.56,-360.03 L 69.58,-351.31,59.43,-354.38 Z"></path><text x="50.72" y="-442.4" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">start crunch-run</text></g><g class="relation" style="opacity: 1;"><title>dispatcher->cloud provider</title><path stroke="#000000" fill="none" d="M 118.61,-517.02 C 117.76,-506.68,117.95,-493.49,122.2,-482.6,123.51,-479.25,125.3,-476,127.36,-472.94"></path><path class="solid" stroke="#000000" fill="#000000" d="M 130.31,-474.84 L 133.71,-464.8,124.8,-470.53 Z"></path><text x="187.72" y="-486.8" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">create/destroy/list VMs</text></g><g class="relation" style="opacity: 1;"><title>dispatcher->controller</title><path stroke="#000000" fill="none" d="M 171.65,-533.15 C 199.28,-529.81,232.11,-521.04,253.22,-499.4,262.74,-489.64,279.17,-406.88,287.33,-363.03"></path><path class="solid" stroke="#000000" fill="#000000" d="M 290.8,-363.51 L 289.17,-353.04,283.91,-362.24 Z"></path><text x="330.03" y="-442.4" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">get container queue</text></g><g class="relation" style="opacity: 1;"><title>controller->crunch-run</title><path stroke="#0000ff" fill="none" d="M 295.12,-316.84 C 297.86,-294.44,298.93,-255.32,278.22,-231.8,261.45,-212.76,235.53,-203.57,211.23,-199.31"></path><path class="solid" stroke="#0000ff" fill="#0000ff" d="M 211.72,-195.84 L 201.31,-197.81,210.68,-202.76 Z"></path><text x="372.04" y="-261.2" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#0000ff">shell/tcp/http (https tunnel)</text></g><g class="node"><title>client</title><path stroke="#000000" fill="none" d="M 427.22,-446.6 m -32.48,0 a 32.48,18 0 1,0 64.96,0 a 32.48,18 0 1,0 -64.96,0"></path><text x="427.22" y="-442.4" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#000000">client</text></g><g class="relation" style="opacity: 1;"><title>client->controller</title><path stroke="#0000ff" fill="none" d="M 409.57,-431.27 C 386.64,-412.66,346.37,-379.96,319.49,-358.15"></path><path class="solid" stroke="#0000ff" fill="#0000ff" d="M 321.59,-355.34 L 311.62,-351.75,317.17,-360.77 Z"></path><text x="459.04" y="-398" text-anchor="middle" font-family="'Times-Roman',serif" font-size="14" fill="#0000ff">shell/tcp/http (https tunnel)</text></g></g></g></svg>
\ No newline at end of file
h2. Quick start
<pre>
-$ git clone https://github.com/arvados/arvados.git
-$ cd arvados/tools/arvbox/bin
-$ ./arvbox start localdemo
+$ curl -O https://git.arvados.org/arvados.git/blob_plain/refs/heads/main:/tools/arvbox/bin/arvbox
+$ chmod +x arvbox
+$ ./arvbox start localdemo latest
$ ./arvbox adduser demouser demo@example.com
</pre>
The cloud dispatch service can run on any node that can connect to the Arvados API service, the cloud provider's API, and the SSH service on cloud VMs. It is not resource-intensive, so you can run it on the API server node.
+More detail about the internal operation of the dispatcher can be found in the "architecture section":{{site.baseurl}}/architecture/dispatchcloud.html.
+
h2(#update-config). Update config.yml
h3. Configure CloudVMs
h3. Test configuration
-notextile. <pre><code>$ <span class="userinput">sudo -u git -i bash -c 'cd /var/www/arvados-api/current && bundle exec script/arvados-git-sync.rb production'</span></code></pre>
+notextile. <pre><code>$ <span class="userinput">sudo -u git -i bash -c 'cd /var/www/arvados-api/current && bin/bundle exec script/arvados-git-sync.rb production'</span></code></pre>
h3. Enable the synchronization script
Create @/etc/cron.d/arvados-git-sync@ with the following content:
<notextile>
-<pre><code><span class="userinput">*/5 * * * * git cd /var/www/arvados-api/current && bundle exec script/arvados-git-sync.rb production</span>
+<pre><code><span class="userinput">*/5 * * * * git cd /var/www/arvados-api/current && bin/bundle exec script/arvados-git-sync.rb production</span>
</code></pre>
</notextile>
h2(#update-config). Configure anonymous user token
-{% assign railscmd = "bundle exec ./script/get_anonymous_user_token.rb --get" %}
+{% assign railscmd = "bin/bundle exec ./script/get_anonymous_user_token.rb --get" %}
{% assign railsout = "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" %}
If you intend to use Keep-web to serve public data to anonymous clients, configure it with an anonymous token.
<notextile>
<pre><code>~$ <span class="userinput">arvados-client costanalyzer -h</span>
Usage:
- arvados-client costanalyzer [options ...] uuid [uuid ...]
+ ./arvados-client costanalyzer [options ...] [UUID ...]
- This program analyzes the cost of Arvados container requests. For each uuid
- supplied, it creates a CSV report that lists all the containers used to
- fulfill the container request, together with the machine type and cost of
- each container. At least one uuid must be specified.
+ This program analyzes the cost of Arvados container requests and calculates
+ the total cost across all requests. At least one UUID or a timestamp range
+ must be specified.
- When supplied with the uuid of a container request, it will calculate the
+ When the '-output' option is specified, a set of CSV files with cost details
+ will be written to the provided directory. Each file is a CSV report that lists
+ all the containers used to fulfill the container request, together with the
+ machine type and cost of each container.
+
+ When supplied with the UUID of a container request, it will calculate the
cost of that container request and all its children.
- When supplied with the uuid of a collection, it will see if there is a
- container_request uuid in the properties of the collection, and if so, it
+ When supplied with the UUID of a collection, it will see if there is a
+ container_request UUID in the properties of the collection, and if so, it
will calculate the cost of that container request and all its children.
- When supplied with a project uuid or when supplied with multiple container
- request or collection uuids, it will create a CSV report for each supplied
- uuid, as well as a CSV file with aggregate cost accounting for all supplied
- uuids. The aggregate cost report takes container reuse into account: if a
- container was reused between several container requests, its cost will only
- be counted once.
+ When supplied with a project UUID or when supplied with multiple container
+ request or collection UUIDs, it will calculate the total cost for all
+ supplied UUIDs.
+
+ When supplied with a 'begin' and 'end' timestamp (format:
+ 2006-01-02T15:04:05), it will calculate the cost for all top-level container
+ requests whose containers finished during the specified interval.
+
+ The total cost calculation takes container reuse into account: if a container
+ was reused between several container requests, its cost will only be counted
+ once.
Caveats:
permanent cloud nodes that provide the Arvados services, the cost of data
stored in Arvados, etc.
- - When provided with a project uuid, subprojects will not be considered.
+ - When provided with a project UUID, subprojects will not be considered.
- In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
+ In order to get the data for the UUIDs supplied, the ARVADOS_API_HOST and
ARVADOS_API_TOKEN environment variables must be set.
This program prints the total dollar amount from the aggregate cost
- accounting across all provided uuids on stdout.
-
- When the '-output' option is specified, a set of CSV files with cost details
- will be written to the provided directory.
+ accounting across all provided UUIDs on stdout.
Options:
+ -begin begin
+ timestamp begin for date range operation (format: 2006-01-02T15:04:05)
-cache
create and use a local disk cache of Arvados objects (default true)
+ -end end
+ timestamp end for date range operation (format: 2006-01-02T15:04:05)
-log-level level
logging level (debug, info, ...) (default "info")
-output directory
table(table table-bordered table-condensed).
|_. Field |_. Type |_. Description |
|outputTTL|int|If the value is greater than zero, consider intermediate output collections to be temporary and should be automatically trashed. Temporary collections will be trashed @outputTTL@ seconds after creation. A value of zero means intermediate output should be retained indefinitely (this is the default behavior).
-Note: arvados-cwl-runner currently does not take workflow dependencies into account when setting the TTL on an intermediate output collection. If the TTL is too short, it is possible for a collection to be trashed before downstream steps that consume it are started. The recommended minimum value for TTL is the expected duration of the entire the workflow.|
+Note: arvados-cwl-runner currently does not take workflow dependencies into account when setting the TTL on an intermediate output collection. If the TTL is too short, it is possible for a collection to be trashed before downstream steps that consume it are started. The recommended minimum value for TTL is the expected duration of the entire workflow.|
h2. cwltool:Secrets
The output of arv-copy displays the uuid of the collection generated in the destination cluster. By default, the output is placed in your home project in the destination cluster. If you want to place your collection in an existing project, you can specify the project you want it to be in using the tag @--project-uuid@ followed by the project uuid.
-For example, this will copy the collection to project dstcl-j7d0g-a894213ukjhal12 in the destination cluster.
+For example, this will copy the collection to project @dstcl-j7d0g-a894213ukjhal12@ in the destination cluster.
<notextile> <pre><code>~$ <span class="userinput">arv-copy --src pirca --dst dstcl --project-uuid dstcl-j7d0g-a894213ukjhal12 jutro-4zz18-tv416l321i4r01e
</code></pre>
</notextile>
+Additionally, if you need to specify the storage classes where to save the copied data on the destination cluster, you can do that by using the @--storage-classes LIST@ argument, where @LIST@ is a comma-separated list of storage class names.
+
h3. How to copy a workflow
We will use the uuid @jutro-7fd4e-mkmmq53m1ze6apx@ as an example workflow.
github.com/johannesboyne/gofakes3 v0.0.0-20200716060623-6b2b4cb092cc
github.com/julienschmidt/httprouter v1.2.0
github.com/kevinburke/ssh_config v0.0.0-20171013211458-802051befeb5 // indirect
- github.com/lib/pq v1.3.0
+ github.com/lib/pq v1.10.2
github.com/msteinert/pam v0.0.0-20190215180659-f29b9f28d6f9
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
github.com/opencontainers/image-spec v1.0.1-0.20171125024018-577479e4dc27 // indirect
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
golang.org/x/sys v0.0.0-20210603125802-9665404d3644
- golang.org/x/tools v0.1.0 // indirect
- golang.org/x/sys v0.0.0-20210510120138-977fb7262007
golang.org/x/tools v0.1.2 // indirect
google.golang.org/api v0.13.0
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
+github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210603125802-9665404d3644 h1:CA1DEQ4NdKphKeL70tvsWNdT5oFh1lOjihRcEDROi0I=
-golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210603125802-9665404d3644 h1:CA1DEQ4NdKphKeL70tvsWNdT5oFh1lOjihRcEDROi0I=
+golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
if err != nil {
return err
}
- for _, version := range []string{"1.16.6", "1.17.3", "2.0.2"} {
+ for _, version := range []string{"2.2.19"} {
if !strings.Contains(buf.String(), "("+version+")") {
- err = super.RunProgram(ctx, runner.src, runOptions{}, "gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
+ err = super.RunProgram(ctx, runner.src, runOptions{}, "gem", "install", "--user", "--conservative", "--no-document", "bundler:2.2.19")
if err != nil {
return err
}
AccessViaHosts: map[arvados.URL]arvados.VolumeAccess{
url: {},
},
+ StorageClasses: map[string]bool{
+ "default": true,
+ "foo": true,
+ "bar": true,
+ },
}
}
}
"Volumes.*.ReadOnly": true,
"Volumes.*.Replication": true,
"Volumes.*.StorageClasses": true,
- "Volumes.*.StorageClasses.*": false,
+ "Volumes.*.StorageClasses.*": true,
"Workbench": true,
"Workbench.ActivationContactLink": false,
"Workbench.APIClientConnectTimeout": true,
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/httpserver"
check "gopkg.in/check.v1"
)
}
}
+func (s *IntegrationSuite) TestRequestIDHeader(c *check.C) {
+ conn1 := s.testClusters["z1111"].Conn()
+ rootctx1, _, _ := s.testClusters["z1111"].RootClients()
+ userctx1, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
+
+ coll, err := conn1.CollectionCreate(userctx1, arvados.CreateOptions{})
+ c.Check(err, check.IsNil)
+ specimen, err := conn1.SpecimenCreate(userctx1, arvados.CreateOptions{})
+ c.Check(err, check.IsNil)
+
+ tests := []struct {
+ path string
+ reqIdProvided bool
+ notFoundRequest bool
+ }{
+ {"/arvados/v1/collections", false, false},
+ {"/arvados/v1/collections", true, false},
+ {"/arvados/v1/nonexistant", false, true},
+ {"/arvados/v1/nonexistant", true, true},
+ {"/arvados/v1/collections/" + coll.UUID, false, false},
+ {"/arvados/v1/collections/" + coll.UUID, true, false},
+ {"/arvados/v1/specimens/" + specimen.UUID, false, false},
+ {"/arvados/v1/specimens/" + specimen.UUID, true, false},
+ {"/arvados/v1/collections/z1111-4zz18-0123456789abcde", false, true},
+ {"/arvados/v1/collections/z1111-4zz18-0123456789abcde", true, true},
+ {"/arvados/v1/specimens/z1111-j58dm-0123456789abcde", false, true},
+ {"/arvados/v1/specimens/z1111-j58dm-0123456789abcde", true, true},
+ }
+
+ for _, tt := range tests {
+ c.Log(c.TestName() + " " + tt.path)
+ req, err := http.NewRequest("GET", "https://"+ac1.APIHost+tt.path, nil)
+ c.Assert(err, check.IsNil)
+ customReqId := "abcdeG"
+ if !tt.reqIdProvided {
+ c.Assert(req.Header.Get("X-Request-Id"), check.Equals, "")
+ } else {
+ req.Header.Set("X-Request-Id", customReqId)
+ }
+ resp, err := ac1.Do(req)
+ c.Assert(err, check.IsNil)
+ if tt.notFoundRequest {
+ c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
+ } else {
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ }
+ if !tt.reqIdProvided {
+ c.Check(resp.Header.Get("X-Request-Id"), check.Matches, "^req-[0-9a-zA-Z]{20}$")
+ if tt.notFoundRequest {
+ var jresp httpserver.ErrorResponse
+ err := json.NewDecoder(resp.Body).Decode(&jresp)
+ c.Check(err, check.IsNil)
+ c.Assert(jresp.Errors, check.HasLen, 1)
+ c.Check(jresp.Errors[0], check.Matches, "^.*(req-[0-9a-zA-Z]{20}).*$")
+ }
+ } else {
+ c.Check(resp.Header.Get("X-Request-Id"), check.Equals, customReqId)
+ if tt.notFoundRequest {
+ var jresp httpserver.ErrorResponse
+ err := json.NewDecoder(resp.Body).Decode(&jresp)
+ c.Check(err, check.IsNil)
+ c.Assert(jresp.Errors, check.HasLen, 1)
+ c.Check(jresp.Errors[0], check.Matches, "^.*("+customReqId+").*$")
+ }
+ }
+ }
+}
+
// We test the direct access to the database
// normally an integration test would not have a database access, but in this case we need
// to test tokens that are secret, so there is no API response that will give them back
} else if verified, _ := claims[ctrl.EmailVerifiedClaim].(bool); verified || ctrl.EmailVerifiedClaim == "" {
// Fall back to this info if the People API call
// (below) doesn't return a primary && verified email.
- name, _ := claims["name"].(string)
- if names := strings.Fields(strings.TrimSpace(name)); len(names) > 1 {
- ret.FirstName = strings.Join(names[0:len(names)-1], " ")
- ret.LastName = names[len(names)-1]
- } else if len(names) > 0 {
- ret.FirstName = names[0]
+ givenName, _ := claims["given_name"].(string)
+ familyName, _ := claims["family_name"].(string)
+ if givenName != "" && familyName != "" {
+ ret.FirstName = givenName
+ ret.LastName = familyName
+ } else {
+ name, _ := claims["name"].(string)
+ if names := strings.Fields(strings.TrimSpace(name)); len(names) > 1 {
+ ret.FirstName = strings.Join(names[0:len(names)-1], " ")
+ ret.LastName = names[len(names)-1]
+ } else if len(names) > 0 {
+ ret.FirstName = names[0]
+ }
}
ret.Email, _ = claims[ctrl.EmailClaim].(string)
}
s.fakeProvider.AuthEmail = "active-user@arvados.local"
s.fakeProvider.AuthEmailVerified = true
s.fakeProvider.AuthName = "Fake User Name"
+ s.fakeProvider.AuthGivenName = "Fake"
+ s.fakeProvider.AuthFamilyName = "User Name"
s.fakeProvider.ValidCode = fmt.Sprintf("abcdefgh-%d", time.Now().Unix())
s.fakeProvider.PeopleAPIResponse = map[string]interface{}{}
c.Check(token, check.Matches, `v2/zzzzz-gj3su-.{15}/.{32,50}`)
authinfo := getCallbackAuthInfo(c, s.railsSpy)
- c.Check(authinfo.FirstName, check.Equals, "Fake User")
- c.Check(authinfo.LastName, check.Equals, "Name")
+ c.Check(authinfo.FirstName, check.Equals, "Fake")
+ c.Check(authinfo.LastName, check.Equals, "User Name")
c.Check(authinfo.Email, check.Equals, "active-user@arvados.local")
c.Check(authinfo.AlternateEmails, check.HasLen, 0)
func (s *OIDCLoginSuite) TestGoogleLogin_RealName(c *check.C) {
s.fakeProvider.AuthEmail = "joe.smith@primary.example.com"
+ s.fakeProvider.AuthEmailVerified = true
s.fakeProvider.PeopleAPIResponse = map[string]interface{}{
"names": []map[string]interface{}{
{
c.Check(authinfo.LastName, check.Equals, "Psmith")
}
-func (s *OIDCLoginSuite) TestGoogleLogin_OIDCRealName(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_OIDCNameWithoutGivenAndFamilyNames(c *check.C) {
s.fakeProvider.AuthName = "Joe P. Smith"
+ s.fakeProvider.AuthGivenName = ""
+ s.fakeProvider.AuthFamilyName = ""
s.fakeProvider.AuthEmail = "joe.smith@primary.example.com"
state := s.startLogin(c)
s.localdb.Login(context.Background(), arvados.LoginOptions{
Preemptible bool
}
+type consumption struct {
+ cost float64
+ duration float64
+}
+
+func (c *consumption) Add(n consumption) {
+ c.cost += n.cost
+ c.duration += n.duration
+}
+
type arrayFlags []string
func (i *arrayFlags) String() string {
return
}
-func addContainerLine(logger *logrus.Logger, node nodeInfo, cr arvados.ContainerRequest, container arvados.Container) (csv string, cost float64) {
+func addContainerLine(logger *logrus.Logger, node nodeInfo, cr arvados.ContainerRequest, container arvados.Container) (string, consumption) {
+ var csv string
+ var containerConsumption consumption
csv = cr.UUID + ","
csv += cr.Name + ","
csv += container.UUID + ","
if container.FinishedAt != nil {
csv += container.FinishedAt.String() + ","
delta = container.FinishedAt.Sub(*container.StartedAt)
- csv += strconv.FormatFloat(delta.Seconds(), 'f', 0, 64) + ","
+ csv += strconv.FormatFloat(delta.Seconds(), 'f', 3, 64) + ","
} else {
csv += ",,"
}
price = node.Price
size = node.ProviderType
}
- cost = delta.Seconds() / 3600 * price
- csv += size + "," + fmt.Sprintf("%+v", node.Preemptible) + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"
- return
+ containerConsumption.cost = delta.Seconds() / 3600 * price
+ containerConsumption.duration = delta.Seconds()
+ csv += size + "," + fmt.Sprintf("%+v", node.Preemptible) + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(containerConsumption.cost, 'f', 8, 64) + "\n"
+ return csv, containerConsumption
}
func loadCachedObject(logger *logrus.Logger, file string, uuid string, object interface{}) (reload bool) {
return
}
-func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) {
- cost = make(map[string]float64)
+func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]consumption, err error) {
+ cost = make(map[string]consumption)
var project arvados.Group
err = loadObject(logger, ac, uuid, uuid, cache, &project)
items := value.([]interface{})
for _, item := range items {
itemMap := item.(map[string]interface{})
- crCsv, err := generateCrCsv(logger, itemMap["uuid"].(string), arv, ac, kc, resultsDir, cache)
+ crInfo, err := generateCrInfo(logger, itemMap["uuid"].(string), arv, ac, kc, resultsDir, cache)
if err != nil {
return nil, fmt.Errorf("error generating container_request CSV: %s", err.Error())
}
- for k, v := range crCsv {
+ for k, v := range crInfo {
cost[k] = v
}
}
return
}
-func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) {
+func generateCrInfo(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]consumption, err error) {
- cost = make(map[string]float64)
+ cost = make(map[string]consumption)
csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Preemptible,Hourly node cost,Total cost\n"
var tmpCsv string
- var tmpTotalCost float64
- var totalCost float64
+ var total, tmpTotal consumption
logger.Debugf("Processing %s", uuid)
var crUUID = uuid
logger.Errorf("Skipping container request %s: error getting node %s: %s", cr.UUID, cr.UUID, err)
return nil, nil
}
- tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container)
+ tmpCsv, total = addContainerLine(logger, topNode, cr, container)
csv += tmpCsv
- totalCost += tmpTotalCost
- cost[container.UUID] = totalCost
+ cost[container.UUID] = total
// Find all container requests that have the container we found above as requesting_container_uuid
var childCrs arvados.ContainerRequestList
if err != nil {
return nil, fmt.Errorf("error loading object %s: %s", cr2.ContainerUUID, err)
}
- tmpCsv, tmpTotalCost = addContainerLine(logger, node, cr2, c2)
- cost[cr2.ContainerUUID] = tmpTotalCost
+ tmpCsv, tmpTotal = addContainerLine(logger, node, cr2, c2)
+ cost[cr2.ContainerUUID] = tmpTotal
csv += tmpCsv
- totalCost += tmpTotalCost
+ total.Add(tmpTotal)
}
logger.Debug("Done collecting child containers")
- csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n"
+ csv += "TOTAL,,,,,," + strconv.FormatFloat(total.duration, 'f', 3, 64) + ",,,," + strconv.FormatFloat(total.cost, 'f', 2, 64) + "\n"
if resultsDir != "" {
// Write the resulting CSV file
}
}()
- cost := make(map[string]float64)
+ cost := make(map[string]consumption)
for uuid := range uuidChannel {
logger.Debugf("Considering %s", uuid)
}
} else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") {
// This is a container request
- var crCsv map[string]float64
- crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, c.resultsDir, c.cache)
+ var crInfo map[string]consumption
+ crInfo, err = generateCrInfo(logger, uuid, arv, ac, kc, c.resultsDir, c.cache)
if err != nil {
err = fmt.Errorf("error generating CSV for uuid %s: %s", uuid, err.Error())
exitcode = 2
return
}
- for k, v := range crCsv {
+ for k, v := range crInfo {
cost[k] = v
}
} else if strings.Contains(uuid, "-tpzed-") {
var csv string
- csv = "# Aggregate cost accounting for uuids:\n"
+ csv = "# Aggregate cost accounting for uuids:\n# UUID, Duration in seconds, Total cost\n"
for _, uuid := range c.uuids {
csv += "# " + uuid + "\n"
}
- var total float64
+ var total consumption
for k, v := range cost {
- csv += k + "," + strconv.FormatFloat(v, 'f', 8, 64) + "\n"
- total += v
+ csv += k + "," + strconv.FormatFloat(v.duration, 'f', 3, 64) + "," + strconv.FormatFloat(v.cost, 'f', 8, 64) + "\n"
+ total.Add(v)
}
- csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n"
+ csv += "TOTAL," + strconv.FormatFloat(total.duration, 'f', 3, 64) + "," + strconv.FormatFloat(total.cost, 'f', 2, 64) + "\n"
if c.resultsDir != "" {
// Write the resulting CSV file
}
// Output the total dollar amount on stdout
- fmt.Fprintf(stdout, "%s\n", strconv.FormatFloat(total, 'f', 8, 64))
+ fmt.Fprintf(stdout, "%s\n", strconv.FormatFloat(total.cost, 'f', 2, 64))
return
}
uuid2Report, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00916192")
- c.Check(string(uuid2Report), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00588088")
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,763.467,,,,0.01")
+ c.Check(string(uuid2Report), check.Matches, "(?ms).*TOTAL,,,,,,488.775,,,,0.01")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err := ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
- c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,0.01492030")
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,1245.564,0.01")
}
func (*Suite) TestContainerRequestUUID(c *check.C) {
c.Assert(err, check.IsNil)
// Make sure the 'preemptible' flag was picked up
c.Check(string(uuidReport), check.Matches, "(?ms).*,Standard_E4s_v3,true,.*")
- c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,86462.000,,,,7.01")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err := ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
- c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889")
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,86462.000,7.01")
}
func (*Suite) TestCollectionUUID(c *check.C) {
uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,86462.000,,,,7.01")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err := ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
- c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889")
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,86462.000,7.01")
}
func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,86462.000,,,,7.01")
uuidReport2, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,86462.000,,,,42.27")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err := ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
- c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000")
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,172924.000,49.28")
stdout.Truncate(0)
stderr.Truncate(0)
uuidReport, err = ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,86462.000,,,,7.01")
uuidReport2, err = ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,86462.000,,,,42.27")
re = regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
matches = re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err = ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
- c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000")
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,172924.000,49.28")
}
func (*Suite) TestUncommittedContainerRequest(c *check.C) {
c.Assert(stderr.String(), check.Matches, "(?ms).*No container associated with container request .*")
// Check that the total amount was printed to stdout
- c.Check(stdout.String(), check.Matches, "0.00588088\n")
+ c.Check(stdout.String(), check.Matches, "0.01\n")
}
func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
c.Assert(stderr.String(), check.Not(check.Matches), "(?ms).*supplied uuids in .*")
// Check that the total amount was printed to stdout
- c.Check(stdout.String(), check.Matches, "0.01492030\n")
+ c.Check(stdout.String(), check.Matches, "0.01\n")
stdout.Truncate(0)
stderr.Truncate(0)
uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00916192")
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,763.467,,,,0.01")
uuidReport2, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00588088")
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,488.775,,,,0.01")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err := ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
- c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,0.01492030")
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,1245.564,0.01")
}
// IKeepClient is the minimal Keep API methods used by crunch-run.
type IKeepClient interface {
- PutB(buf []byte) (string, int, error)
+ BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error)
ReadAt(locator string, p []byte, off int) (int, error)
ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error)
LocalLocator(locator string) (string, error)
// ContainerRunner is the main stateful struct used for a single execution of a
// container.
type ContainerRunner struct {
- executor containerExecutor
+ executor containerExecutor
+ executorStdin io.Closer
+ executorStdout io.Closer
+ executorStderr io.Closer
// Dispatcher client is initialized with the Dispatcher token.
// This is a privileged token used to manage container status
ExitCode *int
NewLogWriter NewLogWriter
CrunchLog *ThrottledLogger
- Stdout io.WriteCloser
- Stderr io.WriteCloser
logUUID string
logMtx sync.Mutex
LogCollection arvados.CollectionFileSystem
return "", fmt.Errorf("cannot choose from multiple tar files in image collection: %v", tarfiles)
}
imageID := tarfiles[0][:len(tarfiles[0])-4]
- imageFile := runner.ArvMountPoint + "/by_id/" + runner.Container.ContainerImage + "/" + tarfiles[0]
+ imageTarballPath := runner.ArvMountPoint + "/by_id/" + runner.Container.ContainerImage + "/" + imageID + ".tar"
runner.CrunchLog.Printf("Using Docker image id %q", imageID)
- if !runner.executor.ImageLoaded(imageID) {
- runner.CrunchLog.Print("Loading Docker image from keep")
- err = runner.executor.LoadImage(imageFile)
- if err != nil {
- return "", err
- }
- } else {
- runner.CrunchLog.Print("Docker image is available")
+ runner.CrunchLog.Print("Loading Docker image from keep")
+ err = runner.executor.LoadImage(imageID, imageTarballPath, runner.Container, runner.ArvMountPoint,
+ runner.containerClient)
+ if err != nil {
+ return "", err
}
+
return imageID, nil
}
} else {
arvMountCmd = append(arvMountCmd, "--mount-by-id", "by_id")
}
+ arvMountCmd = append(arvMountCmd, "--mount-by-id", "by_uuid")
arvMountCmd = append(arvMountCmd, runner.ArvMountPoint)
runner.ArvMount, err = runner.RunArvMount(arvMountCmd, token)
// CreateContainer creates the docker container.
func (runner *ContainerRunner) CreateContainer(imageID string, bindmounts map[string]bindmount) error {
- var stdin io.ReadCloser
+ var stdin io.ReadCloser = ioutil.NopCloser(bytes.NewReader(nil))
if mnt, ok := runner.Container.Mounts["stdin"]; ok {
switch mnt.Kind {
case "collection":
if !runner.enableMemoryLimit {
ram = 0
}
+ runner.executorStdin = stdin
+ runner.executorStdout = stdout
+ runner.executorStderr = stderr
return runner.executor.Create(containerSpec{
Image: imageID,
VCPUs: runner.Container.RuntimeConstraints.VCPUs,
}
runner.ExitCode = &exitcode
+ var returnErr error
+ if err = runner.executorStdin.Close(); err != nil {
+ err = fmt.Errorf("error closing container stdin: %s", err)
+ runner.CrunchLog.Printf("%s", err)
+ returnErr = err
+ }
+ if err = runner.executorStdout.Close(); err != nil {
+ err = fmt.Errorf("error closing container stdout: %s", err)
+ runner.CrunchLog.Printf("%s", err)
+ if returnErr == nil {
+ returnErr = err
+ }
+ }
+ if err = runner.executorStderr.Close(); err != nil {
+ err = fmt.Errorf("error closing container stderr: %s", err)
+ runner.CrunchLog.Printf("%s", err)
+ if returnErr == nil {
+ returnErr = err
+ }
+ }
+
if runner.statReporter != nil {
runner.statReporter.Stop()
err = runner.statLogger.Close()
runner.CrunchLog.Printf("error closing crunchstat logs: %v", err)
}
}
- return nil
+ return returnErr
}
func (runner *ContainerRunner) updateLogs() {
}
}
}
+ runner.ArvMount = nil
}
if runner.ArvMountPoint != "" {
if rmerr := os.Remove(runner.ArvMountPoint); rmerr != nil {
runner.CrunchLog.Printf("While cleaning up arv-mount directory %s: %v", runner.ArvMountPoint, rmerr)
}
+ runner.ArvMountPoint = ""
}
if rmerr := os.RemoveAll(runner.parentTemp); rmerr != nil {
}
checkErr("stopHoststat", runner.stopHoststat())
checkErr("CommitLogs", runner.CommitLogs())
+ runner.CleanupDirs()
checkErr("UpdateContainerFinal", runner.UpdateContainerFinal())
}()
exit chan int
}
-func (e *stubExecutor) ImageLoaded(imageID string) bool { return e.imageLoaded }
-func (e *stubExecutor) LoadImage(filename string) error { e.loaded = filename; return e.loadErr }
+func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
+ containerClient *arvados.Client) error {
+ e.loaded = tarball
+ return e.loadErr
+}
func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
func (e *stubExecutor) Start() error { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
func (e *stubExecutor) CgroupID() string { return "cgroupid" }
func (e *stubExecutor) Stop() error { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
func (e *stubExecutor) Close() { e.closed = true }
func (e *stubExecutor) Wait(context.Context) (int, error) {
- defer e.created.Stdout.Close()
- defer e.created.Stderr.Close()
return <-e.exit, e.waitErr
}
return locator, nil
}
-func (client *KeepTestClient) PutB(buf []byte) (string, int, error) {
- client.Content = buf
- return fmt.Sprintf("%x+%d", md5.Sum(buf), len(buf)), len(buf), nil
+func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
+ client.Content = opts.Data
+ return arvados.BlockWriteResponse{
+ Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
+ }, nil
}
func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
imageID, err = s.runner.LoadImage()
c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
c.Check(s.executor.loaded, Equals, "")
-
- // if executor reports image is already loaded, LoadImage should not be called
- s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
- s.executor.imageLoaded = true
- s.executor.loaded = ""
- s.executor.loadErr = nil
- imageID, err = s.runner.LoadImage()
- c.Check(err, IsNil)
- c.Check(s.executor.loaded, Equals, "")
- c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
}
type ArvErrorTestClient struct{}
return nil, errors.New("KeepError")
}
-func (*KeepErrorTestClient) PutB(buf []byte) (string, int, error) {
- return "", 0, errors.New("KeepError")
+func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
+ return arvados.BlockWriteResponse{}, errors.New("KeepError")
}
func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
func (s *TestSuite) TestRunContainer(c *C) {
s.executor.runFunc = func() {
fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
- s.executor.created.Stdout.Close()
- s.executor.created.Stderr.Close()
s.executor.exit <- 0
}
c.Check(err, IsNil)
c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
"--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
- "--mount-by-pdh", "by_id", realTemp + "/keep1"})
+ "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
c.Check(err, IsNil)
c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
"--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
- "--mount-by-pdh", "by_id", realTemp + "/keep1"})
+ "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
c.Check(err, IsNil)
c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
"--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
- "--mount-by-pdh", "by_id", realTemp + "/keep1"})
+ "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
c.Check(err, IsNil)
c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
"--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
- "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
+ "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
c.Check(err, IsNil)
c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
"--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
- "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
+ "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
c.Check(bindmounts, DeepEquals, map[string]bindmount{
"/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
"/keepout": {realTemp + "/keep1/tmp0", false},
c.Check(err, IsNil)
c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
"--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
- "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
+ "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
c.Check(bindmounts, DeepEquals, map[string]bindmount{
"/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
"/keepout": {realTemp + "/keep1/tmp0", false},
c.Check(err, IsNil)
c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
"--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
- "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
+ "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
c.Check(bindmounts, DeepEquals, map[string]bindmount{
"/tmp": {realTemp + "/tmp2", false},
"/tmp/foo": {realTemp + "/keep1/tmp0", true},
"strings"
"time"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
dockertypes "github.com/docker/docker/api/types"
dockercontainer "github.com/docker/docker/api/types/container"
dockerclient "github.com/docker/docker/client"
}, err
}
-func (e *dockerExecutor) ImageLoaded(imageID string) bool {
+func (e *dockerExecutor) LoadImage(imageID string, imageTarballPath string, container arvados.Container, arvMountPoint string,
+ containerClient *arvados.Client) error {
_, _, err := e.dockerclient.ImageInspectWithRaw(context.TODO(), imageID)
- return err == nil
-}
+ if err == nil {
+ // already loaded
+ return nil
+ }
-func (e *dockerExecutor) LoadImage(filename string) error {
- f, err := os.Open(filename)
+ f, err := os.Open(imageTarballPath)
if err != nil {
return err
}
}
}
-func (e *dockerExecutor) startIO(stdin io.ReadCloser, stdout, stderr io.WriteCloser) error {
+func (e *dockerExecutor) startIO(stdin io.Reader, stdout, stderr io.Writer) error {
resp, err := e.dockerclient.ContainerAttach(context.TODO(), e.containerID, dockertypes.ContainerAttachOptions{
Stream: true,
Stdin: stdin != nil,
return nil
}
-func (e *dockerExecutor) handleStdin(stdin io.ReadCloser, conn io.Writer, closeConn func() error) error {
- defer stdin.Close()
+func (e *dockerExecutor) handleStdin(stdin io.Reader, conn io.Writer, closeConn func() error) error {
defer closeConn()
_, err := io.Copy(conn, stdin)
if err != nil {
// Handle docker log protocol; see
// https://docs.docker.com/engine/reference/api/docker_remote_api_v1.15/#attach-to-a-container
-func (e *dockerExecutor) handleStdoutStderr(stdout, stderr io.WriteCloser, reader io.Reader) error {
+func (e *dockerExecutor) handleStdoutStderr(stdout, stderr io.Writer, reader io.Reader) error {
header := make([]byte, 8)
var err error
for err == nil {
if err != nil {
return fmt.Errorf("error copying stdout/stderr from docker: %v", err)
}
- err = stdout.Close()
- if err != nil {
- return fmt.Errorf("error writing stdout: close: %v", err)
- }
- err = stderr.Close()
- if err != nil {
- return fmt.Errorf("error writing stderr: close: %v", err)
- }
return nil
}
import (
"io"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
"golang.org/x/net/context"
)
EnableNetwork bool
NetworkMode string // docker network mode, normally "default"
CgroupParent string
- Stdin io.ReadCloser
- Stdout io.WriteCloser
- Stderr io.WriteCloser
+ Stdin io.Reader
+ Stdout io.Writer
+ Stderr io.Writer
}
// containerExecutor is an interface to a container runtime
// (docker/singularity).
type containerExecutor interface {
- // ImageLoaded determines whether the given image is already
- // available to use without calling ImageLoad.
- ImageLoaded(imageID string) bool
-
// ImageLoad loads the image from the given tarball such that
// it can be used to create/start a container.
- LoadImage(filename string) error
+ LoadImage(imageID string, imageTarballPath string, container arvados.Container, keepMount string,
+ containerClient *arvados.Client) error
// Wait for the container process to finish, and return its
// exit code. If applicable, also remove the stopped container
"strings"
"time"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
"golang.org/x/net/context"
. "gopkg.in/check.v1"
)
Stdout: nopWriteCloser{&s.stdout},
Stderr: nopWriteCloser{&s.stderr},
}
- err := s.executor.LoadImage(busyboxDockerImage(c))
+ err := s.executor.LoadImage("", busyboxDockerImage(c), arvados.Container{}, "", nil)
c.Assert(err, IsNil)
}
}
}
+func (s *executorSuite) TestExecWorkingDir(c *C) {
+ s.spec.WorkingDir = "/tmp"
+ s.spec.Command = []string{"sh", "-c", "pwd"}
+ s.checkRun(c, 0)
+ c.Check(s.stdout.String(), Equals, "/tmp\n")
+}
+
func (s *executorSuite) TestExecStdoutStderr(c *C) {
s.spec.Command = []string{"sh", "-c", "echo foo; echo -n bar >&2; echo baz; echo waz >&2"}
s.checkRun(c, 0)
package crunchrun
import (
+ "fmt"
"io/ioutil"
"os"
"os/exec"
+ "sort"
"syscall"
+ "time"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
"golang.org/x/net/context"
)
}, nil
}
-func (e *singularityExecutor) ImageLoaded(string) bool {
- return false
+func (e *singularityExecutor) getOrCreateProject(ownerUuid string, name string, containerClient *arvados.Client) (*arvados.Group, error) {
+ var gp arvados.GroupList
+ err := containerClient.RequestAndDecode(&gp,
+ arvados.EndpointGroupList.Method,
+ arvados.EndpointGroupList.Path,
+ nil, arvados.ListOptions{Filters: []arvados.Filter{
+ arvados.Filter{"owner_uuid", "=", ownerUuid},
+ arvados.Filter{"name", "=", name},
+ arvados.Filter{"group_class", "=", "project"},
+ },
+ Limit: 1})
+ if err != nil {
+ return nil, err
+ }
+ if len(gp.Items) == 1 {
+ return &gp.Items[0], nil
+ }
+
+ var rgroup arvados.Group
+ err = containerClient.RequestAndDecode(&rgroup,
+ arvados.EndpointGroupCreate.Method,
+ arvados.EndpointGroupCreate.Path,
+ nil, map[string]interface{}{
+ "group": map[string]string{
+ "owner_uuid": ownerUuid,
+ "name": name,
+ "group_class": "project",
+ },
+ })
+ if err != nil {
+ return nil, err
+ }
+ return &rgroup, nil
+}
+
+func (e *singularityExecutor) checkImageCache(dockerImageID string, container arvados.Container, arvMountPoint string,
+ containerClient *arvados.Client) (collection *arvados.Collection, err error) {
+
+ // Cache the image to keep
+ cacheGroup, err := e.getOrCreateProject(container.RuntimeUserUUID, ".cache", containerClient)
+ if err != nil {
+ return nil, fmt.Errorf("error getting '.cache' project: %v", err)
+ }
+ imageGroup, err := e.getOrCreateProject(cacheGroup.UUID, "auto-generated singularity images", containerClient)
+ if err != nil {
+ return nil, fmt.Errorf("error getting 'auto-generated singularity images' project: %s", err)
+ }
+
+ collectionName := fmt.Sprintf("singularity image for %v", dockerImageID)
+ var cl arvados.CollectionList
+ err = containerClient.RequestAndDecode(&cl,
+ arvados.EndpointCollectionList.Method,
+ arvados.EndpointCollectionList.Path,
+ nil, arvados.ListOptions{Filters: []arvados.Filter{
+ arvados.Filter{"owner_uuid", "=", imageGroup.UUID},
+ arvados.Filter{"name", "=", collectionName},
+ },
+ Limit: 1})
+ if err != nil {
+ return nil, fmt.Errorf("error querying for collection '%v': %v", collectionName, err)
+ }
+ var imageCollection arvados.Collection
+ if len(cl.Items) == 1 {
+ imageCollection = cl.Items[0]
+ } else {
+ collectionName := collectionName + " " + time.Now().UTC().Format(time.RFC3339)
+ exp := time.Now().Add(24 * 7 * 2 * time.Hour)
+ err = containerClient.RequestAndDecode(&imageCollection,
+ arvados.EndpointCollectionCreate.Method,
+ arvados.EndpointCollectionCreate.Path,
+ nil, map[string]interface{}{
+ "collection": map[string]string{
+ "owner_uuid": imageGroup.UUID,
+ "name": collectionName,
+ "trash_at": exp.UTC().Format(time.RFC3339),
+ },
+ })
+ if err != nil {
+ return nil, fmt.Errorf("error creating '%v' collection: %s", collectionName, err)
+ }
+
+ }
+
+ return &imageCollection, nil
}
// LoadImage will satisfy ContainerExecuter interface transforming
// containerImage into a sif file for later use.
-func (e *singularityExecutor) LoadImage(imageTarballPath string) error {
- e.logf("building singularity image")
- // "singularity build" does not accept a
- // docker-archive://... filename containing a ":" character,
- // as in "/path/to/sha256:abcd...1234.tar". Workaround: make a
- // symlink that doesn't have ":" chars.
- err := os.Symlink(imageTarballPath, e.tmpdir+"/image.tar")
+func (e *singularityExecutor) LoadImage(dockerImageID string, imageTarballPath string, container arvados.Container, arvMountPoint string,
+ containerClient *arvados.Client) error {
+
+ var imageFilename string
+ var sifCollection *arvados.Collection
+ var err error
+ if containerClient != nil {
+ sifCollection, err = e.checkImageCache(dockerImageID, container, arvMountPoint, containerClient)
+ if err != nil {
+ return err
+ }
+ imageFilename = fmt.Sprintf("%s/by_uuid/%s/image.sif", arvMountPoint, sifCollection.UUID)
+ } else {
+ imageFilename = e.tmpdir + "/image.sif"
+ }
+
+ if _, err := os.Stat(imageFilename); os.IsNotExist(err) {
+ e.logf("building singularity image")
+ // "singularity build" does not accept a
+ // docker-archive://... filename containing a ":" character,
+ // as in "/path/to/sha256:abcd...1234.tar". Workaround: make a
+ // symlink that doesn't have ":" chars.
+ err := os.Symlink(imageTarballPath, e.tmpdir+"/image.tar")
+ if err != nil {
+ return err
+ }
+
+ build := exec.Command("singularity", "build", imageFilename, "docker-archive://"+e.tmpdir+"/image.tar")
+ e.logf("%v", build.Args)
+ out, err := build.CombinedOutput()
+ // INFO: Starting build...
+ // Getting image source signatures
+ // Copying blob ab15617702de done
+ // Copying config 651e02b8a2 done
+ // Writing manifest to image destination
+ // Storing signatures
+ // 2021/04/22 14:42:14 info unpack layer: sha256:21cbfd3a344c52b197b9fa36091e66d9cbe52232703ff78d44734f85abb7ccd3
+ // INFO: Creating SIF file...
+ // INFO: Build complete: arvados-jobs.latest.sif
+ e.logf("%s", out)
+ if err != nil {
+ return err
+ }
+ }
+
+ if containerClient == nil {
+ e.imageFilename = imageFilename
+ return nil
+ }
+
+ // update TTL to now + two weeks
+ exp := time.Now().Add(24 * 7 * 2 * time.Hour)
+
+ uuidPath, err := containerClient.PathForUUID("update", sifCollection.UUID)
if err != nil {
- return err
+ e.logf("error PathForUUID: %v", err)
+ return nil
}
- e.imageFilename = e.tmpdir + "/image.sif"
- build := exec.Command("singularity", "build", e.imageFilename, "docker-archive://"+e.tmpdir+"/image.tar")
- e.logf("%v", build.Args)
- out, err := build.CombinedOutput()
- // INFO: Starting build...
- // Getting image source signatures
- // Copying blob ab15617702de done
- // Copying config 651e02b8a2 done
- // Writing manifest to image destination
- // Storing signatures
- // 2021/04/22 14:42:14 info unpack layer: sha256:21cbfd3a344c52b197b9fa36091e66d9cbe52232703ff78d44734f85abb7ccd3
- // INFO: Creating SIF file...
- // INFO: Build complete: arvados-jobs.latest.sif
- e.logf("%s", out)
+ var imageCollection arvados.Collection
+ err = containerClient.RequestAndDecode(&imageCollection,
+ arvados.EndpointCollectionUpdate.Method,
+ uuidPath,
+ nil, map[string]interface{}{
+ "collection": map[string]string{
+ "name": fmt.Sprintf("singularity image for %v", dockerImageID),
+ "trash_at": exp.UTC().Format(time.RFC3339),
+ },
+ })
+ if err == nil {
+ // If we just wrote the image to the cache, the
+ // response also returns the updated PDH
+ e.imageFilename = fmt.Sprintf("%s/by_id/%s/image.sif", arvMountPoint, imageCollection.PortableDataHash)
+ return nil
+ }
+
+ e.logf("error updating/renaming collection for cached sif image: %v", err)
+ // Failed to update but maybe it lost a race and there is
+ // another cached collection in the same place, so check the cache
+ // again
+ sifCollection, err = e.checkImageCache(dockerImageID, container, arvMountPoint, containerClient)
if err != nil {
return err
}
+ e.imageFilename = fmt.Sprintf("%s/by_id/%s/image.sif", arvMountPoint, sifCollection.PortableDataHash)
+
return nil
}
}
func (e *singularityExecutor) Start() error {
- args := []string{"singularity", "exec", "--containall", "--no-home", "--cleanenv"}
+ args := []string{"singularity", "exec", "--containall", "--no-home", "--cleanenv", "--pwd", e.spec.WorkingDir}
if !e.spec.EnableNetwork {
args = append(args, "--net", "--network=none")
}
false: "rw",
true: "ro",
}
- for path, mount := range e.spec.BindMounts {
+ var binds []string
+ for path, _ := range e.spec.BindMounts {
+ binds = append(binds, path)
+ }
+ sort.Strings(binds)
+ for _, path := range binds {
+ mount := e.spec.BindMounts[path]
args = append(args, "--bind", mount.HostPath+":"+path+":"+readonlyflag[mount.ReadOnly])
}
- args = append(args, e.imageFilename)
- args = append(args, e.spec.Command...)
// This is for singularity 3.5.2. There are some behaviors
// that will change in singularity 3.6, please see:
// https://sylabs.io/guides/3.5/user-guide/environment_and_metadata.html
env := make([]string, 0, len(e.spec.Env))
for k, v := range e.spec.Env {
- env = append(env, "SINGULARITYENV_"+k+"="+v)
+ if k == "HOME" {
+ // $HOME is a special case
+ args = append(args, "--home="+v)
+ } else {
+ env = append(env, "SINGULARITYENV_"+k+"="+v)
+ }
}
+ args = append(args, e.imageFilename)
+ args = append(args, e.spec.Command...)
+
path, err := exec.LookPath(args[0])
if err != nil {
return err
}
}
- nodejsversion := "v10.23.1"
+ nodejsversion := "v12.22.2"
if havenodejsversion, err := exec.Command("/usr/local/bin/node", "--version").CombinedOutput(); err == nil && string(havenodejsversion) == nodejsversion+"\n" {
logger.Print("nodejs " + nodejsversion + " already installed")
} else {
{"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger"},
{"touch", "log/production.log"},
{"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
- {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:2.2.19"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"},
help=argparse.SUPPRESS)
parser.add_argument("--thread-count", type=int,
- default=4, help="Number of threads to use for job submit and output collection.")
+ default=0, help="Number of threads to use for job submit and output collection.")
parser.add_argument("--http-timeout", type=int,
default=5*60, dest="http_timeout", help="API request timeout in seconds. Default is 300 seconds (5 minutes).")
def visit(v, cur_id):
if isinstance(v, dict):
- if v.get("class") in ("CommandLineTool", "Workflow"):
+ if v.get("class") in ("CommandLineTool", "Workflow", "ExpressionTool"):
if tool.metadata["cwlVersion"] == "v1.0" and "id" not in v:
raise SourceLine(v, None, Exception).makeError("Embedded process object is missing required 'id' field, add an 'id' or use to cwlVersion: v1.1")
if "id" in v:
if "path" in v and "location" not in v:
v["location"] = v["path"]
del v["path"]
- if "location" in v and not v["location"].startswith("keep:"):
- v["location"] = merged_map[cur_id].resolved[v["location"]]
- if "location" in v and v["location"] in merged_map[cur_id].secondaryFiles:
- v["secondaryFiles"] = merged_map[cur_id].secondaryFiles[v["location"]]
+ if "location" in v and cur_id in merged_map:
+ if v["location"] in merged_map[cur_id].resolved:
+ v["location"] = merged_map[cur_id].resolved[v["location"]]
+ if v["location"] in merged_map[cur_id].secondaryFiles:
+ v["secondaryFiles"] = merged_map[cur_id].secondaryFiles[v["location"]]
if v.get("class") == "DockerRequirement":
v["http://arvados.org/cwl#dockerCollectionPDH"] = arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, v, True,
arvrunner.project_uuid,
# build.
install_requires=[
'cwltool==3.0.20210319143721',
- 'schema-salad==7.1.20210316164414',
+ 'schema-salad==7.1.20210611090601',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
- 'ciso8601 >= 2.0.0'
+ 'ciso8601 >= 2.0.0',
+ 'networkx < 2.6'
],
extras_require={
':os.name=="posix" and python_version<"3"': ['subprocess32 >= 3.5.1'],
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.1
+class: ExpressionTool
+inputs:
+ file1:
+ type: File
+ default:
+ class: File
+ location: keep:f225e6259bdd63bc7240599648dde9f1+97/hg19.fa
+outputs:
+ val: string
+requirements:
+ InlineJavascriptRequirement: {}
+expression: "$({val: inputs.file1.location})"
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+sampleName: woble
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+$namespaces:
+ sbg: https://www.sevenbridges.com/
+class: "Workflow"
+cwlVersion: v1.1
+label: "check that sbg x/y fields are correctly ignored"
+inputs:
+ - id: sampleName
+ type: string
+ label: Sample name
+ 'sbg:x': -22
+ 'sbg:y': 33.4296875
+outputs:
+ - id: outstr
+ type: string
+ outputSource: step1/outstr
+steps:
+ step1:
+ in:
+ sampleName: sampleName
+ out: [outstr]
+ run:
+ class: CommandLineTool
+ inputs:
+ sampleName: string
+ stdout: out.txt
+ outputs:
+ outstr:
+ type: string
+ outputBinding:
+ glob: out.txt
+ loadContents: true
+ outputEval: $(self[0].contents)
+ arguments: [echo, "-n", "foo", $(inputs.sampleName), "bar"]
}
tool: 17801-runtime-outdir.cwl
doc: "Test issue 17801 - bug using $(runtime.outdir) to capture the output directory"
+
+- job: null
+ output:
+ "val": "keep:f225e6259bdd63bc7240599648dde9f1+97/hg19.fa"
+ tool: 17858-pack-visit-crash.cwl
+ doc: "Test issue 17858 - keep ref default inputs on ExpressionTool"
+
+- job: 17879-ignore-sbg-fields-job.yml
+ output:
+ "outstr": "foo woble bar"
+ tool: 17879-ignore-sbg-fields.cwl
+ doc: "Test issue 17879 - ignores sbg fields"
'state': 'Committed',
'command': ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
'name': 'submit_wf.cwl',
expect_container["command"] = [
'arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--disable-reuse', "--collection-cache-size=256",
'--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container["command"] = [
'arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container["use_existing"] = False
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256",
'--debug', '--on-error=stop',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256",
"--output-name="+output_name, '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256", "--debug",
"--storage-classes=foo", '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256", '--debug',
'--on-error=continue',
"--intermediate-output-ttl=3600",
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256",
'--debug', '--on-error=continue',
"--trash-intermediate",
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256",
"--output-tags="+output_tags, '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
'container_image': '999999999999999999999999999999d3+99',
'command': ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
'cwd': '/var/spool/cwl',
'container_image': "999999999999999999999999999999d3+99",
'command': ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
'cwd': '/var/spool/cwl',
expect_container["owner_uuid"] = project_uuid
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- "--eval-timeout=20", "--thread-count=4",
+ "--eval-timeout=20", "--thread-count=0",
'--enable-reuse', "--collection-cache-size=256", '--debug',
'--on-error=continue',
'--project-uuid='+project_uuid,
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=60.0', '--thread-count=4',
+ '--eval-timeout=60.0', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256",
'--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=500",
'--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
}
expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
"--disable-validate",
"--disable-color",
"--eval-timeout=20",
- '--thread-count=4',
+ '--thread-count=0',
"--enable-reuse",
"--collection-cache-size=256",
'--debug',
"bufio"
"context"
"encoding/json"
+ "io"
"net"
"github.com/sirupsen/logrus"
ReturnTo string `json:"return_to"` // Redirect to this URL after logging out
}
+type BlockWriteOptions struct {
+ Hash string
+ Data []byte
+ Reader io.Reader
+ DataSize int // Must be set if Data is nil.
+ RequestID string
+ StorageClasses []string
+ Replicas int
+ Attempts int
+}
+
+type BlockWriteResponse struct {
+ Locator string
+ Replicas int
+}
+
type API interface {
ConfigGet(ctx context.Context) (json.RawMessage, error)
Login(ctx context.Context, options LoginOptions) (LoginResponse, error)
GatewayAddress string `json:"gateway_address"`
InteractiveSessionStarted bool `json:"interactive_session_started"`
OutputStorageClasses []string `json:"output_storage_classes"`
+ RuntimeUserUUID string `json:"runtime_user_uuid"`
+ RuntimeAuthScopes []string `json:"runtime_auth_scopes"`
+ RuntimeToken string `json:"runtime_token"`
}
// ContainerRequest is an arvados#container_request resource.
package arvados
-import "io"
+import (
+ "context"
+ "io"
+)
type fsBackend interface {
keepClient
type keepClient interface {
ReadAt(locator string, p []byte, off int) (int, error)
- PutB(p []byte) (string, int, error)
+ BlockWrite(context.Context, BlockWriteOptions) (BlockWriteResponse, error)
LocalLocator(locator string) (string, error)
}
type collectionFileSystem struct {
fileSystem
- uuid string
+ uuid string
+ replicas int
+ storageClasses []string
}
// FileSystem returns a CollectionFileSystem for the collection.
modTime = time.Now()
}
fs := &collectionFileSystem{
- uuid: c.UUID,
+ uuid: c.UUID,
+ storageClasses: c.StorageClassesDesired,
fileSystem: fileSystem{
fsBackend: keepBackend{apiClient: client, keepClient: kc},
thr: newThrottle(concurrentWriters),
},
}
+ if r := c.ReplicationDesired; r != nil {
+ fs.replicas = *r
+ }
root := &dirnode{
fs: fs,
treenode: treenode{
// filenode implements inode.
type filenode struct {
parent inode
- fs FileSystem
+ fs *collectionFileSystem
fileinfo fileinfo
segments []segment
// number of times `segments` has changed in a
fn.fs.throttle().Acquire()
go func() {
defer close(done)
- locator, _, err := fn.FS().PutB(buf)
+ resp, err := fn.FS().BlockWrite(context.Background(), BlockWriteOptions{
+ Data: buf,
+ Replicas: fn.fs.replicas,
+ StorageClasses: fn.fs.storageClasses,
+ })
fn.fs.throttle().Release()
fn.Lock()
defer fn.Unlock()
fn.memsize -= int64(len(buf))
fn.segments[idx] = storedSegment{
kc: fn.FS(),
- locator: locator,
+ locator: resp.Locator,
size: len(buf),
offset: 0,
length: len(buf),
go func() {
defer close(done)
defer close(errs)
- locator, _, err := dn.fs.PutB(block)
+ resp, err := dn.fs.BlockWrite(context.Background(), BlockWriteOptions{
+ Data: block,
+ Replicas: dn.fs.replicas,
+ StorageClasses: dn.fs.storageClasses,
+ })
dn.fs.throttle().Release()
if err != nil {
errs <- err
data := ref.fn.segments[ref.idx].(*memSegment).buf
ref.fn.segments[ref.idx] = storedSegment{
kc: dn.fs,
- locator: locator,
+ locator: resp.Locator,
size: blocksize,
offset: offsets[idx],
length: len(data),
import (
"bytes"
+ "context"
"crypto/md5"
"errors"
"fmt"
type keepClientStub struct {
blocks map[string][]byte
refreshable map[string]bool
- onPut func(bufcopy []byte) // called from PutB, before acquiring lock
+ onWrite func(bufcopy []byte) // called from WriteBlock, before acquiring lock
authToken string // client's auth token (used for signing locators)
sigkey string // blob signing key
sigttl time.Duration // blob signing ttl
return copy(p, buf[off:]), nil
}
-func (kcs *keepClientStub) PutB(p []byte) (string, int, error) {
- locator := SignLocator(fmt.Sprintf("%x+%d", md5.Sum(p), len(p)), kcs.authToken, time.Now().Add(kcs.sigttl), kcs.sigttl, []byte(kcs.sigkey))
- buf := make([]byte, len(p))
- copy(buf, p)
- if kcs.onPut != nil {
- kcs.onPut(buf)
+func (kcs *keepClientStub) BlockWrite(_ context.Context, opts BlockWriteOptions) (BlockWriteResponse, error) {
+ if opts.Data == nil {
+ panic("oops, stub is not made for this")
+ }
+ locator := SignLocator(fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)), kcs.authToken, time.Now().Add(kcs.sigttl), kcs.sigttl, []byte(kcs.sigkey))
+ buf := make([]byte, len(opts.Data))
+ copy(buf, opts.Data)
+ if kcs.onWrite != nil {
+ kcs.onWrite(buf)
+ }
+ for _, sc := range opts.StorageClasses {
+ if sc != "default" {
+ return BlockWriteResponse{}, fmt.Errorf("stub does not write storage class %q", sc)
+ }
}
kcs.Lock()
defer kcs.Unlock()
kcs.blocks[locator[:32]] = buf
- return locator, 1, nil
+ return BlockWriteResponse{Locator: locator, Replicas: 1}, nil
}
var reRemoteSignature = regexp.MustCompile(`\+[AR][^+]*`)
c.Check(ok, check.Equals, true)
}
+func (s *CollectionFSSuite) TestUnattainableStorageClasses(c *check.C) {
+ fs, err := (&Collection{
+ StorageClassesDesired: []string{"unobtainium"},
+ }).FileSystem(s.client, s.kc)
+ c.Assert(err, check.IsNil)
+
+ f, err := fs.OpenFile("/foo", os.O_CREATE|os.O_WRONLY, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = f.Write([]byte("food"))
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+ _, err = fs.MarshalManifest(".")
+ c.Assert(err, check.ErrorMatches, `.*stub does not write storage class \"unobtainium\"`)
+}
+
func (s *CollectionFSSuite) TestColonInFilename(c *check.C) {
fs, err := (&Collection{
ManifestText: "./foo:foo 3858f62230ac3c915f300c664312c63f+3 0:3:bar:bar\n",
proceed := make(chan struct{})
var started, concurrent int32
blk2done := false
- s.kc.onPut = func([]byte) {
+ s.kc.onWrite = func([]byte) {
atomic.AddInt32(&concurrent, 1)
switch atomic.AddInt32(&started, 1) {
case 1:
fs, err := (&Collection{}).FileSystem(s.client, s.kc)
c.Assert(err, check.IsNil)
- s.kc.onPut = func([]byte) {
+ s.kc.onWrite = func([]byte) {
// discard flushed data -- otherwise the stub will use
// unlimited memory
time.Sleep(time.Millisecond)
c.Assert(err, check.IsNil)
var flushed int64
- s.kc.onPut = func(p []byte) {
+ s.kc.onWrite = func(p []byte) {
atomic.AddInt64(&flushed, int64(len(p)))
}
time.AfterFunc(10*time.Second, func() { close(timeout) })
var putCount, concurrency int64
var unflushed int64
- s.kc.onPut = func(p []byte) {
+ s.kc.onWrite = func(p []byte) {
defer atomic.AddInt64(&unflushed, -int64(len(p)))
cur := atomic.AddInt64(&concurrency, 1)
defer atomic.AddInt64(&concurrency, -1)
})
wrote := 0
- s.kc.onPut = func(p []byte) {
+ s.kc.onWrite = func(p []byte) {
s.kc.Lock()
s.kc.blocks = map[string][]byte{}
wrote++
}
func (s *CollectionFSSuite) TestFlushShort(c *check.C) {
- s.kc.onPut = func([]byte) {
+ s.kc.onWrite = func([]byte) {
s.kc.Lock()
s.kc.blocks = map[string][]byte{}
s.kc.Unlock()
// Importing arvadostest would be an import cycle, so these
// fixtures are duplicated here [until fs moves to a separate
// package].
- fixtureActiveToken = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
- fixtureAProjectUUID = "zzzzz-j7d0g-v955i6s2oi1cbso"
- fixtureThisFilterGroupUUID = "zzzzz-j7d0g-thisfiltergroup"
- fixtureAFilterGroupTwoUUID = "zzzzz-j7d0g-afiltergrouptwo"
- fixtureAFilterGroupThreeUUID = "zzzzz-j7d0g-filtergroupthre"
- fixtureFooAndBarFilesInDirUUID = "zzzzz-4zz18-foonbarfilesdir"
- fixtureFooCollectionName = "zzzzz-4zz18-fy296fx3hot09f7 added sometime"
- fixtureFooCollectionPDH = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
- fixtureFooCollection = "zzzzz-4zz18-fy296fx3hot09f7"
- fixtureNonexistentCollection = "zzzzz-4zz18-totallynotexist"
- fixtureBlobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
- fixtureBlobSigningTTL = 336 * time.Hour
+ fixtureActiveToken = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
+ fixtureAProjectUUID = "zzzzz-j7d0g-v955i6s2oi1cbso"
+ fixtureThisFilterGroupUUID = "zzzzz-j7d0g-thisfiltergroup"
+ fixtureAFilterGroupTwoUUID = "zzzzz-j7d0g-afiltergrouptwo"
+ fixtureAFilterGroupThreeUUID = "zzzzz-j7d0g-filtergroupthre"
+ fixtureFooAndBarFilesInDirUUID = "zzzzz-4zz18-foonbarfilesdir"
+ fixtureFooCollectionName = "zzzzz-4zz18-fy296fx3hot09f7 added sometime"
+ fixtureFooCollectionPDH = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
+ fixtureFooCollection = "zzzzz-4zz18-fy296fx3hot09f7"
+ fixtureNonexistentCollection = "zzzzz-4zz18-totallynotexist"
+ fixtureStorageClassesDesiredArchive = "zzzzz-4zz18-3t236wr12769qqa"
+ fixtureBlobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
+ fixtureBlobSigningTTL = 336 * time.Hour
)
var _ = check.Suite(&SiteFSSuite{})
c.Check(len(fis), check.Equals, 0)
}
+func (s *SiteFSSuite) TestUpdateStorageClasses(c *check.C) {
+ f, err := s.fs.OpenFile("/by_id/"+fixtureStorageClassesDesiredArchive+"/newfile", os.O_CREATE|os.O_RDWR, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = f.Write([]byte("nope"))
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+ err = s.fs.Sync()
+ c.Assert(err, check.ErrorMatches, `.*stub does not write storage class "archive"`)
+}
+
func (s *SiteFSSuite) TestByUUIDAndPDH(c *check.C) {
f, err := s.fs.Open("/by_id")
c.Assert(err, check.IsNil)
AuthEmail string
AuthEmailVerified bool
AuthName string
+ AuthGivenName string
+ AuthFamilyName string
AccessTokenPayload map[string]interface{}
PeopleAPIResponse map[string]interface{}
"email": p.AuthEmail,
"email_verified": p.AuthEmailVerified,
"name": p.AuthName,
+ "given_name": p.AuthGivenName,
+ "family_name": p.AuthFamilyName,
"alt_verified": true, // for custom claim tests
"alt_email": "alt_email@example.com", // for custom claim tests
"alt_username": "desired-username", // for custom claim tests
json.NewEncoder(w).Encode(map[string]interface{}{
"sub": "fake-user-id",
"name": p.AuthName,
- "given_name": p.AuthName,
- "family_name": "",
+ "given_name": p.AuthGivenName,
+ "family_name": p.AuthFamilyName,
"alt_username": "desired-username",
"email": p.AuthEmail,
"email_verified": p.AuthEmailVerified,
}
req.Header.Set(HeaderRequestID, gen.Next())
}
+ w.Header().Set("X-Request-Id", req.Header.Get("X-Request-Id"))
h.ServeHTTP(w, req)
})
}
import (
"bytes"
+ "context"
"crypto/md5"
"errors"
"fmt"
"sync"
"time"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
- "git.arvados.org/arvados.git/sdk/go/asyncbuf"
"git.arvados.org/arvados.git/sdk/go/httpserver"
)
multipleResponseError
}
-type InsufficientReplicasError error
+type InsufficientReplicasError struct{ error }
-type OversizeBlockError error
+type OversizeBlockError struct{ error }
-var ErrOversizeBlock = OversizeBlockError(errors.New("Exceeded maximum block size (" + strconv.Itoa(BLOCKSIZE) + ")"))
+var ErrOversizeBlock = OversizeBlockError{error: errors.New("Exceeded maximum block size (" + strconv.Itoa(BLOCKSIZE) + ")")}
var MissingArvadosApiHost = errors.New("Missing required environment variable ARVADOS_API_HOST")
var MissingArvadosApiToken = errors.New("Missing required environment variable ARVADOS_API_TOKEN")
var InvalidLocatorError = errors.New("Invalid locator")
// Returns an InsufficientReplicasError if 0 <= replicas <
// kc.Wants_replicas.
func (kc *KeepClient) PutHR(hash string, r io.Reader, dataBytes int64) (string, int, error) {
- // Buffer for reads from 'r'
- var bufsize int
- if dataBytes > 0 {
- if dataBytes > BLOCKSIZE {
- return "", 0, ErrOversizeBlock
- }
- bufsize = int(dataBytes)
- } else {
- bufsize = BLOCKSIZE
- }
-
- buf := asyncbuf.NewBuffer(make([]byte, 0, bufsize))
- go func() {
- _, err := io.Copy(buf, HashCheckingReader{r, md5.New(), hash})
- buf.CloseWithError(err)
- }()
- return kc.putReplicas(hash, buf.NewReader, dataBytes)
+ resp, err := kc.BlockWrite(context.Background(), arvados.BlockWriteOptions{
+ Hash: hash,
+ Reader: r,
+ DataSize: int(dataBytes),
+ })
+ return resp.Locator, resp.Replicas, err
}
// PutHB writes a block to Keep. The hash of the bytes is given in
//
// Return values are the same as for PutHR.
func (kc *KeepClient) PutHB(hash string, buf []byte) (string, int, error) {
- newReader := func() io.Reader { return bytes.NewBuffer(buf) }
- return kc.putReplicas(hash, newReader, int64(len(buf)))
+ resp, err := kc.BlockWrite(context.Background(), arvados.BlockWriteOptions{
+ Hash: hash,
+ Data: buf,
+ })
+ return resp.Locator, resp.Replicas, err
}
// PutB writes a block to Keep. It computes the hash itself.
//
// Return values are the same as for PutHR.
func (kc *KeepClient) PutB(buffer []byte) (string, int, error) {
- hash := fmt.Sprintf("%x", md5.Sum(buffer))
- return kc.PutHB(hash, buffer)
+ resp, err := kc.BlockWrite(context.Background(), arvados.BlockWriteOptions{
+ Data: buffer,
+ })
+ return resp.Locator, resp.Replicas, err
}
// PutR writes a block to Keep. It first reads all data from r into a buffer
import (
"bytes"
+ "context"
"crypto/md5"
- "errors"
"fmt"
"io"
"io/ioutil"
"testing"
"time"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
. "gopkg.in/check.v1"
UploadToStubHelper(c, st,
func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, uploadStatusChan chan uploadStatus) {
- go kc.uploadToKeepServer(url, st.expectPath, nil, reader, uploadStatusChan, int64(len("foo")), kc.getRequestID())
+ go kc.uploadToKeepServer(url, st.expectPath, nil, reader, uploadStatusChan, len("foo"), kc.getRequestID())
writer.Write([]byte("foo"))
writer.Close()
UploadToStubHelper(c, st,
func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, uploadStatusChan chan uploadStatus) {
- go kc.uploadToKeepServer(url, st.expectPath, nil, reader, uploadStatusChan, int64(len("foo")), kc.getRequestID())
+ go kc.uploadToKeepServer(url, st.expectPath, nil, reader, uploadStatusChan, len("foo"), kc.getRequestID())
writer.Write([]byte("foo"))
writer.Close()
func (s *StandaloneSuite) TestPutWithStorageClasses(c *C) {
nServers := 5
for _, trial := range []struct {
- replicas int
- classes []string
- minRequests int
- maxRequests int
- success bool
+ replicas int
+ clientClasses []string
+ putClasses []string // putClasses takes precedence over clientClasses
+ minRequests int
+ maxRequests int
+ success bool
}{
- {1, []string{"class1"}, 1, 1, true},
- {2, []string{"class1"}, 1, 2, true},
- {3, []string{"class1"}, 2, 3, true},
- {1, []string{"class1", "class2"}, 1, 1, true},
- {nServers*2 + 1, []string{"class1"}, nServers, nServers, false},
- {1, []string{"class404"}, nServers, nServers, false},
- {1, []string{"class1", "class404"}, nServers, nServers, false},
+ {1, []string{"class1"}, nil, 1, 1, true},
+ {2, []string{"class1"}, nil, 1, 2, true},
+ {3, []string{"class1"}, nil, 2, 3, true},
+ {1, []string{"class1", "class2"}, nil, 1, 1, true},
+ {3, nil, []string{"class1"}, 2, 3, true},
+ {1, nil, []string{"class1", "class2"}, 1, 1, true},
+ {1, []string{"class404"}, []string{"class1", "class2"}, 1, 1, true},
+ {1, []string{"class1"}, []string{"class404", "class2"}, nServers, nServers, false},
+ {nServers*2 + 1, []string{"class1"}, nil, nServers, nServers, false},
+ {1, []string{"class404"}, nil, nServers, nServers, false},
+ {1, []string{"class1", "class404"}, nil, nServers, nServers, false},
+ {1, nil, []string{"class1", "class404"}, nServers, nServers, false},
} {
c.Logf("%+v", trial)
st := &StubPutHandler{
arv, _ := arvadosclient.MakeArvadosClient()
kc, _ := MakeKeepClient(arv)
kc.Want_replicas = trial.replicas
- kc.StorageClasses = trial.classes
+ kc.StorageClasses = trial.clientClasses
arv.ApiToken = "abc123"
localRoots := make(map[string]string)
writableLocalRoots := make(map[string]string)
}
kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
- _, _, err := kc.PutB([]byte("foo"))
+ _, err := kc.BlockWrite(context.Background(), arvados.BlockWriteOptions{
+ Data: []byte("foo"),
+ StorageClasses: trial.putClasses,
+ })
if trial.success {
c.Check(err, check.IsNil)
} else {
_, replicas, err := kc.PutB([]byte("foo"))
- c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
+ c.Check(err, FitsTypeOf, InsufficientReplicasError{})
c.Check(replicas, Equals, 1)
c.Check(<-st.handled, Equals, ks1[0].url)
}
_, replicas, err := kc.PutB([]byte("foo"))
<-st.handled
- c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
+ c.Check(err, FitsTypeOf, InsufficientReplicasError{})
c.Check(replicas, Equals, 2)
}
_, replicas, err := kc.PutB([]byte("foo"))
- c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
+ c.Check(err, FitsTypeOf, InsufficientReplicasError{})
c.Check(replicas, Equals, 1)
c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)])
_, replicas, err := kc.PutB([]byte("foo"))
- c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
+ c.Check(err, FitsTypeOf, InsufficientReplicasError{})
c.Check(replicas, Equals, 0)
}
package keepclient
import (
+ "bytes"
+ "context"
"crypto/md5"
"errors"
"fmt"
"strconv"
"strings"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/asyncbuf"
)
// DebugPrintf emits debug messages. The easiest way to enable
}
func (kc *KeepClient) uploadToKeepServer(host string, hash string, classesTodo []string, body io.Reader,
- uploadStatusChan chan<- uploadStatus, expectedLength int64, reqid string) {
+ uploadStatusChan chan<- uploadStatus, expectedLength int, reqid string) {
var req *http.Request
var err error
return
}
- req.ContentLength = expectedLength
+ req.ContentLength = int64(expectedLength)
if expectedLength > 0 {
req.Body = ioutil.NopCloser(body)
} else {
}
}
-func (kc *KeepClient) putReplicas(
- hash string,
- getReader func() io.Reader,
- expectedLength int64) (locator string, replicas int, err error) {
-
- reqid := kc.getRequestID()
+func (kc *KeepClient) BlockWrite(ctx context.Context, req arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
+ var resp arvados.BlockWriteResponse
+ var getReader func() io.Reader
+ if req.Data == nil && req.Reader == nil {
+ return resp, errors.New("invalid BlockWriteOptions: Data and Reader are both nil")
+ }
+ if req.DataSize < 0 {
+ return resp, fmt.Errorf("invalid BlockWriteOptions: negative DataSize %d", req.DataSize)
+ }
+ if req.DataSize > BLOCKSIZE || len(req.Data) > BLOCKSIZE {
+ return resp, ErrOversizeBlock
+ }
+ if req.Data != nil {
+ if req.DataSize > len(req.Data) {
+ return resp, errors.New("invalid BlockWriteOptions: DataSize > len(Data)")
+ }
+ if req.DataSize == 0 {
+ req.DataSize = len(req.Data)
+ }
+ getReader = func() io.Reader { return bytes.NewReader(req.Data[:req.DataSize]) }
+ } else {
+ buf := asyncbuf.NewBuffer(make([]byte, 0, req.DataSize))
+ go func() {
+ _, err := io.Copy(buf, HashCheckingReader{req.Reader, md5.New(), req.Hash})
+ buf.CloseWithError(err)
+ }()
+ getReader = buf.NewReader
+ }
+ if req.Hash == "" {
+ m := md5.New()
+ _, err := io.Copy(m, getReader())
+ if err != nil {
+ return resp, err
+ }
+ req.Hash = fmt.Sprintf("%x", m.Sum(nil))
+ }
+ if req.StorageClasses == nil {
+ req.StorageClasses = kc.StorageClasses
+ }
+ if req.Replicas == 0 {
+ req.Replicas = kc.Want_replicas
+ }
+ if req.RequestID == "" {
+ req.RequestID = kc.getRequestID()
+ }
+ if req.Attempts == 0 {
+ req.Attempts = 1 + kc.Retries
+ }
// Calculate the ordering for uploading to servers
- sv := NewRootSorter(kc.WritableLocalRoots(), hash).GetSortedRoots()
+ sv := NewRootSorter(kc.WritableLocalRoots(), req.Hash).GetSortedRoots()
// The next server to try contacting
nextServer := 0
}()
}()
- replicasWanted := kc.Want_replicas
replicasTodo := map[string]int{}
- for _, c := range kc.StorageClasses {
- replicasTodo[c] = replicasWanted
+ for _, c := range req.StorageClasses {
+ replicasTodo[c] = req.Replicas
}
- replicasDone := 0
replicasPerThread := kc.replicasPerService
if replicasPerThread < 1 {
// unlimited or unknown
- replicasPerThread = replicasWanted
+ replicasPerThread = req.Replicas
}
- retriesRemaining := 1 + kc.Retries
+ retriesRemaining := req.Attempts
var retryServers []string
lastError := make(map[string]string)
}
}
if !trackingClasses {
- maxConcurrency = replicasWanted - replicasDone
+ maxConcurrency = req.Replicas - resp.Replicas
}
if maxConcurrency < 1 {
// If there are no non-zero entries in
for active*replicasPerThread < maxConcurrency {
// Start some upload requests
if nextServer < len(sv) {
- DebugPrintf("DEBUG: [%s] Begin upload %s to %s", reqid, hash, sv[nextServer])
- go kc.uploadToKeepServer(sv[nextServer], hash, classesTodo, getReader(), uploadStatusChan, expectedLength, reqid)
+ DebugPrintf("DEBUG: [%s] Begin upload %s to %s", req.RequestID, req.Hash, sv[nextServer])
+ go kc.uploadToKeepServer(sv[nextServer], req.Hash, classesTodo, getReader(), uploadStatusChan, req.DataSize, req.RequestID)
nextServer++
active++
} else {
msg += resp + "; "
}
msg = msg[:len(msg)-2]
- return locator, replicasDone, InsufficientReplicasError(errors.New(msg))
+ return resp, InsufficientReplicasError{error: errors.New(msg)}
}
break
}
}
- DebugPrintf("DEBUG: [%s] Replicas remaining to write: %v active uploads: %v", reqid, replicasTodo, active)
+ DebugPrintf("DEBUG: [%s] Replicas remaining to write: %v active uploads: %v", req.RequestID, replicasTodo, active)
if active < 1 {
break
}
if status.statusCode == http.StatusOK {
delete(lastError, status.url)
- replicasDone += status.replicasStored
+ resp.Replicas += status.replicasStored
if len(status.classesStored) == 0 {
// Server doesn't report
// storage classes. Give up
delete(replicasTodo, className)
}
}
- locator = status.response
+ resp.Locator = status.response
} else {
msg := fmt.Sprintf("[%d] %s", status.statusCode, status.response)
if len(msg) > 100 {
sv = retryServers
}
- return locator, replicasDone, nil
+ return resp, nil
}
func parseStorageClassesConfirmedHeader(hdr string) (map[string]int, error) {
copy_opts.add_argument(
'--project-uuid', dest='project_uuid',
help='The UUID of the project at the destination to which the collection or workflow should be copied.')
+ copy_opts.add_argument(
+ '--storage-classes', dest='storage_classes',
+ help='Comma separated list of storage classes to be used when saving data to the destinaton Arvados instance.')
copy_opts.add_argument(
'object_uuid',
parents=[copy_opts, arv_cmd.retry_opt])
args = parser.parse_args()
+ if args.storage_classes:
+ args.storage_classes = [x for x in args.storage_classes.strip().replace(' ', '').split(',') if x]
+
if args.verbose:
logger.setLevel(logging.DEBUG)
else:
if not body["name"]:
body['name'] = "copied from " + collection_uuid
+ if args.storage_classes:
+ body['storage_classes_desired'] = args.storage_classes
+
body['owner_uuid'] = args.project_uuid
dst_collection = dst.collections().create(body=body, ensure_unique_name=True).execute(num_retries=args.retries)
if progress_writer:
progress_writer.report(obj_uuid, bytes_written, bytes_expected)
data = src_keep.get(word)
- dst_locator = dst_keep.put(data)
+ dst_locator = dst_keep.put(data, classes=(args.storage_classes or []))
dst_locators[blockhash] = dst_locator
bytes_written += loc.size
dst_manifest.write(' ')
with c.open('foo', 'wt') as f:
f.write('foo')
c.save_new("arv-copy foo collection", owner_uuid=src_proj)
+ coll_record = api.collections().get(uuid=c.manifest_locator()).execute()
+ assert coll_record['storage_classes_desired'] == ['default']
dest_proj = api.groups().create(body={"group": {"name": "arv-copy dest project", "group_class": "project"}}).execute()["uuid"]
assert len(contents["items"]) == 0
try:
- self.run_copy(["--project-uuid", dest_proj, src_proj])
+ self.run_copy(["--project-uuid", dest_proj, "--storage-classes", "foo", src_proj])
except SystemExit as e:
assert e.code == 0
assert contents["items"][0]["uuid"] != c.manifest_locator()
assert contents["items"][0]["name"] == "arv-copy foo collection"
assert contents["items"][0]["portable_data_hash"] == c.portable_data_hash()
+ assert contents["items"][0]["storage_classes_desired"] == ["foo"]
finally:
os.environ['HOME'] = home_was
activemodel (>= 3.0.0)
activesupport (>= 3.0.0)
rack (>= 1.1.0)
- addressable (2.7.0)
+ addressable (2.8.0)
public_suffix (>= 2.0.2, < 5.0)
andand (1.3.3)
arel (9.0.0)
end
err[:errors] ||= args
err[:errors].map! do |err|
- err += " (" + Thread.current[:request_id] + ")"
+ err += " (#{request.request_id})"
end
err[:error_token] = [Time.now.utc.to_i, "%08x" % rand(16 ** 8)].join("+")
status = err.delete(:status) || 422
end
def set_current_request_id
- req_id = request.headers['X-Request-Id']
- if !req_id || req_id.length < 1 || req_id.length > 1024
- # Client-supplied ID is either missing or too long to be
- # considered friendly.
- req_id = "req-" + Random::DEFAULT.rand(2**128).to_s(36)[0..19]
- end
- response.headers['X-Request-Id'] = Thread.current[:request_id] = req_id
- Rails.logger.tagged(req_id) do
+ Rails.logger.tagged(request.request_id) do
yield
end
- Thread.current[:request_id] = nil
end
def append_info_to_payload(payload)
# already know how to properly treat them.
attribute :secret_mounts, :jsonbHash, default: {}
attribute :runtime_status, :jsonbHash, default: {}
- attribute :runtime_auth_scopes, :jsonbHash, default: {}
+ attribute :runtime_auth_scopes, :jsonbArray, default: []
attribute :output_storage_classes, :jsonbArray, default: ["default"]
serialize :environment, Hash
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+module CustomRequestId
+ def make_request_id(req_id)
+ if !req_id || req_id.length < 1 || req_id.length > 1024
+ # Client-supplied ID is either missing or too long to be
+ # considered friendly.
+ internal_request_id
+ else
+ req_id
+ end
+ end
+
+ def internal_request_id
+ "req-" + Random::DEFAULT.rand(2**128).to_s(36)[0..19]
+ end
+end
+
+class ActionDispatch::RequestId
+ # Instead of using the default UUID-like format for X-Request-Id headers,
+ # use our own.
+ prepend CustomRequestId
+end
\ No newline at end of file
token_time = token.split('+', 2).first.to_i
assert_operator(token_time, :>=, @start_stamp, "error token too old")
assert_operator(token_time, :<=, now_timestamp, "error token too new")
- json_response['errors'].each do |err|
- assert_match(/req-[a-z0-9]{20}/, err, "X-Request-Id value missing on error message")
- end
end
def check_404(errmsg="Path not found")
check_error_token
end
- test "X-Request-Id header" do
- authorize_with :spectator
- get(:index)
- assert_match /^req-[0-9a-zA-Z]{20}$/, response.headers['X-Request-Id']
- end
-
- # The response header is the one that gets logged, so this test also
- # ensures we log the ID supplied in the request, if any.
- test "X-Request-Id given by client" do
- authorize_with :spectator
- @request.headers['X-Request-Id'] = 'abcdefG'
- get(:index)
- assert_equal 'abcdefG', response.headers['X-Request-Id']
- end
-
- test "X-Request-Id given by client is ignored if too long" do
- authorize_with :spectator
- @request.headers['X-Request-Id'] = 'abcdefG' * 1000
- get(:index)
- assert_match /^req-[0-9a-zA-Z]{20}$/, response.headers['X-Request-Id']
- end
-
['foo', '', 'FALSE', 'TRUE', nil, [true], {a:true}, '"true"'].each do |bogus|
test "bogus boolean parameter #{bogus.inspect} returns error" do
@controller = Arvados::V1::GroupsController.new
assert_nil assigns(:object)
assert_not_nil json_response['errors']
assert_response 404
+ assert_match /^req-[0-9a-zA-Z]{20}$/, response.headers['X-Request-Id']
end
end
"Unexpected new route: #{route.path.spec}")
end
end
+
+ test "X-Request-Id header" do
+ get "/", headers: auth(:spectator)
+ assert_match /^req-[0-9a-zA-Z]{20}$/, response.headers['X-Request-Id']
+ end
+
+ test "X-Request-Id header on non-existant object URL" do
+ get "/arvados/v1/container_requests/invalid",
+ params: {:format => :json}, headers: auth(:active)
+ assert_response 404
+ assert_match /^req-[0-9a-zA-Z]{20}$/, response.headers['X-Request-Id']
+ end
+
+ # The response header is the one that gets logged, so this test also
+ # ensures we log the ID supplied in the request, if any.
+ test "X-Request-Id given by client" do
+ get "/", headers: auth(:spectator).merge({'X-Request-Id': 'abcdefG'})
+ assert_equal 'abcdefG', response.headers['X-Request-Id']
+ end
+
+ test "X-Request-Id given by client is ignored if too long" do
+ authorize_with :spectator
+ long_reqId = 'abcdefG' * 1000
+ get "/", headers: auth(:spectator).merge({'X-Request-Id': long_reqId})
+ assert_match /^req-[0-9a-zA-Z]{20}$/, response.headers['X-Request-Id']
+ end
end
"syscall"
"time"
+ "git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/dispatch"
return nil
}
+ loader := config.NewLoader(nil, logger)
+ cfg, err := loader.Load()
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return fmt.Errorf("config error: %s", err)
+ }
+
logger.Printf("crunch-dispatch-local %s started", version)
runningCmds = make(map[string]*exec.Cmd)
+ var client arvados.Client
+ client.APIHost = cluster.Services.Controller.ExternalURL.Host
+ client.AuthToken = cluster.SystemRootToken
+ client.Insecure = cluster.TLS.Insecure
+
+ if client.APIHost != "" || client.AuthToken != "" {
+ // Copy real configs into env vars so [a]
+ // MakeArvadosClient() uses them, and [b] they get
+ // propagated to crunch-run via SLURM.
+ os.Setenv("ARVADOS_API_HOST", client.APIHost)
+ os.Setenv("ARVADOS_API_TOKEN", client.AuthToken)
+ os.Setenv("ARVADOS_API_HOST_INSECURE", "")
+ if client.Insecure {
+ os.Setenv("ARVADOS_API_HOST_INSECURE", "1")
+ }
+ os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
+ } else {
+ logger.Warnf("Client credentials missing from config, so falling back on environment variables (deprecated).")
+ }
+
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
logger.Errorf("error making Arvados client: %v", err)
dispatcher := dispatch.Dispatcher{
Logger: logger,
Arv: arv,
- RunContainer: (&LocalRun{startFunc, make(chan bool, 8), ctx}).run,
+ RunContainer: (&LocalRun{startFunc, make(chan bool, 8), ctx, cluster}).run,
PollPeriod: time.Duration(*pollInterval) * time.Second,
}
startCmd func(container arvados.Container, cmd *exec.Cmd) error
concurrencyLimit chan bool
ctx context.Context
+ cluster *arvados.Cluster
}
// Run a container.
waitGroup.Add(1)
defer waitGroup.Done()
- cmd := exec.Command(*crunchRunCommand, uuid)
+ cmd := exec.Command(*crunchRunCommand, "--runtime-engine="+lr.cluster.Containers.RuntimeEngine, uuid)
cmd.Stdin = nil
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stderr
return cmd.Start()
}
+ cl := arvados.Cluster{Containers: arvados.ContainersConfig{RuntimeEngine: "docker"}}
+
dispatcher.RunContainer = func(d *dispatch.Dispatcher, c arvados.Container, s <-chan arvados.Container) error {
defer cancel()
- return (&LocalRun{startCmd, make(chan bool, 8), ctx}).run(d, c, s)
+ return (&LocalRun{startCmd, make(chan bool, 8), ctx, &cl}).run(d, c, s)
}
err = dispatcher.Run(ctx)
return cmd.Start()
}
+ cl := arvados.Cluster{Containers: arvados.ContainersConfig{RuntimeEngine: "docker"}}
+
dispatcher.RunContainer = func(d *dispatch.Dispatcher, c arvados.Container, s <-chan arvados.Container) error {
defer cancel()
- return (&LocalRun{startCmd, make(chan bool, 8), ctx}).run(d, c, s)
+ return (&LocalRun{startCmd, make(chan bool, 8), ctx, &cl}).run(d, c, s)
}
re := regexp.MustCompile(`(?ms).*` + expected + `.*`)
// append() here avoids modifying crunchRunCommand's
// underlying array, which is shared with other goroutines.
crArgs := append([]string(nil), crunchRunCommand...)
+ crArgs = append(crArgs, "--runtime-engine="+disp.cluster.Containers.RuntimeEngine)
crArgs = append(crArgs, container.UUID)
crScript := strings.NewReader(execScript(crArgs))
import (
"bytes"
"crypto/md5"
- "errors"
"fmt"
"io/ioutil"
"math/rand"
content := []byte("TestDesiredReplicas")
hash := fmt.Sprintf("%x", md5.Sum(content))
- for _, kc.Want_replicas = range []int{0, 1, 2} {
+ for _, kc.Want_replicas = range []int{0, 1, 2, 3} {
locator, rep, err := kc.PutB(content)
- c.Check(err, Equals, nil)
- c.Check(rep, Equals, kc.Want_replicas)
- if rep > 0 {
- c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
+ if kc.Want_replicas < 3 {
+ c.Check(err, Equals, nil)
+ c.Check(rep, Equals, kc.Want_replicas)
+ if rep > 0 {
+ c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
+ }
+ } else {
+ c.Check(err, ErrorMatches, ".*503.*")
}
}
}
hash2, rep, err := kc.PutB([]byte("bar"))
c.Check(hash2, Equals, "")
c.Check(rep, Equals, 0)
- c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
+ c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError{})
blocklen, _, err := kc.Ask(hash)
c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
} else {
c.Check(hash2, Equals, "")
c.Check(rep, Equals, 0)
- c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
+ c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError{})
}
logbuf.Reset()
}
if existing_groups.index(addgroup).nil?
# User should be in group, but isn't, so add them.
STDERR.puts "Add user #{username} to #{addgroup} group"
- system("adduser", username, addgroup)
+ system("usermod", "-aG", addgroup, username)
end
end
if groups.index(removegroup).nil?
# User is in a group, but shouldn't be, so remove them.
STDERR.puts "Remove user #{username} from #{removegroup} group"
- system("deluser", username, removegroup)
+ system("gpasswd", "-d", username, removegroup)
end
end
WORKBENCH2_ROOT="$ARVBOX_DATA/workbench2"
fi
+if test -z "$ARVADOS_BRANCH" ; then
+ ARVADOS_BRANCH=main
+fi
+
+if test -z "$WORKBENCH2_BRANCH" ; then
+ WORKBENCH2_BRANCH=main
+fi
+
PG_DATA="$ARVBOX_DATA/postgres"
VAR_DATA="$ARVBOX_DATA/var"
PASSENGER="$ARVBOX_DATA/passenger"
GOSTUFF="$ARVBOX_DATA/gopath"
RLIBS="$ARVBOX_DATA/Rlibs"
ARVADOS_CONTAINER_PATH="/var/lib/arvados-arvbox"
-GEM_HOME="/var/lib/arvados/lib/ruby/gems/2.5.0"
+GEM_HOME="/var/lib/arvados/lib/ruby/gems/2.7.0"
getip() {
docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $ARVBOX_CONTAINER
if ! test -d "$ARVADOS_ROOT" ; then
git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT"
+ git -C "$ARVADOS_ROOT" checkout $ARVADOS_BRANCH
fi
if ! test -d "$COMPOSER_ROOT" ; then
git clone https://github.com/arvados/composer.git "$COMPOSER_ROOT"
git -C "$COMPOSER_ROOT" checkout arvados-fork
- git -C "$COMPOSER_ROOT" pull
fi
if ! test -d "$WORKBENCH2_ROOT" ; then
git clone https://git.arvados.org/arvados-workbench2.git "$WORKBENCH2_ROOT"
+ git -C "$ARVADOS_ROOT" checkout $WORKBENCH2_BRANCH
fi
if [[ "$CONFIG" = test ]] ; then
BUILDTYPE=dev
fi
- docker build --build-arg=BUILDTYPE=$BUILDTYPE $NO_CACHE --build-arg=arvados_version=$GITHEAD --build-arg=workdir=/tools/arvbox/lib/arvbox/docker -t arvados/arvbox-base:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.base" "$LOCAL_ARVADOS_ROOT"
+ if test "$ARVADOS_BRANCH" = "main" ; then
+ ARVADOS_BRANCH=$GITHEAD
+ fi
+
+ docker build --build-arg=BUILDTYPE=$BUILDTYPE $NO_CACHE \
+ --build-arg=arvados_version=$ARVADOS_BRANCH \
+ --build-arg=workbench2_version=$WORKBENCH2_BRANCH \
+ --build-arg=workdir=/tools/arvbox/lib/arvbox/docker \
+ -t arvados/arvbox-base:$GITHEAD \
+ -f "$ARVBOX_DOCKER/Dockerfile.base" \
+ "$LOCAL_ARVADOS_ROOT"
docker tag $FORCE arvados/arvbox-base:$GITHEAD arvados/arvbox-base:latest
- docker build $NO_CACHE -t arvados/arvbox-$BUILDTYPE:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.$BUILDTYPE" "$ARVBOX_DOCKER"
+ docker build $NO_CACHE \
+ --build-arg=arvados_version=$ARVADOS_BRANCH \
+ --build-arg=workbench2_version=$WORKBENCH2_BRANCH \
+ -t arvados/arvbox-$BUILDTYPE:$GITHEAD \
+ -f "$ARVBOX_DOCKER/Dockerfile.$BUILDTYPE" \
+ "$ARVBOX_DOCKER"
docker tag $FORCE arvados/arvbox-$BUILDTYPE:$GITHEAD arvados/arvbox-$BUILDTYPE:latest
}
else
echo "Usage: $0 $subcmd <start|stop|restart> <service>"
echo "Available services:"
- exec docker execa $ARVBOX_CONTAINER ls /etc/service
+ exec docker exec $ARVBOX_CONTAINER ls /etc/service
fi
;;
cd /usr/src/arvados/services/api
export DISABLE_DATABASE_ENVIRONMENT_CHECK=1
export RAILS_ENV=development
-flock $GEM_HOME/gems.lock bundle exec rake db:drop
+flock $GEM_HOME/gems.lock bin/bundle exec rake db:drop
rm $ARVADOS_CONTAINER_PATH/api_database_setup
rm $ARVADOS_CONTAINER_PATH/superuser_token
sv start api
RUN cd /usr/src && \
git clone --no-checkout https://git.arvados.org/arvados.git && \
git -C arvados checkout ${arvados_version} && \
- git -C arvados pull && \
git clone --no-checkout https://github.com/arvados/composer.git && \
git -C composer checkout ${composer_version} && \
- git -C composer pull && \
git clone --no-checkout https://git.arvados.org/arvados-workbench2.git workbench2 && \
git -C workbench2 checkout ${workbench2_version} && \
- git -C workbench2 pull && \
chown -R 1000:1000 /usr/src
# avoid rebuilding arvados-server, it's already been built as part of the base image
RUN sudo -u arvbox /var/lib/arvbox/service/vm/run-service --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/keepproxy/run-service --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/arv-git-httpd/run-service --only-deps
-RUN sudo -u arvbox /var/lib/arvbox/service/crunch-dispatch-local/run-service --only-deps
+RUN /var/lib/arvbox/service/crunch-dispatch-local/run --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/websockets/run --only-deps
RUN sudo -u arvbox /usr/local/lib/arvbox/keep-setup.sh --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/sdk/run-service
fi
if ! test -f $ARVADOS_CONTAINER_PATH/api_database_setup ; then
- flock $GEM_HOME/gems.lock bundle exec rake db:setup
+ flock $GEM_HOME/gems.lock bin/bundle exec rake db:setup
touch $ARVADOS_CONTAINER_PATH/api_database_setup
fi
if ! test -s $ARVADOS_CONTAINER_PATH/superuser_token ; then
- superuser_tok=$(flock $GEM_HOME/gems.lock bundle exec ./script/create_superuser_token.rb)
+ superuser_tok=$(flock $GEM_HOME/gems.lock bin/bundle exec ./script/create_superuser_token.rb)
echo "$superuser_tok" > $ARVADOS_CONTAINER_PATH/superuser_token
fi
rm -rf tmp
mkdir -p tmp/cache
-flock $GEM_HOME/gems.lock bundle exec rake db:migrate
+flock $GEM_HOME/gems.lock bin/bundle exec rake db:migrate
run_bundler() {
if test -f Gemfile.lock ; then
- # The 'gem install bundler line below' is cf.
- # https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html,
- # until we get bundler 2.7.10/3.0.0 or higher
- flock $GEM_HOME/gems.lock gem install bundler --no-document -v "$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tail -n 1|tr -d ' ')"
frozen=--frozen
else
frozen=""
fi
- # if ! test -x $GEM_HOME/bin/bundler ; then
- # bundleversion=2.0.2
- # bundlergem=$(ls -r $GEM_HOME/cache/bundler-${bundleversion}.gem 2>/dev/null | head -n1 || true)
- # if test -n "$bundlergem" ; then
- # flock $GEM_HOME/gems.lock gem install --verbose --local --no-document $bundlergem
- # else
- # flock $GEM_HOME/gems.lock gem install --verbose --no-document bundler --version ${bundleversion}
- # fi
- # fi
- # Make sure to put the gem binaries in the right place
- flock /var/lib/arvados/lib/ruby/gems/2.5.0/gems.lock bundler config bin $GEM_HOME/bin
- if ! flock $GEM_HOME/gems.lock bundler install --verbose --local --no-deployment $frozen "$@" ; then
- flock $GEM_HOME/gems.lock bundler install --verbose --no-deployment $frozen "$@"
+ BUNDLER=bundler
+ if test -x $PWD/bin/bundler ; then
+ # If present, use the one associated with rails workbench or API
+ BUNDLER=$PWD/bin/bundler
+ fi
+ if ! flock $GEM_HOME/gems.lock $BUNDLER install --verbose --local --no-deployment $frozen "$@" ; then
+ flock $GEM_HOME/gems.lock $BUNDLER install --verbose --no-deployment $frozen "$@"
fi
}
fi
run_bundler --without=development
-flock $GEM_HOME/gems.lock bundle exec passenger-config build-native-support
-flock $GEM_HOME/gems.lock bundle exec passenger-config install-standalone-runtime
+flock $GEM_HOME/gems.lock bin/bundle exec passenger-config build-native-support
+flock $GEM_HOME/gems.lock bin/bundle exec passenger-config install-standalone-runtime
if test "$1" = "--only-deps" ; then
exit
touch $ARVADOS_CONTAINER_PATH/api.ready
-exec bundle exec passenger start --port=${services[api]}
+exec bin/bundle exec passenger start --port=${services[api]}
+++ /dev/null
-/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+exec 2>&1
+set -ex -o pipefail
+
+# singularity can use suid
+chown root /var/lib/arvados/bin/singularity \
+ /var/lib/arvados/etc/singularity/singularity.conf \
+ /var/lib/arvados/etc/singularity/capability.json \
+ /var/lib/arvados/etc/singularity/ecl.toml
+chmod u+s /var/lib/arvados/bin/singularity
+
+exec /usr/local/lib/arvbox/runsu.sh $0-service $1
fi
done
-if ! (ps x | grep -v grep | grep "crunch-dispatch") > /dev/null ; then
+if ! (ps ax | grep -v grep | grep "crunch-dispatch") > /dev/null ; then
waiting="$waiting crunch-dispatch"
fi
if test "$1" != "--only-deps" ; then
openssl verify -CAfile $root_cert $server_cert
- exec bundle exec passenger start --port=${services[workbench]} \
+ exec bin/bundle exec passenger start --port=${services[workbench]} \
--ssl --ssl-certificate=$ARVADOS_CONTAINER_PATH/server-cert-${localip}.pem \
--ssl-certificate-key=$ARVADOS_CONTAINER_PATH/server-cert-${localip}.key \
--user arvbox
fi
run_bundler --without=development
-flock $GEM_HOME/gems.lock bundle exec passenger-config build-native-support
-flock $GEM_HOME/gems.lock bundle exec passenger-config install-standalone-runtime
+flock $GEM_HOME/gems.lock bin/bundle exec passenger-config build-native-support
+flock $GEM_HOME/gems.lock bin/bundle exec passenger-config install-standalone-runtime
mkdir -p /usr/src/arvados/apps/workbench/tmp
if test "$1" = "--only-deps" ; then
$RAILS_ENV:
keep_web_url: https://example.com/c=%{uuid_or_pdh}
EOF
- RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bundle exec rake npm:install
+ RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bin/bundle exec rake npm:install
rm config/application.yml
exit
fi
secret_token=$(cat $ARVADOS_CONTAINER_PATH/workbench_secret_token)
-RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bundle exec rake npm:install
-flock $GEM_HOME/gems.lock bundle exec rake assets:precompile
+RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bin/bundle exec rake npm:install
+flock $GEM_HOME/gems.lock bin/bundle exec rake assets:precompile
arv api_client create --api-client "$apiclient"
fi
-export HTTPS=false
# Can't use "yarn start", need to run the dev server script
# directly so that the TERM signal from "sv restart" gets to the
# right process.
export VERSION=$(./version-at-commit.sh)
-exec node node_modules/react-scripts-ts/scripts/start.js
+export BROWSER=none
+export CI=true
+node --version
+exec node node_modules/react-scripts/scripts/start.js
ProviderType: t3.small
VCPUs: 2
RAM: 2GiB
- IncludedScratch: 50GB
AddedScratch: 50GB
Price: 0.0208
c5large:
ProviderType: c5.large
VCPUs: 2
RAM: 4GiB
- IncludedScratch: 50GB
AddedScratch: 50GB
Price: 0.085
m5large:
ProviderType: m5.large
VCPUs: 2
RAM: 8GiB
- IncludedScratch: 50GB
AddedScratch: 50GB
Price: 0.096
c5xlarge:
ProviderType: c5.xlarge
VCPUs: 4
RAM: 8GiB
- IncludedScratch: 100GB
AddedScratch: 100GB
Price: 0.17
m5xlarge:
ProviderType: m5.xlarge
VCPUs: 4
RAM: 16GiB
- IncludedScratch: 100GB
AddedScratch: 100GB
Price: 0.192
m5xlarge_extradisk:
ProviderType: m5.xlarge
VCPUs: 4
RAM: 16GiB
- IncludedScratch: 400GB
AddedScratch: 400GB
Price: 0.193
c52xlarge:
ProviderType: c5.2xlarge
VCPUs: 8
RAM: 16GiB
- IncludedScratch: 200GB
AddedScratch: 200GB
Price: 0.34
m52xlarge:
ProviderType: m5.2xlarge
VCPUs: 8
RAM: 32GiB
- IncludedScratch: 200GB
AddedScratch: 200GB
Price: 0.384
c54xlarge:
ProviderType: c5.4xlarge
VCPUs: 16
RAM: 32GiB
- IncludedScratch: 400GB
AddedScratch: 400GB
Price: 0.68
m54xlarge:
ProviderType: m5.4xlarge
VCPUs: 16
RAM: 64GiB
- IncludedScratch: 400GB
AddedScratch: 400GB
Price: 0.768
projects / arvados.git / commitdiff
