## Quick start
-Curoverse maintains an Arvados public cloud demo at
-[https://cloud.curoverse.com](https://cloud.curoverse.com). A Google account
-is required to log in.
+Veritas Genetics maintains a public installation of Arvados for evaluation and trial use, the [Arvados Playground](https://playground.arvados.org). A Google account is required to log in.
To try out Arvados on your local workstation, you can use Arvbox, which
provides Arvados components pre-installed in a Docker container (requires
mini_portile2 (~> 2.3.0)
npm-rails (0.2.1)
rails (>= 3.2)
- oj (3.5.0)
+ oj (3.6.4)
os (0.9.6)
passenger (5.2.1)
rack
simplecov-html (0.10.2)
simplecov-rcov (0.2.3)
simplecov (>= 0.4.1)
- sprockets (3.7.1)
+ sprockets (3.7.2)
concurrent-ruby (~> 1.0)
rack (> 1, < 3)
sprockets-rails (3.2.1)
wiselinks
BUNDLED WITH
- 1.16.1
+ 1.16.2
url = 'https://' + url;
}
url = new URL(url);
- return db.discoveryDoc({baseURL: url.origin}).map(function() {
+ return m.request(url.origin + '/discovery/v1/apis/arvados/v1/rest').then(function() {
return url.origin + '/';
}).catch(function(err) {
// If url is a Workbench site (and isn't too old),
// discovery doc from a session's API server.
discoveryDoc: function(session) {
var cache = db.discoveryCache[session.baseURL];
- if (!cache) {
+ if (!cache && session) {
db.discoveryCache[session.baseURL] = cache = m.stream();
- m.request(session.baseURL+'discovery/v1/apis/arvados/v1/rest')
+ var baseURL = session.baseURL;
+ if (baseURL[baseURL.length - 1] !== '/') {
+ baseURL += '/';
+ }
+ m.request(baseURL+'discovery/v1/apis/arvados/v1/rest')
.then(function (dd) {
// Just in case we're talking with an old API server.
dd.remoteHosts = dd.remoteHosts || {};
autoLoadRemoteHosts: function() {
var sessions = db.loadAll();
var doc = db.discoveryDoc(db.loadLocal());
+ if (doc === undefined) { return; }
doc.map(function(d) {
Object.keys(d.remoteHosts).map(function(uuidPrefix) {
if (!(sessions[uuidPrefix])) {
attrs['cwd'] = "/var/spool/cwl"
attrs['output_path'] = "/var/spool/cwl"
+ # runtime constriants
+ runtime_constraints = {
+ "vcpus" => 1,
+ "ram" => 1024 * 1024 * 1024,
+ "API" => true
+ }
+
input_defaults = {}
if wf_json
- inputs = get_cwl_inputs(wf_json)
- inputs.each do |input|
+ main = get_cwl_main(wf_json)
+ main[:inputs].each do |input|
if input[:default]
input_defaults[cwl_shortname(input[:id])] = input[:default]
end
end
+ if main[:hints]
+ main[:hints].each do |hint|
+ if hint[:class] == "http://arvados.org/cwl#WorkflowRunnerResources"
+ if hint[:coresMin]
+ runtime_constraints["vcpus"] = hint[:coresMin]
+ end
+ if hint[:ramMin]
+ runtime_constraints["ram"] = hint[:ramMin] * 1024 * 1024
+ end
+ end
+ end
+ end
end
# mounts
end
attrs['mounts'] = mounts
- # runtime constriants
- runtime_constraints = {
- "vcpus" => 1,
- "ram" => 256000000,
- "API" => true
- }
attrs['runtime_constraints'] = runtime_constraints
else
raise ArgumentError, "Unsupported template uuid: #{template_uuid}"
lt
end
- def get_cwl_inputs(workflow)
- if workflow[:inputs]
- return workflow[:inputs]
+ def get_cwl_main(workflow)
+ if workflow[:"$graph"].nil?
+ return workflow
else
workflow[:"$graph"].each do |tool|
if tool[:id] == "#main"
- return tool[:inputs]
+ return tool
end
end
end
end
+ def get_cwl_inputs(workflow)
+ get_cwl_main(workflow)[:inputs]
+ end
+
+
def cwl_shortname(id)
if id[0] == "#"
id = id[1..-1]
arvados_api_client.discovery[:source_version]
end
+ # Get the packageVersion given in the API server's discovery
+ # document.
+ def api_package_version
+ arvados_api_client.discovery[:packageVersion]
+ end
+
# URL for browsing source code for the given version.
def version_link_target version
"https://arvados.org/projects/arvados/repository/changes?rev=#{version.sub(/-.*/, "")}"
additional_info_str = additional_info.map {|k,v| "#{k}=#{v}"}.join("\n")
additional_info['api_source_version'] = api_source_version
+ additional_info['api_package_version'] = api_package_version
additional_info['generated_at'] = generated_at
additional_info['workbench_version'] = AppVersion.hash
+ additional_info['workbench_package_version'] = AppVersion.package_version
additional_info['arvados_base'] = arvados_base
additional_info['support_email'] = support_email
additional_info['error_message'] = params[:error_message] if params[:error_message]
<label for="wb_version" class="col-sm-4 control-label"> Workbench version </label>
<div class="col-sm-8">
<p class="form-control-static" name="wb_version">
- <%= link_to AppVersion.hash, version_link_target(AppVersion.hash) %>
+ <%= AppVersion.package_version %> (<%= link_to AppVersion.hash, version_link_target(AppVersion.hash) %>)
</p>
</div>
</div>
<label for="server_version" class="col-sm-4 control-label"> API version </label>
<div class="col-sm-8">
<p class="form-control-static" name="server_version">
- <%= link_to api_source_version, version_link_target(api_source_version) %>
+ <%= api_package_version %> (<%= link_to api_source_version, version_link_target(api_source_version) %>)
</p>
</div>
</div>
i18n.fallbacks: true
active_support.deprecation: :notify
profiling_enabled: false
+ log_level: info
arvados_insecure_https: false
# "git log".
source_version: false
+ # Override the automatic package string. With the default value of
+ # false, the package string is read from package-build.version in
+ # Rails.root (included in vendor packages).
+ package_version: false
+
# report notification to and from addresses
issue_reporter_email_from: arvados@example.com
issue_reporter_email_to: arvados@example.com
def self.forget
@hash = nil
+ @package_version = nil
end
# Return abbrev commit hash for current code version: "abc1234", or
@hash || "unknown"
end
+
+ def self.package_version
+ if (cached = Rails.configuration.package_version || @package_version)
+ return cached
+ end
+
+ begin
+ @package_version = IO.read(Rails.root.join("package-build.version")).strip
+ rescue Errno::ENOENT
+ @package_version = "unknown"
+ end
+
+ @package_version
+ end
end
end
if Capybara.current_driver == :selenium
page.execute_script("window.localStorage.clear()")
+ else
+ page.driver.restart if defined?(page.driver.restart)
end
Capybara.reset_sessions!
end
end
def run_test_server
- env_script = nil
Dir.chdir PYTHON_TESTS_DIR do
- # These are no-ops if we're running within run-tests.sh (except
- # that we do get a useful env_script back from "start", even
- # though it doesn't need to start up a new server).
- env_script = check_output %w(python ./run_test_server.py start --auth admin)
- check_output %w(python ./run_test_server.py start_arv-git-httpd)
- check_output %w(python ./run_test_server.py start_keep-web)
- check_output %w(python ./run_test_server.py start_nginx)
- # This one isn't a no-op, even under run-tests.sh.
check_output %w(python ./run_test_server.py start_keep)
end
- test_env = {}
- env_script.each_line do |line|
- line = line.chomp
- if 0 == line.index('export ')
- toks = line.sub('export ', '').split '=', 2
- $stderr.puts "run_test_server.py: #{toks[0]}=#{toks[1]}"
- test_env[toks[0]] = toks[1]
- end
- end
- test_env
end
def stop_test_server
Dir.chdir PYTHON_TESTS_DIR do
check_output %w(python ./run_test_server.py stop_keep)
- # These are no-ops if we're running within run-tests.sh
- check_output %w(python ./run_test_server.py stop_nginx)
- check_output %w(python ./run_test_server.py stop_arv-git-httpd)
- check_output %w(python ./run_test_server.py stop_keep-web)
- check_output %w(python ./run_test_server.py stop)
end
@@server_is_running = false
end
stop_test_server
end
- test_env = run_test_server
- $application_config['arvados_login_base'] = "https://#{test_env['ARVADOS_API_HOST']}/login"
- $application_config['arvados_v1_base'] = "https://#{test_env['ARVADOS_API_HOST']}/arvados/v1"
+ run_test_server
+ $application_config['arvados_login_base'] = "https://#{ENV['ARVADOS_API_HOST']}/login"
+ $application_config['arvados_v1_base'] = "https://#{ENV['ARVADOS_API_HOST']}/arvados/v1"
$application_config['arvados_insecure_host'] = true
ActiveSupport::TestCase.reset_application_config
#distribution(s)|name|version|iteration|type|architecture|extra fpm arguments
debian8,debian9,centos7|python-gflags|2.0|2|python|all
debian8,debian9,ubuntu1404,ubuntu1604,centos7|google-api-python-client|1.6.2|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|apache-libcloud|2.3.0|3|python|all|--depends 'python-requests >= 2.4.3'
debian8,debian9,ubuntu1404,centos7|oauth2client|1.5.2|2|python|all
debian8,debian9,ubuntu1404,centos7|pyasn1|0.1.7|2|python|all
debian8,debian9,ubuntu1404,centos7|pyasn1-modules|0.0.5|2|python|all
debian8,debian9,ubuntu1404,ubuntu1604,centos7|rdflib|4.2.2|2|python|all
debian8,debian9,ubuntu1404,centos7|shellescape|3.4.1|2|python|all
debian8,debian9,ubuntu1404,ubuntu1604,centos7|mistune|0.7.3|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|typing|3.5.3.0|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|typing|3.6.4|2|python|all
debian8,debian9,ubuntu1404,ubuntu1604,centos7|avro|1.8.1|2|python|all
debian8,debian9,ubuntu1404,centos7|ruamel.ordereddict|0.4.9|2|python|amd64
debian8,debian9,ubuntu1404,ubuntu1604,centos7|cachecontrol|0.11.7|2|python|all
centos7|pbr|0.11.1|2|python|all
centos7|pyparsing|2.1.10|2|python|all
centos7|keepalive|0.5|2|python|all
+centos7|networkx|1.11|0|python|all
+centos7|psutil|5.0.1|0|python|all
debian8,debian9,ubuntu1404,ubuntu1604,centos7|lockfile|0.12.2|2|python|all|--epoch 1
debian8,debian9,ubuntu1404,ubuntu1604,centos7|subprocess32|3.5.1|2|python|all
all|ruamel.yaml|0.14.12|2|python|amd64|--python-setup-py-arguments --single-version-externally-managed
all|futures|3.0.5|2|python|all
all|future|0.16.0|2|python|all
all|future|0.16.0|2|python3|all
+all|mypy-extensions|0.3.0|1|python|all
+all|prov|1.5.1|0|python|all
+all|bagit|1.6.4|0|python|all
+all|typing-extensions|3.6.5|0|python|all
#
# SPDX-License-Identifier: AGPL-3.0
-LIBCLOUD_PIN=2.3.0
+LIBCLOUD_PIN=2.3.1.dev1
-using_fork=false
+using_fork=true
if [[ $using_fork = true ]]; then
LIBCLOUD_PIN_SRC="https://github.com/curoverse/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip"
else
export ARV_PACKAGES_DIR="/arvados/packages/$target"
-dpkg-deb -x $(ls -t "$ARV_PACKAGES_DIR/$1"_*.deb | head -n1) .
+if [[ -f $(ls -t "$ARV_PACKAGES_DIR/$1"_*.deb | head -n1) ]] ; then
+ debpkg=$(ls -t "$ARV_PACKAGES_DIR/$1"_*.deb | head -n1)
+else
+ debpkg=$(ls -t "$ARV_PACKAGES_DIR/processed/$1"_*.deb | head -n1)
+fi
+
+dpkg-deb -x $debpkg .
while read so && [ -n "$so" ]; do
echo
go get github.com/kardianos/govendor
package_go_binary cmd/arvados-client arvados-client \
"Arvados command line tool (beta)"
+package_go_binary cmd/arvados-server arvados-server \
+ "Arvados server daemons"
+package_go_binary cmd/arvados-server arvados-controller \
+ "Arvados cluster controller daemon"
package_go_binary sdk/go/crunchrunner crunchrunner \
"Crunchrunner executes a command inside a container and uploads the output"
package_go_binary services/arv-git-httpd arvados-git-httpd \
fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage" --iteration "$iteration"
fi
-## if libcloud becomes our own fork see
-## https://dev.arvados.org/issues/12268#note-27
+# Forked libcloud
+if test_package_presence "$PYTHON2_PKG_PREFIX"-apache-libcloud "$LIBCLOUD_PIN" python 2
+then
+ LIBCLOUD_DIR=$(mktemp -d)
+ (
+ cd $LIBCLOUD_DIR
+ git clone $DASHQ_UNLESS_DEBUG https://github.com/curoverse/libcloud.git .
+ git checkout $DASHQ_UNLESS_DEBUG apache-libcloud-$LIBCLOUD_PIN
+ # libcloud is absurdly noisy without -q, so force -q here
+ OLD_DASHQ_UNLESS_DEBUG=$DASHQ_UNLESS_DEBUG
+ DASHQ_UNLESS_DEBUG=-q
+ handle_python_package
+ DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
+ )
+
+ # libcloud >= 2.3.0 now requires python-requests 2.4.3 or higher, otherwise
+ # it throws
+ # ImportError: No module named packages.urllib3.poolmanager
+ # when loaded. We only see this problem on ubuntu1404, because that is our
+ # only supported distribution that ships with a python-requests older than
+ # 2.4.3.
+ fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud "" python "" --iteration 2 --depends 'python-requests >= 2.4.3'
+ rm -rf $LIBCLOUD_DIR
+fi
# Python 2 dependencies
declare -a PIP_DOWNLOAD_SWITCHES=(--no-deps)
-#!/bin/bash
+#!/bin/bash -xe
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: AGPL-3.0
declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
ARVADOS_BUILDING_VERSION="$(git describe --abbrev=0).$(date -ud "@$git_ts" +%Y%m%d%H%M%S)"
echo "$ARVADOS_BUILDING_VERSION"
-}
+}
nohash_version_from_git() {
if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
# Arvados SDK and the SDK has changed.
declare -a checkdirs=(vendor)
if grep -qr git.curoverse.com/arvados .; then
- checkdirs+=(sdk/go)
- if [[ "$prog" -eq "crunch-dispatch-slurm" ]]; then
- checkdirs+=(lib/dispatchcloud)
- fi
+ checkdirs+=(sdk/go lib)
fi
for dir in ${checkdirs[@]}; do
cd "$GOPATH/src/git.curoverse.com/arvados.git/$dir"
repo_subdir=${pkgname:0:1}
fi
- repo_pkg_list=$(curl -o - http://apt.arvados.org/pool/${D}/main/${repo_subdir}/)
+ repo_pkg_list=$(curl -s -o - http://apt.arvados.org/pool/${D}/main/${repo_subdir}/)
echo ${repo_pkg_list} |grep -q ${complete_pkgname}
- if [ $? -eq 0 ]; then
+ if [ $? -eq 0 ] ; then
echo "Package $complete_pkgname exists, not rebuilding!"
curl -o ./${complete_pkgname} http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${complete_pkgname}
return 1
+ elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then
+ echo "Package $complete_pkgname exists, not rebuilding!"
+ return 1
else
echo "Package $complete_pkgname not found, building"
return 0
cd "$srcdir"
local license_path="$1"; shift
local version="$(version_from_git)"
+ echo "$version" >package-build.version
local scripts_dir="$(mktemp --tmpdir -d "$pkgname-XXXXXXXX.scripts")" && \
(
set -e
apps/workbench_benchmark
apps/workbench_profile
cmd/arvados-client
+cmd/arvados-server
doc
lib/cli
lib/cmd
+lib/controller
lib/crunchstat
lib/dispatchcloud
services/api
declare -A skip
declare -A testargs
skip[apps/workbench_profile]=1
+# nodemanager_integration tests are not reliable, see #12061.
+skip[services/nodemanager_integration]=1
while [[ -n "$1" ]]
do
rm -f "$WORKSPACE/tmp/api.pid"
fi
cd "$WORKSPACE" \
- && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
+ && eval $(python sdk/python/tests/run_test_server.py start --auth admin || echo fail=1) \
&& export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
&& export ARVADOS_TEST_API_INSTALLED="$$" \
+ && python sdk/python/tests/run_test_server.py start_controller \
&& python sdk/python/tests/run_test_server.py start_keep_proxy \
&& python sdk/python/tests/run_test_server.py start_keep-web \
&& python sdk/python/tests/run_test_server.py start_arv-git-httpd \
&& python sdk/python/tests/run_test_server.py start_ws \
- && python sdk/python/tests/run_test_server.py start_nginx \
+ && eval $(python sdk/python/tests/run_test_server.py start_nginx || echo fail=1) \
&& (env | egrep ^ARVADOS)
+ if [[ -n "$fail" ]]; then
+ return 1
+ fi
}
stop_services() {
&& python sdk/python/tests/run_test_server.py stop_ws \
&& python sdk/python/tests/run_test_server.py stop_keep-web \
&& python sdk/python/tests/run_test_server.py stop_keep_proxy \
+ && python sdk/python/tests/run_test_server.py stop_controller \
&& python sdk/python/tests/run_test_server.py stop
}
fi
done
+rm -vf "${WORKSPACE}/tmp/*.log"
+
setup_ruby_environment() {
if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then
source "$HOME/.rvm/scripts/rvm"
set -e
mkdir -p "$GOPATH/src/git.curoverse.com"
rmdir -v --parents --ignore-fail-on-non-empty "${temp}/GOPATH"
+ if [[ ! -h "$GOPATH/src/git.curoverse.com/arvados.git" ]]; then
+ for d in \
+ "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH" \
+ "$GOPATH/src/git.curoverse.com/arvados.git/tmp" \
+ "$GOPATH/src/git.curoverse.com/arvados.git"; do
+ [[ -d "$d" ]] && rmdir "$d"
+ done
+ fi
for d in \
- "$GOPATH/src/git.curoverse.com/arvados.git/arvados.git" \
- "$GOPATH/src/git.curoverse.com/arvados.git"; do
- [[ -d "$d" ]] && rmdir "$d"
+ "$GOPATH/src/git.curoverse.com/arvados.git/arvados" \
+ "$GOPATH/src/git.curoverse.com/arvados.git"; do
[[ -h "$d" ]] && rm "$d"
done
- ln -vsnfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
+ ln -vsfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
go get -v github.com/kardianos/govendor
cd "$GOPATH/src/git.curoverse.com/arvados.git"
if [[ -n "$short" ]]; then
gem install --user-install bundler || fatal 'Could not install bundler'
fi
+# Jenkins config requires that glob tmp/*.log match something. Ensure
+# that happens even if we don't end up running services that set up
+# logging.
+mkdir -p "${WORKSPACE}/tmp/" || fatal "could not mkdir ${WORKSPACE}/tmp"
+touch "${WORKSPACE}/tmp/controller.log" || fatal "could not touch ${WORKSPACE}/tmp/controller.log"
+
retry() {
remain="${repeat}"
while :
;;
esac
if [[ -z "${skip[$suite]}" && -z "${skip[$1]}" && \
- (-z "${only}" || "${only}" == "${suite}" || \
- "${only}" == "${1}") ]]; then
+ (-z "${only}" || "${only}" == "${suite}" || \
+ "${only}" == "${1}") ||
+ "${only}" == "${2}" ]]; then
retry do_test_once ${@}
else
title "Skipping ${1} tests"
do_install() {
skipit=false
- if [[ -z "${only_install}" || "${only_install}" == "${1}" ]]; then
+ if [[ -z "${only_install}" || "${only_install}" == "${1}" || "${only_install}" == "${2}" ]]; then
retry do_install_once ${@}
else
skipit=true
declare -a gostuff
gostuff=(
cmd/arvados-client
+ cmd/arvados-server
lib/cli
lib/cmd
+ lib/controller
lib/crunchstat
lib/dispatchcloud
sdk/go/arvados
package main
import (
- "fmt"
- "io"
"os"
- "regexp"
- "runtime"
"git.curoverse.com/arvados.git/lib/cli"
"git.curoverse.com/arvados.git/lib/cmd"
)
var (
- version = "dev"
- cmdVersion cmd.Handler = versionCmd{}
- handler = cmd.Multi(map[string]cmd.Handler{
- "-e": cmdVersion,
- "version": cmdVersion,
- "-version": cmdVersion,
- "--version": cmdVersion,
+ version = "dev"
+ handler = cmd.Multi(map[string]cmd.Handler{
+ "-e": cmd.Version(version),
+ "version": cmd.Version(version),
+ "-version": cmd.Version(version),
+ "--version": cmd.Version(version),
"copy": cli.Copy,
"create": cli.Create,
})
)
-type versionCmd struct{}
-
-func (versionCmd) RunCommand(prog string, args []string, _ io.Reader, stdout, _ io.Writer) int {
- prog = regexp.MustCompile(` -*version$`).ReplaceAllLiteralString(prog, "")
- fmt.Fprintf(stdout, "%s %s (%s)\n", prog, version, runtime.Version())
- return 0
-}
-
func fixLegacyArgs(args []string) []string {
flags, _ := cli.LegacyFlagSet()
return cmd.SubcommandToFront(args, flags)
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+[Unit]
+Description=Arvados controller
+Documentation=https://doc.arvados.org/
+After=network.target
+AssertPathExists=/etc/arvados/config.yml
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
+StartLimitInterval=0
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
+StartLimitIntervalSec=0
+
+[Service]
+Type=notify
+EnvironmentFile=-/etc/arvados/environment
+ExecStart=/usr/bin/arvados-controller
+Restart=always
+RestartSec=1
+
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
+[Install]
+WantedBy=multi-user.target
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "os"
+
+ "git.curoverse.com/arvados.git/lib/cmd"
+ "git.curoverse.com/arvados.git/lib/controller"
+)
+
+var (
+ version = "dev"
+ handler = cmd.Multi(map[string]cmd.Handler{
+ "version": cmd.Version(version),
+ "-version": cmd.Version(version),
+ "--version": cmd.Version(version),
+
+ "controller": controller.Command,
+ })
+)
+
+func main() {
+ os.Exit(handler.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
+}
- Run a workflow using Workbench:
- user/getting_started/workbench.html.textile.liquid
- user/tutorials/tutorial-workflow-workbench.html.textile.liquid
+ - user/composer/composer.html.textile.liquid
- Access an Arvados virtual machine:
- user/getting_started/vm-login-with-webshell.html.textile.liquid
- user/getting_started/ssh-access-unix.html.textile.liquid
- user/tutorials/tutorial-keep-mount.html.textile.liquid
- user/topics/keep.html.textile.liquid
- user/topics/arv-copy.html.textile.liquid
+ - user/topics/storage-classes.html.textile.liquid
- Running workflows at the command line:
- user/cwl/cwl-runner.html.textile.liquid
- user/cwl/cwl-run-options.html.textile.liquid
admin:
- Topics:
- admin/index.html.textile.liquid
+ - Upgrading and migrations:
- admin/upgrading.html.textile.liquid
+ - install/migrate-docker19.html.textile.liquid
+ - Users and Groups:
- install/cheat_sheet.html.textile.liquid
- - user/topics/arvados-sync-groups.html.textile.liquid
- - admin/migrating-providers.html.textile.liquid
+ - admin/activation.html.textile.liquid
- admin/merge-remote-account.html.textile.liquid
- - install/migrate-docker19.html.textile.liquid
+ - admin/migrating-providers.html.textile.liquid
+ - user/topics/arvados-sync-groups.html.textile.liquid
+ - Monitoring:
+ - admin/health-checks.html.textile.liquid
+ - admin/metrics.html.textile.liquid
+ - admin/management-token.html.textile.liquid
+ - Cloud:
+ - admin/storage-classes.html.textile.liquid
+ - admin/spot-instances.html.textile.liquid
installguide:
- Overview:
- install/index.html.textile.liquid
- Docker quick start:
- install/arvbox.html.textile.liquid
+ - Arvados on Kubernetes:
+ - install/arvados-on-kubernetes.html.textile.liquid
- Manual installation:
- install/install-manual-prerequisites.html.textile.liquid
+ - install/install-components.html.textile.liquid
+ - Core:
- install/install-postgresql.html.textile.liquid
- - install/install-sso.html.textile.liquid
- install/install-api-server.html.textile.liquid
- - install/install-ws.html.textile.liquid
- - install/install-arv-git-httpd.html.textile.liquid
- - install/install-workbench-app.html.textile.liquid
- - install/install-shell-server.html.textile.liquid
- - install/create-standard-objects.html.textile.liquid
+ - install/install-controller.html.textile.liquid
+ - Keep:
- install/install-keepstore.html.textile.liquid
+ - install/configure-fs-storage.html.textile.liquid
+ - install/configure-s3-object-storage.html.textile.liquid
- install/configure-azure-blob-storage.html.textile.liquid
- install/install-keepproxy.html.textile.liquid
- install/install-keep-web.html.textile.liquid
- install/install-keep-balance.html.textile.liquid
+ - User interface:
+ - install/install-sso.html.textile.liquid
+ - install/install-workbench-app.html.textile.liquid
+ - install/install-composer.html.textile.liquid
+ - Additional services:
+ - install/install-ws.html.textile.liquid
+ - install/install-shell-server.html.textile.liquid
+ - install/install-arv-git-httpd.html.textile.liquid
- Containers API support on SLURM:
- install/crunch2-slurm/install-prerequisites.html.textile.liquid
- install/crunch2-slurm/install-slurm.html.textile.liquid
table(table table-bordered table-condensed).
|_. Key|_. Type|_. Description|_. Notes|
|partitions|array of strings|The names of one or more compute partitions that may run this container. If not provided, the system will choose where to run the container.|Optional.|
+|preemptible|boolean|If true, the dispatcher will ask for a preemptible cloud node instance (eg: AWS Spot Instance) to run this container.|Optional. Default is false.|
+|max_run_time|integer|Maximum running time (in seconds) that this container will be allowed to run before being cancelled.|Optional. Default is 0 (no limit).|
</notextile>
Finally, reboot the system to make these changes effective.
+
+h2. Create a project for Docker images
+
+Here we create a default project for the standard Arvados Docker images, and give all users read access to it. The project is owned by the system user.
+
+<notextile>
+<pre><code>~$ <span class="userinput">project_uuid=`arv --format=uuid group create --group "{\"owner_uuid\":\"$prefix-tpzed-000000000000000\", \"name\":\"Arvados Standard Docker Images\"}"`</span>
+~$ <span class="userinput">echo "Arvados project uuid is '$project_uuid'"</span>
+~$ <span class="userinput">read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"</span>
+<span class="userinput">{
+ "tail_uuid":"$all_users_group_uuid",
+ "head_uuid":"$project_uuid",
+ "link_class":"permission",
+ "name":"can_read"
+}
+EOF</span>
+</code></pre></notextile>
+
+h2. Download and tag the latest arvados/jobs docker image
+
+In order to start workflows from workbench, there needs to be Docker image tagged @arvados/jobs:latest@. The following command downloads the latest arvados/jobs image from Docker Hub, loads it into Keep, and tags it as 'latest'. In this example @$project_uuid@ should be the the UUID of the "Arvados Standard Docker Images" project.
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv-keepdocker --project-uuid $project_uuid --pull arvados/jobs latest</span>
+</code></pre></notextile>
+
+If the image needs to be downloaded from Docker Hub, the command can take a few minutes to complete, depending on available network bandwidth.
+++ /dev/null
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-<div class="alert alert-block alert-info">
- <button type="button" class="close" data-dismiss="alert">×</button>
- <h4>Note!</h4>
- <p>The SSO server codebase currently uses OpenID 2.0 to talk to Google's authentication service. Google <a href="https://developers.google.com/accounts/docs/OpenID2">has deprecated that protocol</a>. This means that new clients will not be allowed to talk to Google's authentication services anymore over OpenID 2.0, and they will phase out the use of OpenID 2.0 completely in the coming monts. We are working on upgrading the SSO server codebase to a newer protocol. That work should be complete by the end of November 2014. In the mean time, anyone is free to use the existing Curoverse SSO server for any local Arvados installation. Instructions to do so are provided on the "API server":install-api-server.html page.</p>
- <p><strong>Recommendation: skip this step</strong></p>
-</div>
--- /dev/null
+---
+layout: default
+navsection: admin
+title: User activation
+...
+
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+This page describes how new users are created and activated.
+
+"Browser login and management of API tokens is described here.":{{site.baseurl}}/api/tokens.html
+
+h3. Authentication
+
+After completing the authentication process, a callback is made from the SSO server to the API server, providing a user record and @identity_url@ (despite the name, this is actually an Arvados user uuid).
+
+The API server searches for a user record with the @identity_url@ supplied by the SSO. If found, that user account will be used, unless the account has @redirect_to_user_uuid@ set, in which case it will use the user in @redirect_to_user_uuid@ instead (this is used for the "link account":{{site.baseurl}}/user/topics/link-accounts.html feature).
+
+Next, it searches by email address for a "pre-activated account.":#pre-activated
+
+If no existing user record is found, a new user object will be created.
+
+A federated user follows a slightly different flow, whereby a special token is presented and the API server verifies user's identity with the home cluster, however it also results in a user object (representing the remote user) being created.
+
+h3. User setup
+
+If @auto_setup_new_users@ is true, as part of creating the new user object, the user is immediately set up with:
+
+* @can_login@ @permission@ link going (email address → user uuid) which records @identity_url_prefix@
+* Membership in the "All users" group (can read all users, all users can see new user)
+* A new git repo and @can_manage@ permission if @auto_setup_new_users_with_repository@ is true
+* @can_login@ permission to a shell node if @auto_setup_new_users_with_vm_uuid@ is set to the uuid of a vm
+
+Otherwise, an admin must explicitly invoke "setup" on the user via workbench or the API.
+
+h3. User activation
+
+A newly created user is inactive (@is_active@ is false) by default unless @new_users_are_active@.
+
+An inactive user cannot create or update any object, but can read Arvados objects that the user account has permission to read. This implies that if @auto_setup_new_users@ is true, an "inactive" user who has been set up may still be able to do things, such as read things shared with "All users", clone and push to the git repository, or login to a VM.
+
+{% comment %}
+Maybe these services should check is_active.
+
+I believe that when this was originally designed, being able to access git and VM required an ssh key, and an inactive user could not register an ssh key because that required creating a record. However, it is now possible to authenticate to shell VMs and http+git with just an API token.
+{% endcomment %}
+
+At this point, there are two ways a user can be activated.
+
+# An admin can set the @is_active@ field directly. This runs @setup_on_activate@ which sets up oid_login_perm and group membership, but does not set repo or vm (even if if @auto_setup_new_users_with_repository@ and/or @auto_setup_new_users_with_vm_uuid@ are set).
+# Self-activation using the @activate@ method of the users controller.
+
+h3. User agreements
+
+The @activate@ method of the users controller checks if the user @is_invited@ and whether the user has "signed" all the user agreements.
+
+@is_invited@ is true if any of these are true:
+* @is_active@ is true
+* @new_users_are_active@ is true
+* the user account has a permission link to read the system "all users" group.
+
+User agreements are accessed by getting a listing on the @user_agreements@ endpoint. This returns a list of collection uuids. This is executed as a system user, so it bypasses normal read permission checks.
+
+The available user agreements are represented in the Links table as
+
+<pre>
+{
+ "link_class": "signature",
+ "name": "require",
+ "tail_uuid": "*system user uuid*",
+ "head_uuid: "*collection uuid*"
+}
+</pre>
+
+The collection contains the user agreement text file.
+
+On workbench, it checks @is_invited@. If true, it displays the clickthrough agreements which the user can "sign". If @is_invited@ is false, the user ends up at the "inactive user" page.
+
+The @user_agreements/sign@ endpoint creates a Link object:
+
+<pre>
+{
+ "link_class": "signature"
+ "name": "click",
+ "tail_uuid": "*user uuid*",
+ "head_uuid: "*collection uuid*"
+}
+</pre>
+
+This is executed as a system user, so it bypasses the restriction that inactive users cannot create objects.
+
+The @user_agreements/signatures@ endpoint returns the list of Link objects that represent signatures by the current user (created by @sign@).
+
+h3. User profile
+
+The user profile is checked by workbench after checking if user agreements need to be signed. The requirement to fill out the user profile is not enforced by the API server.
+
+h3(#pre-activated). Pre-activate user by email address
+
+You may create a user account for a user that has not yet logged in, and identify the user by email address.
+
+1. As an admin, create a user object:
+
+<pre>
+{
+ "email": "foo@example.com",
+ "username": "barney",
+ "is_active": true
+}
+</pre>
+
+2. Create a link object, where @tail_uuid@ is the user's email address, @head_uuid@ is the user object created in the previous step, and @xxxxx@ is the value of @uuid_prefix@ of the SSO server.
+
+<pre>
+{
+ "link_class": "permission",
+ "name": "can_login",
+ "tail_uuid": "email address",
+ "head_uuid: "user uuid",
+ "properties": {
+ "identity_url_prefix": "xxxxx-tpzed-"
+ }
+}
+</pre>
+
+3. When the user logs in the first time, the email address will be recognized and the user will be associated with the linked user object.
+
+h3. Pre-activate federated user
+
+1. As admin, create a user object with the @uuid@ of the federated user (this is the user's uuid on their home cluster):
+
+<pre>
+{
+ "uuid": "home1-tpzed-000000000000000",
+ "email": "foo@example.com",
+ "username": "barney",
+ "is_active": true
+}
+</pre>
+
+2. When the user logs in, they will be associated with the existing user object.
+
+h3. Auto-activate federated users from trusted clusters
+
+In the API server config, configure @auto_activate_users_from@ with a list of one or more five-character cluster ids. A federated user from one of the listed clusters which @is_active@ on the home cluster will be automatically set up and activated on this cluster.
+
+h3(#deactivating_users). Deactivating users
+
+Setting @is_active@ is not sufficient to lock out a user. The user can call @activate@ to become active again. Instead, use @unsetup@:
+
+* Delete oid_login_perms
+* Delete git repository permission links
+* Delete VM login permission links
+* Remove from "All users" group
+* Delete any "signatures"
+* Clear preferences / profile
+* Mark as inactive
+
+{% comment %}
+Does not revoke @is_admin@, so you can't unsetup an admin unless you turn admin off first.
+
+"inactive" does not prevent user from reading things they previously had access to.
+
+Does not revoke API tokens.
+{% endcomment %}
+
+h3. Activation flows
+
+h4. Private instance
+
+Policy: users must be manually approved.
+
+<pre>
+auto_setup_new_users: false
+new_users_are_active: false
+</pre>
+
+# User is created. Not set up. @is_active@ is false.
+# Workbench checks @is_invited@ and finds it is false. User gets "inactive user" page.
+# Admin goes to user page and clicks either "setup user" or manually @is_active@ to true.
+# Clicking "setup user" sets up the user. This includes adding the user to "All users" which qualifies the user as @is_invited@.
+# On refreshing workbench, the user is still inactive, but is able to self-activate after signing clickthrough agreements (if any).
+# Alternately, directly setting @is_active@ to true also sets up the user, but workbench won't display clickthrough agreements (because the user is already active).
+
+h4. Federated instance
+
+Policy: users from other clusters in the federation are activated, users from outside the federation must be manually approved
+
+<pre>
+auto_setup_new_users: false
+new_users_are_active: false
+auto_activate_users_from: [home1]
+</pre>
+
+# Federated user arrives claiming to be from cluster 'home1'
+# API server authenticates user as being from cluster 'home1'
+# Because 'home1' is in @auto_activate_users_from@ the user is set up and activated.
+# User can immediately start using workbench.
+
+h4. Open instance
+
+Policy: anybody who shows up and signs the agreements is activated.
+
+<pre>
+auto_setup_new_users: true
+new_users_are_active: false
+</pre>
+
+# User is created and auto-setup. At this point, @is_active@ is false, but user has been added to "All users" group.
+# Workbench checks @is_invited@ and finds it is true, because the user is a member of "All users" group.
+# Workbench presents user with list of user agreements, user reads and clicks "sign" for each one.
+# Workbench tries to activate user.
+# User is activated.
+
+h4. Developer instance
+
+Policy: avoid wasting developer's time during development/testing
+
+<pre>
+auto_setup_new_users: true
+new_users_are_active: true
+</pre>
+
+# User is created, immediately auto-setup, and auto-activated.
+# User can immediately start using workbench.
--- /dev/null
+---
+layout: default
+navsection: admin
+title: Health checks
+...
+
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+Health check endpoints are found at @/_health/ping@ on many Arvados services. The purpose of the health check is to offer a simple method of determining if a service can be reached and allow the service to self-report any problems, suitable for integrating into operational alert systems.
+
+To access health check endpoints, services must be configured with a "management token":management-token.html .
+
+Health check endpoints return a JSON object with the field @health@. This has a value of either @OK@ or @ERROR@. On error, it may also include a field @error@ with additional information. Examples:
+
+<pre>
+{
+ "health": "OK"
+}
+</pre>
+
+<pre>
+{
+ "health": "ERROR"
+ "error": "Inverted polarity in the warp core"
+}
+</pre>
+
+h2. Healthcheck aggregator
+
+The service @arvados-health@ performs health checks on all configured services and returns a single value of @OK@ or @ERROR@ for the entire cluster. It exposes the endpoint @/_health/all@ .
+
+The healthcheck aggregator uses the @NodeProfile@ section of the cluster-wide @arvados.yml@ configuration file. Here is an example.
+
+<pre>
+Cluster:
+ # The cluster uuid prefix
+ zzzzz:
+ NodeProfile:
+ # For each node, the profile name corresponds to a
+ # locally-resolvable hostname, and describes which Arvados
+ # services are available on that machine.
+ api:
+ arvados-controller:
+ Listen: 8000
+ arvados-api-server:
+ Listen: 8001
+ manage:
+ arvados-node-manager:
+ Listen: 8002
+ workbench:
+ arvados-workbench:
+ Listen: 8003
+ arvados-ws:
+ Listen: 8004
+ keep:
+ keep-web:
+ Listen: 8005
+ keepproxy:
+ Listen: 8006
+ keep0:
+ keepstore:
+ Listen: 25701
+ keep1:
+ keepstore:
+ Listen: 25701
+</pre>
--- /dev/null
+---
+layout: default
+navsection: admin
+title: Management token
+...
+
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+To enable and collect health checks and metrics, services must be configured with a "management token".
+
+Services must have ManagementToken configured. This is used to authorize access monitoring endpoints. If ManagementToken is not configured, monitoring endpoints will return the error @404 disabled@.
+
+To access a monitoring endpoint, the requester must provide the HTTP header @Authorization: Bearer (ManagementToken)@.
+
+h2. API server
+
+Set @ManagementToken@ in the appropriate section of @application.yml@
+
+<pre>
+production:
+ # Token to be included in all healthcheck requests. Disabled by default.
+ # Server expects request header of the format "Authorization: Bearer xxx"
+ ManagementToken: xxx
+</pre>
+
+h2. Node Manager
+
+Set @port@ (the listen port) and @ManagementToken@ in the @Manage@ section of @node-manager.ini@.
+
+<pre>
+[Manage]
+# The management server responds to http://addr:port/status.json with
+# a snapshot of internal state.
+
+# Management server listening address (default 127.0.0.1)
+#address = 0.0.0.0
+
+# Management server port number (default -1, server is disabled)
+#port = 8989
+
+ManagementToken = xxx
+</pre>
+
+h2. Other services
+
+The following services also support monitoring. Set @ManagementToken@ in the respective yaml config file for each service.
+
+* keepstore
+* keep-web
+* keepproxy
+* arv-git-httpd
+* websockets
--- /dev/null
+---
+layout: default
+navsection: admin
+title: Metrics
+...
+
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+Metrics endpoints are found at @/status.json@ on many Arvados services. The purpose of metrics are to provide statistics about the operation of a service, suitable for diagnosing how well a service is performing under load.
+
+To access metrics endpoints, services must be configured with a "management token":management-token.html .
+
+h2. Keepstore
+
+h3. Root
+
+table(table table-bordered table-condensed).
+|_. Attribute|_. Type|_. Description|
+|Volumes| array of "volumeStatusEnt":#volumeStatusEnt ||
+|BufferPool| "PoolStatus":#PoolStatus ||
+|PullQueue| "WorkQueueStatus":#WorkQueueStatus ||
+|TrashQueue| "WorkQueueStatus":#WorkQueueStatus ||
+|RequestsCurrent| int ||
+|RequestsMax| int ||
+|Version| string ||
+
+h3(#volumeStatusEnt). volumeStatusEnt
+
+table(table table-bordered table-condensed).
+|_. Attribute|_. Type|_. Description|
+|Label| string||
+|Status| "VolumeStatus":#VolumeStatus ||
+|VolumeStats| "ioStats":#ioStats ||
+
+h3(#VolumeStatus). VolumeStatus
+
+table(table table-bordered table-condensed).
+|_. Attribute|_. Type|_. Description|
+|MountPoint| string||
+|DeviceNum| uint64||
+|BytesFree| uint64||
+|BytesUsed| uint64||
+
+h3(#ioStats). ioStats
+
+table(table table-bordered table-condensed).
+|_. Attribute|_. Type|_. Description|
+|Errors| uint64||
+|Ops| uint64||
+|CompareOps| uint64||
+|GetOps| uint64||
+|PutOps| uint64||
+|TouchOps| uint64||
+|InBytes| uint64||
+|OutBytes| uint64||
+
+h3(#PoolStatus). PoolStatus
+
+table(table table-bordered table-condensed).
+|_. Attribute|_. Type|_. Description|
+|BytesAllocatedCumulative| uint64||
+|BuffersMax| int||
+|BuffersInUse| int||
+
+h3(#WorkQueueStatus). WorkQueueStatus
+
+table(table table-bordered table-condensed).
+|_. Attribute|_. Type|_. Description|
+|InProgress| int||
+|Queued| int||
+
+h3. Example response
+
+<pre>
+{
+ "Volumes": [
+ {
+ "Label": "[UnixVolume /var/lib/arvados/keep0]",
+ "Status": {
+ "MountPoint": "/var/lib/arvados/keep0",
+ "DeviceNum": 65029,
+ "BytesFree": 222532972544,
+ "BytesUsed": 435456679936
+ },
+ "InternalStats": {
+ "Errors": 0,
+ "InBytes": 1111,
+ "OutBytes": 0,
+ "OpenOps": 1,
+ "StatOps": 4,
+ "FlockOps": 0,
+ "UtimesOps": 0,
+ "CreateOps": 0,
+ "RenameOps": 0,
+ "UnlinkOps": 0,
+ "ReaddirOps": 0
+ }
+ }
+ ],
+ "BufferPool": {
+ "BytesAllocatedCumulative": 67108864,
+ "BuffersMax": 20,
+ "BuffersInUse": 0
+ },
+ "PullQueue": {
+ "InProgress": 0,
+ "Queued": 0
+ },
+ "TrashQueue": {
+ "InProgress": 0,
+ "Queued": 0
+ },
+ "RequestsCurrent": 1,
+ "RequestsMax": 40,
+ "Version": "dev"
+}
+</pre>
+
+h2. Node manager
+
+The node manager status end point provides a snapshot of internal status at the time of the most recent wishlist update.
+
+table(table table-bordered table-condensed).
+|_. Attribute|_. Type|_. Description|
+|nodes_booting|int|Number of nodes in booting state|
+|nodes_unpaired|int|Number of nodes in unpaired state|
+|nodes_busy|int|Number of nodes in busy state|
+|nodes_idle|int|Number of nodes in idle state|
+|nodes_fail|int|Number of nodes in fail state|
+|nodes_down|int|Number of nodes in down state|
+|nodes_shutdown|int|Number of nodes in shutdown state|
+|nodes_wish|int|Number of nodes in the current wishlist|
+|node_quota|int|Current node count ceiling due to cloud quota limits|
+|config_max_nodes|int|Configured max node count|
+
+h3. Example
+
+<pre>
+{
+ "actor_exceptions": 0,
+ "idle_times": {
+ "compute1": 0,
+ "compute3": 0,
+ "compute2": 0,
+ "compute4": 0
+ },
+ "create_node_errors": 0,
+ "destroy_node_errors": 0,
+ "nodes_idle": 0,
+ "config_max_nodes": 8,
+ "list_nodes_errors": 0,
+ "node_quota": 8,
+ "Version": "1.1.4.20180719160944",
+ "nodes_wish": 0,
+ "nodes_unpaired": 0,
+ "nodes_busy": 4,
+ "boot_failures": 0
+}
+</pre>
--- /dev/null
+---
+layout: default
+navsection: admin
+title: Using AWS Spot instances
+...
+
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+This page describes how to set up the system to take advantage of "Amazon's EC2 spot instances":https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-spot-instances.html.
+
+h3. Nodemanager
+
+Nodemanager should have configured cloud sizes that include the @preemptible@ boolean parameter. For example, for every on-demand cloud node size, you could create a @.spot@ variant, like this:
+
+<pre>
+[Size m4.large]
+cores = 2
+scratch = 32000
+
+[Size m4.large.spot]
+cores = 2
+instance_type = m4.large
+preemptible = true
+scratch = 32000
+</pre>
+
+h3. Slurm dispatcher
+
+The @crunch-dispatch-slurm@ service needs a matching instance type configuration on @/etc/arvados/config.yml@, following the previous example:
+
+<pre>
+Clusters:
+ uuid_prefix:
+ InstanceTypes:
+ - Name: m4.large
+ VCPUs: 2
+ RAM: 7782000000
+ Scratch: 32000000000
+ Price: 0.1
+ - Name: m4.large.spot
+ Preemptible: true
+ VCPUs: 2
+ RAM: 7782000000
+ Scratch: 32000000000
+ Price: 0.1
+</pre>
+
+@InstanceType@ names should match those defined on nodemanager's config file because it's @crunch-dispatch-slurm@'s job to select the instance type and communicate the decision to @nodemanager@ via Slurm.
+
+h3. API Server
+
+Container requests will need the @preemptible@ scheduling parameter included, to make the dispatcher request a spot instance. The API Server configuration file includes an option that when active, will auto assign the @preemptible@ parameter to any new child container request if it doesn't have it already. To activate this feature, the following should be added to the @application.yml@ file:
+
+<pre>
+preemptible_instances: true
+</pre>
+
+With this configuration active, child container requests should include the @preemptible = false@ parameter at creation time to avoid being scheduled for spot instance usage.
+
+h3. AWS Permissions
+
+When requesting spot instances, Amazon's API may return an authorization error depending on how users and permissions are set on the account. If this is the case check nodemanager's log for:
+
+<pre>
+BaseHTTPError: AuthFailure.ServiceLinkedRoleCreationNotPermitted: The provided credentials do not have permission to create the service-linked role for EC2 Spot Instances.
+</pre>
+
+The account needs to have a service linked role created. This can be done by logging into the AWS account, go to _IAM Management_ → _Roles_ and create the @AWSServiceRoleForEC2Spot@ role by clicking on the @Create@ button, selecting @EC2@ service and @EC2 - Spot Instances@ use case.
+
+h3. Cost Tracking
+
+Amazon's Spot instances prices are declared at instance request time and defined by the maximum price that the user is willing to pay per hour. By default, this price is the same amount as the on-demand version of each instance type, and this setting is the one that nodemanager uses for now, as it doesn't include any pricing data to the spot instance request.
+
+The real price that a spot instance has at any point in time is discovered at the end of each usage hour, depending on instance demand. For this reason, AWS provides a data feed subscription to get hourly logs, as described on "Amazon's User Guide":https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-data-feeds.html.
\ No newline at end of file
--- /dev/null
+---
+layout: default
+navsection: admin
+title: Configuring storage classes
+...
+
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+Storage classes (alternately known as "storage tiers") allow you to control which volumes should be used to store particular collection data blocks. This can be used to implement data storage policies such as moving data to archival storage.
+
+The storage classes for each volume are set in the per-volume "keepstore configuration":{{site.baseurl}}/install/install-keepstore.html
+
+<pre>
+Volumes:
+ - ... Volume configuration ...
+ #
+ # If no storage classes are specified, will use [default]
+ #
+ StorageClasses: null
+
+ - ... Volume configuration ...
+ #
+ # Specify this volume is in the "archival" storage class.
+ #
+ StorageClasses: [archival]
+
+</pre>
+
+Names of storage classes are internal to the cluster and decided by the administrator. Aside from "default", Arvados currently does not define any standard storage class names.
+
+h3. Using storage classes
+
+"Discussed in the user guide":{{site.baseurl}}/user/topics/storage-classes.html
+
+h3. Storage management notes
+
+The "keep-balance":{{site.baseurl}}/install/install-keep-balance.html service is responsible for deciding which blocks should be placed on which keepstore volumes. As part of the rebalancing behavior, it will determine where a block should go in order to satisfy the desired storage classes, and issue pull requests to copy the block from its original volume to the desired volume. The block will subsequently be moved to trash on the original volume.
+
+If a block appears in multiple collections with different storage classes, the block will be stored in separate volumes for each storage class, even if that results in overreplication, unless there is a volume which has all the desired storage classes.
+
+If a collection has a desired storage class which is not available in any keepstore volume, the collection's blocks will remain in place, and an error will appear in the @keep-balance@ logs.
+
+This feature does not provide a hard guarantee on where data will be stored. Data may be written to default storage and moved to the desired storage class later. If controlling data locality is a hard requirement (such as legal restrictions on the location of data) we recommend setting up multiple Arvados clusters.
TODO: extract this information based on git commit messages and generate changelogs / release notes automatically.
{% endcomment %}
+h3. 2018-07-31: "#13497":https://dev.arvados.org/issues/13497 "db5107dca":https://dev.arvados.org/projects/arvados/repository/revisions/db5107dca adds a new system service, arvados-controller
+* "Install the controller":../install/install-controller.html after upgrading your system.
+* Verify your setup by confirming that API calls appear in the controller's logs (_e.g._, @journalctl -fu arvados-controller@) while loading a workbench page.
+
h3. 2018-04-05: v1.1.4 regression in arvados-cwl-runner for workflows that rely on implicit discovery of secondaryFiles
h4. Secondary files missing from toplevel workflow inputs
This bug will be fixed in an upcoming release of Arvados.
-h3. 2017-12-08: #11908 commit:8f987a9271 now requires minimum of Postgres 9.4 (previously 9.3)
+h3. 2017-12-08: "#11908":https://dev.arvados.org/issues/11908 "8f987a9271":https://dev.arvados.org/projects/arvados/repository/revisions/8f987a9271 now requires minimum of Postgres 9.4 (previously 9.3)
* Debian 8 (pg 9.4) and Debian 9 (pg 9.6) do not require an upgrade
* Ubuntu 16.04 (pg 9.5) does not require an upgrade
* Ubuntu 14.04 (pg 9.3) requires upgrade to Postgres 9.4: https://www.postgresql.org/download/linux/ubuntu/
*# Install the @rh-postgresql94@ backport package from either Software Collections: http://doc.arvados.org/install/install-postgresql.html or the Postgres developers: https://www.postgresql.org/download/linux/redhat/
*# Restore from the backup using @psql@
-h3. 2017-09-25: #12032 commit:68bdf4cbb now requires minimum of Postgres 9.3 (previously 9.1)
+h3. 2017-09-25: "#12032":https://dev.arvados.org/issues/12032 "68bdf4cbb":https://dev.arvados.org/projects/arvados/repository/revisions/68bdf4cbb now requires minimum of Postgres 9.3 (previously 9.1)
* Debian 8 (pg 9.4) and Debian 9 (pg 9.6) do not require an upgrade
* Ubuntu 16.04 (pg 9.5) does not require an upgrade
* Ubuntu 14.04 (pg 9.3) is compatible, however upgrading to Postgres 9.4 is recommended: https://www.postgresql.org/download/linux/ubuntu/
*# Install the @rh-postgresql94@ backport package from either Software Collections: http://doc.arvados.org/install/install-postgresql.html or the Postgres developers: https://www.postgresql.org/download/linux/redhat/
*# Restore from the backup using @psql@
-h3. 2017-06-30: #11807 commit:55aafbb converts old "jobs" database records from YAML to JSON, making the upgrade process slower than usual.
-* The migration can take some time if your database contains a substantial number of YAML-serialized rows (i.e., you installed Arvados before March 3, 2017 commit:660a614 and used the jobs/pipelines APIs). Otherwise, the upgrade will be no slower than usual.
+h3. 2017-06-30: "#11807":https://dev.arvados.org/issues/11807 "55aafbb":https://dev.arvados.org/projects/arvados/repository/revisions/55aafbb converts old "jobs" database records from YAML to JSON, making the upgrade process slower than usual.
+* The migration can take some time if your database contains a substantial number of YAML-serialized rows (i.e., you installed Arvados before March 3, 2017 "660a614":https://dev.arvados.org/projects/arvados/repository/revisions/660a614 and used the jobs/pipelines APIs). Otherwise, the upgrade will be no slower than usual.
* The conversion runs as a database migration, i.e., during the deb/rpm package upgrade process, while your API server is unavailable.
* Expect it to take about 1 minute per 20K jobs that have ever been created/run.
-h3. 2017-06-05: #9005 commit:cb230b0 reduces service discovery overhead in keep-web requests.
+h3. 2017-06-05: "#9005":https://dev.arvados.org/issues/9005 "cb230b0":https://dev.arvados.org/projects/arvados/repository/revisions/cb230b0 reduces service discovery overhead in keep-web requests.
* When upgrading keep-web _or keepproxy_ to/past this version, make sure to update API server as well. Otherwise, a bad token in a request can cause keep-web to fail future requests until either keep-web restarts or API server gets upgraded.
-h3. 2017-04-12: #11349 commit:2c094e2 adds a "management" http server to nodemanager.
+h3. 2017-04-12: "#11349":https://dev.arvados.org/issues/11349 "2c094e2":https://dev.arvados.org/projects/arvados/repository/revisions/2c094e2 adds a "management" http server to nodemanager.
* To enable it, add to your configuration file: <pre>[Manage]
address = 127.0.0.1
port = 8989</pre> (see example configuration files in source:services/nodemanager/doc or https://doc.arvados.org/install/install-nodemanager.html for more info)
* The server responds to @http://{address}:{port}/status.json@ with a summary of how many nodes are in each state (booting, busy, shutdown, etc.)
-h3. 2017-03-23: #10766 commit:e8cc0d7 replaces puma with arvados-ws as the recommended websocket server.
+h3. 2017-03-23: "#10766":https://dev.arvados.org/issues/10766 "e8cc0d7":https://dev.arvados.org/projects/arvados/repository/revisions/e8cc0d7 replaces puma with arvados-ws as the recommended websocket server.
* See http://doc.arvados.org/install/install-ws.html for install/upgrade instructions.
* Remove the old puma server after the upgrade is complete. Example, with runit: <pre>
$ sudo sv down /etc/sv/puma
$ systemctl stop puma
</pre>
-h3. 2017-03-06: #11168 commit:660a614 uses JSON instead of YAML to encode hashes and arrays in the database.
+h3. 2017-03-06: "#11168":https://dev.arvados.org/issues/11168 "660a614":https://dev.arvados.org/projects/arvados/repository/revisions/660a614 uses JSON instead of YAML to encode hashes and arrays in the database.
* Aside from a slight performance improvement, this should have no externally visible effect.
* Downgrading past this version is not supported, and is likely to cause errors. If this happens, the solution is to upgrade past this version.
* After upgrading, make sure to restart puma and crunch-dispatch-* processes.
-h3. 2017-02-03: #10969 commit:74a9dec introduces a Docker image format compatibility check: the @arv keep docker@ command prevents users from inadvertently saving docker images that compute nodes won't be able to run.
+h3. 2017-02-03: "#10969":https://dev.arvados.org/issues/10969 "74a9dec":https://dev.arvados.org/projects/arvados/repository/revisions/74a9dec introduces a Docker image format compatibility check: the @arv keep docker@ command prevents users from inadvertently saving docker images that compute nodes won't be able to run.
* If your compute nodes run a version of *docker older than 1.10* you must override the default by adding to your API server configuration (@/etc/arvados/api/application.yml@): <pre><code class="yaml">docker_image_formats: ["v1"]</code></pre>
-* Refer to the comments above @docker_image_formats@ in @/var/www/arvados-api/current/config/application.default.yml@ or source:services/api/config/application.default.yml or issue #10969 for more detail.
+* Refer to the comments above @docker_image_formats@ in @/var/www/arvados-api/current/config/application.default.yml@ or source:services/api/config/application.default.yml or issue "#10969":https://dev.arvados.org/issues/10969 for more detail.
* *NOTE:* This does *not* include any support for migrating existing Docker images from v1 to v2 format. This will come later: for now, sites running Docker 1.9 or earlier should still *avoid upgrading Docker further than 1.9.*
-h3. 2016-09-27: several Debian and RPM packages -- keep-balance (commit:d9eec0b), keep-web (commit:3399e63), keepproxy (commit:6de67b6), and arvados-git-httpd (commit:9e27ddf) -- now enable their respective components using systemd. These components prefer YAML configuration files over command line flags (commit:3bbe1cd).
+h3. 2016-09-27: several Debian and RPM packages -- keep-balance ("d9eec0b":https://dev.arvados.org/projects/arvados/repository/revisions/d9eec0b), keep-web ("3399e63":https://dev.arvados.org/projects/arvados/repository/revisions/3399e63), keepproxy ("6de67b6":https://dev.arvados.org/projects/arvados/repository/revisions/6de67b6), and arvados-git-httpd ("9e27ddf":https://dev.arvados.org/projects/arvados/repository/revisions/9e27ddf) -- now enable their respective components using systemd. These components prefer YAML configuration files over command line flags ("3bbe1cd":https://dev.arvados.org/projects/arvados/repository/revisions/3bbe1cd).
* On Debian-based systems using systemd, services are enabled automatically when packages are installed.
* On RedHat-based systems using systemd, unit files are installed but services must be enabled explicitly: e.g., <code>"sudo systemctl enable keep-web; sudo systemctl start keep-web"</code>.
* The new systemd-supervised services will not start up successfully until configuration files are installed in /etc/arvados/: e.g., <code>"Sep 26 18:23:55 62751f5bb946 keep-web[74]: 2016/09/26 18:23:55 open /etc/arvados/keep-web/keep-web.yml: no such file or directory"</code>
** keepproxy - /etc/arvados/keepproxy/keepproxy.yml
** arvados-git-httpd - /etc/arvados/arv-git-httpd/arv-git-httpd.yml
-h3. 2016-05-31: commit:ae72b172c8 and commit:3aae316c25 install Python modules and scripts to different locations on the filesystem.
-* Previous packages installed these files to the distribution's preferred path under @/usr/local@ (or the equivalent location in a Software Collection). Now they get installed to a path under @/usr@. This improves compatibility with other Python packages provided by the distribution. See #9242 for more background.
+h3. 2016-05-31: "ae72b172c8":https://dev.arvados.org/projects/arvados/repository/revisions/ae72b172c8 and "3aae316c25":https://dev.arvados.org/projects/arvados/repository/revisions/3aae316c25 install Python modules and scripts to different locations on the filesystem.
+* Previous packages installed these files to the distribution's preferred path under @/usr/local@ (or the equivalent location in a Software Collection). Now they get installed to a path under @/usr@. This improves compatibility with other Python packages provided by the distribution. See "#9242":https://dev.arvados.org/issues/9242 for more background.
* If you simply import Python modules from scripts, or call Python tools relying on $PATH, you don't need to make any changes. If you have hardcoded full paths to some of these files (e.g., in symbolic links or configuration files), you will need to update those paths after this upgrade.
-h3. 2016-04-25: commit:eebcb5e requires the crunchrunner package to be installed on compute nodes and shell nodes in order to run CWL workflows.
+h3. 2016-04-25: "eebcb5e":https://dev.arvados.org/projects/arvados/repository/revisions/eebcb5e requires the crunchrunner package to be installed on compute nodes and shell nodes in order to run CWL workflows.
* On each Debian-based compute node and shell node, run: @sudo apt-get install crunchrunner@
* On each Red Hat-based compute node and shell node, run: @sudo yum install crunchrunner@
-h3. 2016-04-21: commit:3c88abd changes the Keep permission signature algorithm.
+h3. 2016-04-21: "3c88abd":https://dev.arvados.org/projects/arvados/repository/revisions/3c88abd changes the Keep permission signature algorithm.
* All software components that generate signatures must be upgraded together. These are: keepstore, API server, keep-block-check, and keep-rsync. For example, if keepstore < 0.1.20160421183420 but API server >= 0.1.20160421183420, clients will not be able to read or write data in Keep.
* Jobs and client operations that are in progress during the upgrade (including arv-put's "resume cache") will fail.
-h3. 2015-01-05: commit:e1276d6e disables Workbench's "Getting Started" popup by default.
+h3. 2015-01-05: "e1276d6e":https://dev.arvados.org/projects/arvados/repository/revisions/e1276d6e disables Workbench's "Getting Started" popup by default.
* If you want new users to continue seeing this popup, set @enable_getting_started_popup: true@ in Workbench's @application.yml@ configuration.
-h3. 2015-12-03: commit:5590c9ac makes a Keep-backed writable scratch directory available in crunch jobs (see #7751)
-* All compute nodes must be upgraded to arvados-fuse >= 0.1.2015112518060 because crunch-job uses some new arv-mount flags (--mount-tmp, --mount-by-pdh) introduced in merge commit:346a558
+h3. 2015-12-03: "5590c9ac":https://dev.arvados.org/projects/arvados/repository/revisions/5590c9ac makes a Keep-backed writable scratch directory available in crunch jobs (see "#7751":https://dev.arvados.org/issues/7751)
+* All compute nodes must be upgraded to arvados-fuse >= 0.1.2015112518060 because crunch-job uses some new arv-mount flags (--mount-tmp, --mount-by-pdh) introduced in merge "346a558":https://dev.arvados.org/projects/arvados/repository/revisions/346a558
* Jobs will fail if the API server (in particular crunch-job from the arvados-cli gem) is upgraded without upgrading arvados-fuse on compute nodes.
-h3. 2015-11-11: commit:1e2ace5 changes recommended config for keep-web (see #5824)
+h3. 2015-11-11: "1e2ace5":https://dev.arvados.org/projects/arvados/repository/revisions/1e2ace5 changes recommended config for keep-web (see "#5824":https://dev.arvados.org/issues/5824)
* proxy/dns/ssl config should be updated to route "https://download.uuid_prefix.arvadosapi.com/" requests to keep-web (alongside the existing "collections" routing)
* keep-web command line adds @-attachment-only-host download.uuid_prefix.arvadosapi.com@
* Workbench config adds @keep_web_download_url@
* More info on the (still beta/non-TOC-linked) "keep-web doc page":http://doc.arvados.org/install/install-keep-web.html
-h3. 2015-11-04: commit:1d1c6de removes stopped containers (see #7444)
+h3. 2015-11-04: "1d1c6de":https://dev.arvados.org/projects/arvados/repository/revisions/1d1c6de removes stopped containers (see "#7444":https://dev.arvados.org/issues/7444)
* arvados-docker-cleaner removes _all_ docker containers as soon as they exit, effectively making @docker run@ default to @--rm@. If you run arvados-docker-cleaner on a host that does anything other than run crunch-jobs, and you still want to be able to use @docker start@, read the "new doc page":http://doc.arvados.org/install/install-compute-node.html to learn how to turn this off before upgrading.
-h3. 2015-11-04: commit:21006cf adds a keep-web service (see #5824)
+h3. 2015-11-04: "21006cf":https://dev.arvados.org/projects/arvados/repository/revisions/21006cf adds a keep-web service (see "#5824":https://dev.arvados.org/issues/5824)
* Nothing relies on it yet, but early adopters can install it now by following http://doc.arvados.org/install/install-keep-web.html (it is not yet linked in the TOC).
!(full-width){{site.baseurl}}/images/Crunch_dispatch.svg!
+h2(#RAM). Understanding RAM requests for containers
+
+The @runtime_constraints@ section of a container specifies working RAM (@ram@) and Keep cache (@keep_cache_ram@). If not specified, containers get a default Keep cache (@container_default_keep_cache_ram@, default 256 MiB). The total RAM requested for a container is the sum of working RAM, Keep cache, and an additional RAM reservation configured by the admin (@ReserveExtraRAM@ in the dispatcher configuration, default zero).
+
+The total RAM request is used to schedule containers onto compute nodes. RAM allocation limits are enforced using kernel controls such as cgroups. A container which requests 1 GiB RAM will only be permitted to allocate up to 1 GiB of RAM, even if scheduled on a 4 GiB node. On HPC systems, a multi-core node may run multiple containers at a time.
+
+When running on the cloud, the memory request (along with CPU and disk) is used to select (and possibly boot) an instance type with adequate resources to run the container. Instance type RAM is derated 5% from the published specification to accomodate virtual machine, kernel and system services overhead.
+
+h3. Calculate minimum instance type RAM for a container
+
+ (RAM request + Keep cache + ReserveExtraRAM) * (100/95)
+
+For example, for a 3 GiB request, default Keep cache, and no extra RAM reserved:
+
+ (3072 + 256) * 1.0526 = 3494 MiB
+
+To run this container, the instance type must have a published RAM size of at least 3494 MiB.
+
+h3. Calculate the maximum requestable RAM for an instance type
+
+ (Instance type RAM * (95/100)) - Keep cache - ReserveExtraRAM
+
+For example, for a 3.75 GiB node, default Keep cache, and no extra RAM reserved:
+
+ (3840 * 0.95) - 256 = 3392 MiB
+
+To run on this instance type, the container can request at most 3392 MiB of working RAM.
+
h2. Job API (deprecated)
# To submit work, create a "job":{{site.baseurl}}/api/methods/jobs.html . If the same job has been submitted in the past, it will return an existing job in @Completed@ state.
|@is_a@|string|Arvados object type|@["head_uuid","is_a","arvados#collection"]@|
|@exists@|string|Test if a subproperty is present.|@["properties","exists","my_subproperty"]@|
-h4. Filtering on subproperties
+h4(#subpropertyfilters). Filtering on subproperties
Some record type have an additional @properties@ attribute that allows recording and filtering on additional key-value pairs. To filter on a subproperty, the value in the @attribute@ position has the form @properties.user_property@. You may also use JSON-LD / RDF style URIs for property keys by enclosing them in @<...>@ for example @properties.<http://example.com/user_property>@. Alternately you may also provide a JSON-LD "@context" field, however at this time JSON-LD contexts are not interpreted by Arvados.
|_. Attribute|_. Type|_. Description|_. Example|
|name|string|||
|description|text|||
+|properties|hash|User-defined metadata, may be used in queries using "subproperty filters":{{site.baseurl}}/api/methods.html#subpropertyfilters ||
|portable_data_hash|string|The MD5 sum of the manifest text stripped of block hints other than the size hint.||
|manifest_text|text|||
|replication_desired|number|Minimum storage replication level desired for each data block referenced by this collection. A value of @null@ signifies that the site default replication level (typically 2) is desired.|@2@|
|_. Attribute|_. Type|_. Description|_. Notes|
|name|string|The name of the container_request.||
|description|string|The description of the container_request.||
-|properties|hash|Client-defined structured data that does not affect how the container is run.||
+|properties|hash|User-defined metadata that does not affect how the container is run. May be used in queries using "subproperty filters":{{site.baseurl}}/api/methods.html#subpropertyfilters||
|state|string|The allowed states are "Uncommitted", "Committed", and "Final".|Once a request is Committed, the only attributes that can be modified are priority, container_uuid, and container_count_max. A request in the "Final" state cannot have any of its functional parts modified (i.e., only name, description, and properties fields can be modified).|
|requesting_container_uuid|string|The uuid of the parent container that created this container_request, if any. Represents a process tree.|The priority of this container_request is inherited from the parent container, if the parent container is cancelled, this container_request will be cancelled as well.|
|container_uuid|string|The uuid of the container that satisfies this container_request. The system may return a preexisting Container that matches the container request criteria. See "Container reuse":#container_reuse for more details.|Container reuse is the default behavior, but may be disabled with @use_existing: false@ to always create a new container.|
|group_class|string|Type of group. This does not affect behavior, but determines how the group is presented in the user interface. For example, @project@ indicates that the group should be displayed by Workbench and arv-mount as a project for organizing and naming objects.|@"project"@
null|
|description|text|||
+|properties|hash|User-defined metadata, may be used in queries using "subproperty filters":{{site.baseurl}}/api/methods.html#subpropertyfilters ||
|writable_by|array|List of UUID strings identifying Users and other Groups that have write permission for this Group. Only users who are allowed to administer the Group will receive a full list. Other users will receive a partial list that includes the Group's owner_uuid and (if applicable) their own user UUID.||
|trash_at|datetime|If @trash_at@ is non-null and in the past, this group and all objects directly or indirectly owned by the group will be hidden from API calls. May be untrashed.||
|delete_at|datetime|If @delete_at@ is non-null and in the past, the group and all objects directly or indirectly owned by the group may be permanently deleted.||
|_. Argument |_. Type |_. Description |_. Location |_. Example |
{background:#ccffcc}.|uuid|string|The UUID of the group in question.|path||
|limit|integer (default 100)|Maximum number of items to return.|query||
-|order|string|Order in which to return matching items. Sort within a resource type by prefixing the attribute with the resource name and a dot.|query|@"collections.modified_at desc"@|
+|order|array|Attributes to use as sort keys to determine the order resources are returned, each optionally followed by @asc@ or @desc@ to indicate ascending or descending order. Sort within a resource type by prefixing the attribute with the resource name and a period.|query|@["collections.modified_at desc"]@|
|filters|array|Conditions for filtering items.|query|@[["uuid", "is_a", "arvados#job"]]@|
|recursive|boolean (default false)|Include items owned by subprojects.|query|@true@|
|tail_uuid|string|The origin or actor in the description or action (may be null).|
|link_class|string|Type of link|
|name|string|Primary value of the link.|
-|properties|hash|Additional information, expressed as a key→value hash. Key: string. Value: string, number, array, or hash.|
+|properties|hash|Additional information, expressed as a key→value hash. Key: string. Value: string, number, array, or hash. May be used in queries using "subproperty filters":{{site.baseurl}}/api/methods.html#subpropertyfilters|
h2. Link classes
|job_uuid|string|The UUID of the job that this node is assigned to work on. If you do not have permission to read the job, this will be null.||
|first_ping_at|datetime|||
|last_ping_at|datetime|||
-|info|hash|||
+|info|hash|Sensitive information about the node (only visible to admin) such as 'ping_secret' and 'ec2_instance_id'. May be used in queries using "subproperty filters":{{site.baseurl}}/api/methods.html#subpropertyfilters||
+|properties|hash|Public information about the node, such as 'total_cpu_cores', 'total_ram_mb', and 'total_scratch_mb'. May be used in queries using "subproperty filters":{{site.baseurl}}/api/methods.html#subpropertyfilters||
h2. Methods
The "browser authentication process is documented in detail on the Arvados wiki.":https://dev.arvados.org/projects/arvados/wiki/Workbench_authentication_process
+h2. User activation
+
+"Creation and activation of new users is described here.":{{site.baseurl}}/admin/activation.html
+
h2. Creating tokens via the API
The browser login method above issues a new token. Using that token, it is possible to make API calls to create additional tokens. To do so, use the @create@ method of the "API client authorizations":{{site.baseurl}}/api/methods/api_client_authorizations.html resource.
.userinput {
color: #d14;
}
+
+table.CodeRay {
+ margin-left: 3em;
+ width: calc(100% - 6em);
+}
+
+td.line-numbers {
+ width: 2em;
+}
img.full-width {
width: 100%
}
+
+img.screenshot {
+ max-width: calc(100% - 2em);
+ border: 3px;
+ border-style: solid;
+ margin-left: 2em;
+ margin-bottom: 2em;
+}
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Arvados on Kubernetes - Google Kubernetes Engine
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+This page documents the setup of the prerequisites to run the "Arvados on Kubernetes":/install/arvados-on-kubernetes.html @Helm@ chart on @Google Kubernetes Engine@ (GKE).
+
+h3. Install tooling
+
+Install @gcloud@:
+
+* Follow the instructions at "https://cloud.google.com/sdk/downloads":https://cloud.google.com/sdk/downloads
+
+Install @kubectl@:
+
+<pre>
+$ gcloud components install kubectl
+</pre>
+
+Install @helm@:
+
+* Follow the instructions at "https://docs.helm.sh/using_helm/#installing-helm":https://docs.helm.sh/using_helm/#installing-helm
+
+h3. Boot the GKE cluster
+
+This can be done via the "cloud console":https://console.cloud.google.com/kubernetes/ or via the command line:
+
+<pre>
+$ gcloud container clusters create <CLUSTERNAME> --zone us-central1-a --machine-type n1-standard-2 --cluster-version 1.10
+</pre>
+
+It takes a few minutes for the cluster to be initialized.
+
+h3. Reserve a static IP
+
+Reserve a "static IP":https://console.cloud.google.com/networking/addresses in GCE. Make sure the IP is in the same region as your GKE cluster, and is of the "Regional" type.
+
+h3. Connect to the GKE cluster.
+
+Via the web:
+* Click the "Connect" button next to your "GKE cluster"https://console.cloud.google.com/kubernetes/.
+* Execute the "Command-line access" command on your development machine.
+
+Alternatively, use this command:
+
+<pre>
+$ gcloud container clusters get-credentials <CLUSTERNAME> --zone us-central1-a --project <YOUR-PROJECT>
+</pre>
+
+Test the connection:
+
+<pre>
+$ kubectl get nodes
+</pre>
+
+Now proceed to the "Initialize helm on the Kubernetes cluster":/install/arvados-on-kubernetes.html#helm section.
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Arvados on Kubernetes - Minikube
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+This page documents the setup of the prerequisites to run the "Arvados on Kubernetes":/install/arvados-on-kubernetes.html @Helm@ chart on @Minikube@.
+
+h3. Install tooling
+
+Install @kubectl@:
+
+* Follow the instructions at "https://kubernetes.io/docs/tasks/tools/install-kubectl/":https://kubernetes.io/docs/tasks/tools/install-kubectl/
+
+Install @helm@:
+
+* Follow the instructions at "https://docs.helm.sh/using_helm/#installing-helm":https://docs.helm.sh/using_helm/#installing-helm
+
+h3. Install Minikube
+
+Follow the instructions at "https://kubernetes.io/docs/setup/minikube/":https://kubernetes.io/docs/setup/minikube/
+
+Test the connection:
+
+<pre>
+$ kubectl get nodes
+</pre>
+
+Now proceed to the "Initialize helm on the Kubernetes cluster":/install/arvados-on-kubernetes.html#helm section.
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Arvados on Kubernetes
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+Arvados on Kubernetes is implemented as a Helm Chart.
+
+{% include 'notebox_begin_warning' %}
+This Helm Chart does not retain any state after it is deleted. An Arvados cluster created with this Helm Chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
+{% include 'notebox_end' %}
+
+h2(#overview). Overview
+
+This Helm Chart provides a basic, small Arvados cluster.
+
+Current limitations, to be addressed in the future:
+
+* An Arvados cluster created with this Helm Chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down.
+* No dynamic scaling of compute nodes (but you can adjust @values.yaml@ and "reload the Helm Chart":#reload
+* All compute nodes are the same size
+* Compute nodes have no cpu/memory/disk constraints yet
+* No git server
+
+h2. Requirements
+
+* Kubernetes 1.10+ cluster with at least 3 nodes, 2 or more cores per node
+* @kubectl@ and @helm@ installed locally, and able to connect to your Kubernetes cluster
+
+If you do not have a Kubernetes cluster already set up, you can use "Google Kubernetes Engine":/install/arvados-on-kubernetes-GKE.html for multi-node development and testing or "another Kubernetes solution":https://kubernetes.io/docs/setup/pick-right-solution/. Minikube is not supported yet.
+
+h2(#helm). Initialize helm on the Kubernetes cluster
+
+If you already have helm running on the Kubernetes cluster, proceed directly to "Start the Arvados cluster":#Start below.
+
+<pre>
+$ helm init
+$ kubectl create serviceaccount --namespace kube-system tiller
+$ kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller
+$ kubectl patch deploy --namespace kube-system tiller-deploy -p '{"spec":{"template":{"spec":{"serviceAccount":"tiller"}}}}'
+</pre>
+
+Test @helm@ by running
+
+<pre>
+$ helm ls
+</pre>
+
+There should be no errors. The command will return nothing.
+
+h2(#git). Clone the repository
+
+Clone the repository and nagivate to the @arvados-kubernetes/charts/arvados@ directory:
+
+<pre>
+$ git clone https://github.com/curoverse/arvados-kubernetes.git
+$ cd arvados-kubernetes/charts/arvados
+</pre>
+
+h2(#Start). Start the Arvados cluster
+
+Next, determine the IP address that the Arvados cluster will use to expose its API, Workbench, etc. If you want this Arvados cluster to be reachable from places other than the local machine, the IP address will need to be routable as appropriate.
+
+<pre>
+$ ./cert-gen.sh <IP ADDRESS>
+</pre>
+
+The @values.yaml@ file contains a number of variables that can be modified. At a minimum, review and/or modify the values for
+
+<pre>
+ adminUserEmail
+ adminUserPassword
+ superUserSecret
+ anonymousUserSecret
+</pre>
+
+Now start the Arvados cluster:
+
+<pre>
+$ helm install --name arvados . --set externalIP=<IP ADDRESS>
+</pre>
+
+At this point, you can use kubectl to see the Arvados cluster boot:
+
+<pre>
+$ kubectl get pods
+$ kubectl get svc
+</pre>
+
+After a few minutes, you can access Arvados Workbench at the IP address specified
+
+* https://<IP ADDRESS>
+
+with the username and password specified in the @values.yaml@ file.
+
+Alternatively, use the Arvados cli tools or SDKs:
+
+Set the environment variables:
+
+<pre>
+$ export ARVADOS_API_TOKEN=<superUserSecret from values.yaml>
+$ export ARVADOS_API_HOST=<STATIC IP>:444
+$ export ARVADOS_API_HOST_INSECURE=true
+</pre>
+
+Test access with:
+
+<pre>
+$ arv user current
+</pre>
+
+h2(#reload). Reload
+
+If you make changes to the Helm Chart (e.g. to @values.yaml@), you can reload Arvados with
+
+<pre>
+$ helm upgrade arvados .
+</pre>
+
+h2. Shut down
+
+{% include 'notebox_begin_warning' %}
+This Helm Chart does not retain any state after it is deleted. An Arvados cluster created with this Helm Chart is entirely ephemeral, and <strong>all data stored on the Arvados cluster will be deleted</strong> when it is shut down. This will be fixed in a future version.
+{% include 'notebox_end' %}
+
+<pre>
+$ helm del arvados --purge
+</pre>
---
layout: default
navsection: admin
-title: User management
+title: User management at the CLI
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-As an alternative to local and network-attached POSIX filesystems, Keepstore can store data in an Azure Storage container.
+Keepstore can store data in one or more Azure Storage containers.
-h2. Create a container
+h2. Set up VMs and Storage Accounts
-Normally, all keepstore services are configured to share a single Azure Storage container.
+Before starting the configuration of individual keepstore servers is good to have an idea of the keepstores servers' final layout. One key decision is the amount of servers and type of VM to run. Azure may change over time the bandwith capacity of each type. After conducting some empirical saturation tests, the conclusion was that the bandwith is proportional to the amount of cores with some exceptions. As a rule of thumb, is better to invest resources in more cores instead of memory or IOps.
+
+Another decision is how many VMs should be running keepstore. For example there could be 8 VMs with one core each or one machine with 8 cores. Or anything in between. Assuming is the same cost for Cloud resources, there is always the benefit of distributing the risk of faulty VMs. The recommendation is to start with 2 VMs and expand in pairs. Having a minimum of 2 cores each. The total amount of VMs will be a function of the budget and the pipeline traffic to avoid saturation during periods of high usage. Standard D v3 family is a balanced choice, making Standard_D2_v3 the 2-core option
+
+There are many options for storage accounts. You can read details from Azure on their documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-introduction. The type of storage and access tier will be a function of the budget and desired responsiveness. A balanced option is to have General-purpose Standard Storage account and use Blob storage, hot access tiers.
+
+Keepstore can be configure to reflect the level of underlaying redundancy the storage will have. This is call data replication option. For example LRS (Locally Redundant Storage) saves 3 copies of the data. There desired redundancy can be chosen at the keepstore layer or at the Storage Accunt layer. The decision where the redundancy will be done and the type of Storage Account data replication (LRS, ZRS, GRS and RA-GRS) has trade-offs. Please read more on https://docs.microsoft.com/en-us/azure/storage/common/storage-redundancy and decide what is best for your needs.
+
+h2. Create a storage container
Using the Azure web portal or command line tool, create or choose a storage account with a suitable redundancy profile and availability region. Use the storage account keys to create a new container.
</code></pre>
</notextile>
+Note that Keepstore services may be configued to use multiple Azure Storage accounts and multiple containers within a storage account.
+
h2. Configure keepstore
Copy the primary storage account key to a file where it will be accessible to keepstore at startup time.
<notextile>
-<pre><code>~$ <span class="userinput">sudo sh -c 'cat >/etc/sv/keepstore/exampleStorageAccountName.key <<EOF'
+<pre><code>~$ <span class="userinput">sudo sh -c 'cat >/etc/arvados/keepstore/azure_storage_account_key.txt <<EOF'
zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz==
EOF</span>
-~$ <span class="userinput">sudo chmod 0400 /etc/sv/keepstore/exampleStorageAccountName.key</span>
+~$ <span class="userinput">sudo chmod 0400 /etc/arvados/keepstore/azure_storage_account_key.txt</span>
</code></pre>
</notextile>
-In your keepstore startup script, instead of specifying a local storage using @-volume /path@ or discovering mount points automatically, use @-azure-*@ arguments to specify the storage container:
+Next, edit the @Volumes@ section of the @keepstore.yml@ config file:
-<notextile>
-<pre><code>#!/bin/sh
+<pre>
+Volumes:
+- # The volume type, this indicates Azure blob storage
+ Type: Azure
-exec 2>&1
-exec keepstore \
- -azure-storage-account-key-file <span class="userinput">/etc/sv/keepstore/exampleStorageAccountName.key</span> \
- -azure-storage-account-name <span class="userinput">exampleStorageAccountName</span> \
- -azure-storage-container-volume <span class="userinput">exampleContainerName</span>
-</code></pre>
-</notextile>
+ # How much replication is performed by the underlying container.
+ # This is used to inform replication decisions at the Keep layer.
+ AzureReplication: 3
-Start (or restart) keepstore, and check its log file to confirm it is using the new configuration.
+ # The storage container to use for the backing store.
+ ContainerName: exampleContainerName
-<notextile>
-<pre><code>2015/10/26 21:06:24 Using volume azure-storage-container:"exampleContainerName" (writable=true)
-</code></pre>
-</notextile>
+ # If true, do not accept write or trash operations, only reads.
+ ReadOnly: false
+
+ # Amount of time to wait for a response before failing the request
+ RequestTimeout: 2m0s
+
+ # The storage account name, used for authentication
+ StorageAccountName: exampleStorageAccountName
+
+ # The storage account secret key, used for authentication
+ StorageAccountKeyFile: /etc/arvados/keepstore/azure_storage_account_key.txt
+
+ # The cloud environment to use. If blank, use the default cloud
+ # environment. See below for an example of an alternate cloud environment.
+ StorageBaseURL: ""
+
+ # Storage classes to associate with this volume. See "Storage
+ # classes" in the "Admin" section of doc.arvados.org.
+ StorageClasses: null
+
+- # Example configuration to use Azure China.
+ #
+ # The alternate cloud environment to use.
+ # Note that cloud environments are different from regions. A
+ # cloud environment is an entirely separate instance of Azure with
+ # separate accounts, requiring separate credentials.
+ #
+ StorageBaseURL: core.chinacloudapi.cn
+ StorageAccountKeyFile: /etc/arvados/keepstore/azure_cn_storage_account_key.txt
+ StorageAccountName: cn-account-name
+ ContainerName: exampleChinaContainerName
+
+ # The rest are the same as above
+ Type: Azure
+ AzureReplication: 3
+ ReadOnly: false
+ RequestTimeout: 10m0s
+ StorageClasses: null
+</pre>
+
+Start (or restart) keepstore, and check its log file to confirm it is using the new configuration.
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Filesystem storage
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+Keepstore can store data in local and network-attached POSIX filesystems.
+
+h2. Setting up filesystem mounts
+
+Volumes are configured in the @Volumes@ section of the configuration file. You may provide multiple volumes for a single keepstore process to manage multiple disks. Keepstore distributes blocks among volumes in round-robin fashion.
+
+<pre>
+Volumes:
+- # The volume type, indicates this is a filesystem directory.
+ Type: Directory
+
+ # The directory that will be used as the backing store.
+ Root: /mnt/local-disk
+
+ # How much replication is performed by the underlying filesystem.
+ # (for example, a network filesystem may provide its own replication).
+ # This is used to inform replication decisions at the Keep layer.
+ DirectoryReplication: 1
+
+ # If true, do not accept write or trash operations, only reads.
+ ReadOnly: false
+
+ # When true, read and write operations (for whole 64MiB blocks) on
+ # an individual volume will queued and issued serially. When
+ # false, read and write operations will be issued concurrently.
+ #
+ # May improve throughput if you experience contention when there are
+ # multiple requests to the same volume.
+ #
+ # When using SSDs, RAID, or a parallel network filesystem, you probably
+ # don't want this.
+ Serialize: false
+
+ # Storage classes to associate with this volume. See "Storage
+ # classes" in the "Admin" section of doc.arvados.org.
+ StorageClasses: null
+
+ # Example of a second volume section
+- DirectoryReplication: 2
+ ReadOnly: false
+ Root: /mnt/network-disk
+ Serialize: false
+ StorageClasses: null
+ Type: Directory
+</pre>
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Configure S3 object storage
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+Keepstore can store data in object storage compatible with the S3 API, such as Amazon S3, Google Cloud Storage, or Ceph RADOS.
+
+h2. Configure keepstore
+
+Copy the "access key" and "secret key" to files where they will be accessible to keepstore at startup time.
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo sh -c 'cat >/etc/arvados/keepstore/aws_s3_access_key.txt <<EOF'
+zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz==
+EOF</span>
+~$ <span class="userinput">sudo chmod 0400 /etc/arvados/keepstore/aws_s3_access_key.txt</span>
+</code></pre>
+</notextile>
+
+Next, edit the @Volumes@ section of the @keepstore.yml@ config file.
+
+h3. Example config for Amazon S3
+
+<pre>
+Volumes:
+- # The volume type, this indicates object storage compatible with the S3 API
+ Type: S3
+
+ # Storage provider. If blank, uses Amazon S3 by default.
+ # See below for example alternate configuration for Google cloud
+ # storage.
+ Endpoint: ""
+
+ # The bucket to use for the backing store.
+ Bucket: example-bucket-name
+
+ # The region where the bucket is located.
+ Region: us-east-1
+
+ # The credentials to use to access the bucket.
+ AccessKeyFile: /etc/arvados/keepstore/aws_s3_access_key.txt
+ SecretKeyFile: /etc/arvados/keepstore/aws_s3_secret_key.txt
+
+ # Maximum time to wait making the initial connection to the backend before
+ # failing the request.
+ ConnectTimeout: 1m0s
+
+ # Page size for s3 "list bucket contents" requests
+ IndexPageSize: 1000
+
+ # True if the region requires a LocationConstraint declaration
+ LocationConstraint: false
+
+ # Maximum eventual consistency latency
+ RaceWindow: 24h0m0s
+
+ # If true, do not accept write or trash operations, only reads.
+ ReadOnly: false
+
+ # Maximum time to wait for a complete response from the backend before
+ # failing the request.
+ ReadTimeout: 2m0s
+
+ # How much replication is performed by the underlying bucket.
+ # This is used to inform replication decisions at the Keep layer.
+ S3Replication: 2
+
+ # Storage classes to associate with this volume. See
+ # "Storage classes" in the "Admin" section of doc.arvados.org.
+ StorageClasses: null
+
+ # Enable deletion (garbage collection) even when TrashLifetime is
+ # zero. WARNING: eventual consistency may result in race conditions
+ # that can cause data loss. Do not enable this unless you know what
+ # you are doing.
+ UnsafeDelete: false
+</pre>
+
+Start (or restart) keepstore, and check its log file to confirm it is using the new configuration.
+
+h3. Example config for Google cloud storage
+
+See previous section for documentation of configuration fields.
+
+<pre>
+Volumes:
+- # Example configuration using alternate storage provider
+ # Configuration for Google cloud storage
+ Endpoint: https://storage.googleapis.com
+ Region: ""
+
+ AccessKeyFile: /etc/arvados/keepstore/gce_s3_access_key.txt
+ SecretKeyFile: /etc/arvados/keepstore/gce_s3_secret_key.txt
+ Bucket: example-bucket-name
+ ConnectTimeout: 1m0s
+ IndexPageSize: 1000
+ LocationConstraint: false
+ RaceWindow: 24h0m0s
+ ReadOnly: false
+ ReadTimeout: 2m0s
+ S3Replication: 2
+ StorageClasses: null
+ UnsafeDelete: false
+</pre>
+
+Start (or restart) keepstore, and check its log file to confirm it is using the new configuration.
---
layout: default
navsection: installguide
-title: Copy pipeline from Curoverse cloud
+title: Copy pipeline from the Arvados Playground
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-This tutorial describes how to find and copy a publicly shared pipeline from Curoverse cloud. Please note that you can use similar steps to copy any template you can access from Curoverse cloud to your cluster.
+This tutorial describes how to find and copy a publicly shared pipeline from the Arvados Playground. Please note that you can use similar steps to copy any template you can access from the Arvados Playground to your cluster.
-h3. Access a public pipeline in Curoverse cloud using Workbench
+h3. Access a public pipeline in the Arvados Playground using Workbench
-Curoverse cloud provides access to some public data, which can be used to experience Arvados in action. Let's access a public pipeline and copy it to your cluster, so that you can run it in your environment.
+the Arvados Playground provides access to some public data, which can be used to experience Arvados in action. Let's access a public pipeline and copy it to your cluster, so that you can run it in your environment.
-Start by visiting the "*Curoverse public projects page*":https://cloud.curoverse.com/projects/public. This page lists all the publicly accessible projects in this arvados installation. Click on one of these projects to open it. We will use "*lobSTR v.3 (Public)*":https://cloud.curoverse.com/projects/qr1hi-j7d0g-up6qgpqz5ie2vfq as the example in this tutorial.
+Start by visiting the "*Arvados Playground public projects page*":https://playground.arvados.org/projects/public. This page lists all the publicly accessible projects in this arvados installation. Click on one of these projects to open it. We will use "*lobSTR v.3 (Public)*":https://playground.arvados.org/projects/qr1hi-j7d0g-up6qgpqz5ie2vfq as the example in this tutorial.
-Once in the "*lobSTR v.3 (Public)*":https://cloud.curoverse.com/projects/qr1hi-j7d0g-up6qgpqz5ie2vfq project, click on the *Pipeline templates* tab. In the pipeline templates tab, you will see a template named *lobSTR v.3*. Click on the <span class="fa fa-lg fa-gears"></span> *Show* button to the left of this name. This will take to you to the "*lobSTR v.3*":https://cloud.curoverse.com/pipeline_templates/qr1hi-p5p6p-9pkaxt6qjnkxhhu template page.
+Once in the "*lobSTR v.3 (Public)*":https://playground.arvados.org/projects/qr1hi-j7d0g-up6qgpqz5ie2vfq project, click on the *Pipeline templates* tab. In the pipeline templates tab, you will see a template named *lobSTR v.3*. Click on the <span class="fa fa-lg fa-gears"></span> *Show* button to the left of this name. This will take to you to the "*lobSTR v.3*":https://playground.arvados.org/pipeline_templates/qr1hi-p5p6p-9pkaxt6qjnkxhhu template page.
Once in this page, you can take the *uuid* of this template from the address bar, which is *qr1hi-p5p6p-9pkaxt6qjnkxhhu*. Next, we will copy this template to your Arvados instance.
-h3. Copying a pipeline template from Curoverse cloud to your cluster
+h3. Copying a pipeline template from the Arvados Playground to your cluster
-As described above, navigate to the publicly shared pipeline template "*lobSTR v.3*":https://cloud.curoverse.com/pipeline_templates/qr1hi-p5p6p-9pkaxt6qjnkxhhu using Curoverse Workbench. We will now copy this template with uuid *qr1hi-p5p6p-9pkaxt6qjnkxhhu* to your cluster.
+As described above, navigate to the publicly shared pipeline template "*lobSTR v.3*":https://playground.arvados.org/pipeline_templates/qr1hi-p5p6p-9pkaxt6qjnkxhhu on the Arvados Playground. We will now copy this template with uuid *qr1hi-p5p6p-9pkaxt6qjnkxhhu* to your cluster.
{% include 'tutorial_expectations' %}
We will use the Arvados *arv-copy* command to copy this template to your cluster. In order to use arv-copy, first you need to setup the source and destination cluster configuration files. Here, *qr1hi* would be the source cluster and your Arvados instance would be the *dst_cluster*.
-During this setup, if you have an account in Curoverse cloud, you can use "your access token":#using-your-token to create the source configuration file. If you do not have an account in Curoverse cloud, you can use the "anonymous access token":#using-anonymous-token for the source cluster configuration.
+During this setup, if you have an account in the Arvados Playground, you can use "your access token":#using-your-token to create the source configuration file. If you do not have an account in the Arvados Playground, you can use the "anonymous access token":#using-anonymous-token for the source cluster configuration.
h4(#using-anonymous-token). *Configuring source and destination setup files using anonymous access token*
h4(#using-your-token). *Configuring source and destination setup files using personal access token*
-If you already have an account in Curoverse cloud, you can follow the instructions in the "*Using arv-copy*":http://doc.arvados.org/user/topics/arv-copy.html user guide to get your *Current token* for source and destination clusters, and use them to create the source *qr1hi.conf* and dst_cluster.conf configuration files.
+If you already have an account in the Arvados Playground, you can follow the instructions in the "*Using arv-copy*":http://doc.arvados.org/user/topics/arv-copy.html user guide to get your *Current token* for source and destination clusters, and use them to create the source *qr1hi.conf* and dst_cluster.conf configuration files.
You can now copy the pipeline template from *qr1hi* to *your cluster* with or without recursion. Replace *dst_cluster* with the *uuid_prefix* of your cluster.
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Create standard objects
-
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-In these steps we use the Arvados CLI tools on the <strong>shell server</strong> to create a few Arvados objects. The CLI tools require an ARVADOS_API_TOKEN environment variable with a valid admin token. If you haven't already done so, set that up as shown in the "API token guide":../user/reference/api-tokens.html.
-
-h3. Arvados repository
-
-Here we create a repository object which will be used to set up a hosted clone of the arvados repository on this cluster.
-
-<notextile>
-<pre><code>~$ <span class="userinput">prefix=`arv --format=uuid user current | cut -d- -f1`</span>
-~$ <span class="userinput">echo "Site prefix is '$prefix'"</span>
-~$ <span class="userinput">all_users_group_uuid="$prefix-j7d0g-fffffffffffffff"</span>
-~$ <span class="userinput">repo_uuid=`arv --format=uuid repository create --repository "{\"owner_uuid\":\"$prefix-tpzed-000000000000000\", \"name\":\"arvados\"}"`</span>
-~$ <span class="userinput">echo "Arvados repository uuid is '$repo_uuid'"</span>
-</code></pre></notextile>
-
-Create a link object to make the repository object readable by the "All users" group, and therefore by every active user. This makes it possible for users to run the bundled Crunch scripts by specifying @"script_version":"master","repository":"arvados"@ rather than pulling the Arvados source tree into their own repositories.
-
-<notextile>
-<pre><code>~$ <span class="userinput">read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"</span>
-<span class="userinput">{
- "tail_uuid":"$all_users_group_uuid",
- "head_uuid":"$repo_uuid",
- "link_class":"permission",
- "name":"can_read"
-}
-EOF</span>
-</code></pre></notextile>
-
-In a couple of minutes, your arvados-git-sync cron job will create an empty repository on your git server. Seed it with the real arvados repository. If your git credential helpers were configured correctly when you "set up your shell server":install-shell-server.html, the "git push" command will use your API token instead of prompting you for a username and password.
-
-<notextile>
-<pre><code>~$ <span class="userinput">cd /tmp</span>
-/tmp$ <span class="userinput">git clone --bare https://github.com/curoverse/arvados.git</span>
-/tmp <span class="userinput">git --git-dir arvados.git push https://git.<b>uuid_prefix.your.domain</b>/arvados.git '*:*'</span>
-</code></pre>
-</notextile>
-
-If you did not set up a HTTPS service, you can push to <code>git@git.uuid_prefix.your.domain:arvados.git</code> using your SSH key, or by logging in to your git server and using sudo.
-
-<notextile>
-<pre><code>gitserver:~$ <span class="userinput">sudo -u git -i bash</span>
-git@gitserver:~$ <span class="userinput">git clone --bare https://github.com/curoverse/arvados.git /tmp/arvados.git</span>
-git@gitserver:~$ <span class="userinput">cd /tmp/arvados.git</span>
-git@gitserver:/tmp/arvados.git$ <span class="userinput">gitolite push /var/lib/arvados/git/repositories/<b>your_arvados_repo_uuid</b>.git '*:*'</span>
-</code></pre>
-</notextile>
-
-h3. Default project for docker images
-
-Here we create a default project for the standard Arvados Docker images, and give all users read access to it. The project is owned by the system user.
-
-<notextile>
-<pre><code>~$ <span class="userinput">project_uuid=`arv --format=uuid group create --group "{\"owner_uuid\":\"$prefix-tpzed-000000000000000\", \"name\":\"Arvados Standard Docker Images\"}"`</span>
-~$ <span class="userinput">echo "Arvados project uuid is '$project_uuid'"</span>
-~$ <span class="userinput">read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"</span>
-<span class="userinput">{
- "tail_uuid":"$all_users_group_uuid",
- "head_uuid":"$project_uuid",
- "link_class":"permission",
- "name":"can_read"
-}
-EOF</span>
-</code></pre></notextile>
-
-h3. Download and tag the latest arvados/jobs docker image
-
-The @arvados-cwl-runner@ needs access to an arvados/jobs image that is tagged as 'latest'. The following command downloads the latest arvados/jobs image from Docker Hub, loads it into Keep, and tags it as 'latest'.
-
-<notextile>
-<pre><code>~$ <span class="userinput">arv-keepdocker --pull arvados/jobs latest</span>
-</code></pre></notextile>
-
-If the image needs to be downloaded from Docker Hub, the command can take a few minutes to complete, depending on available network bandwidth.
This is the only configuration required by crunch-dispatch-slurm. The subsections below describe optional configuration flags you can set inside the main configuration object.
-h3. Client::KeepServiceURIs
+h3(#KeepServiceURIs). Client::KeepServiceURIs
Override Keep service discovery with a predefined list of Keep URIs. This can be useful if the compute nodes run a local keepstore that should handle all Keep traffic. Example:
</code></pre>
</notextile>
-h3. PollPeriod
+h3(#PollPeriod). PollPeriod
crunch-dispatch-slurm polls the API server periodically for new containers to run. The @PollPeriod@ option controls how often this poll happens. Set this to a string of numbers suffixed with one of the time units @ns@, @us@, @ms@, @s@, @m@, or @h@. For example:
</code></pre>
</notextile>
-h3. PrioritySpread
+h3(#PrioritySpread). PrioritySpread
crunch-dispatch-slurm adjusts the "nice" values of its SLURM jobs to ensure containers are prioritized correctly relative to one another. This option tunes the adjustment mechanism.
* If non-Arvados jobs run on your SLURM cluster, and your Arvados containers are waiting too long in the SLURM queue because their "nice" values are too high for them to compete with other SLURM jobs, you should use a smaller PrioritySpread value.
</code></pre>
</notextile>
+h3(#SbatchArguments). SbatchArguments
-
-h3. SbatchArguments
-
-When crunch-dispatch-slurm invokes @sbatch@, you can add switches to the command by specifying @SbatchArguments@. You can use this to send the jobs to specific cluster partitions or add resource requests. Set @SbatchArguments@ to an array of strings. For example:
+When crunch-dispatch-slurm invokes @sbatch@, you can add arguments to the command by specifying @SbatchArguments@. You can use this to send the jobs to specific cluster partitions or add resource requests. Set @SbatchArguments@ to an array of strings. For example:
<notextile>
<pre><code class="userinput">SbatchArguments:
</code></pre>
</notextile>
-h3. CrunchRunCommand: Dispatch to SLURM cgroups
+Note: If an argument is supplied multiple times, @slurm@ uses the value of the last occurrence of the argument on the command line. Arguments specified through Arvados are added after the arguments listed in SbatchArguments. This means, for example, an Arvados container with that specifies @partitions@ in @scheduling_parameter@ will override an occurrence of @--partition@ in SbatchArguments. As a result, for container parameters that can be specified through Arvados, SbatchArguments can be used to specify defaults but not enforce specific policy.
+
+h3(#CrunchRunCommand-cgroups). CrunchRunCommand: Dispatch to SLURM cgroups
If your SLURM cluster uses the @task/cgroup@ TaskPlugin, you can configure Crunch's Docker containers to be dispatched inside SLURM's cgroups. This provides consistent enforcement of resource constraints. To do this, use a crunch-dispatch-slurm configuration like the following:
</code></pre>
</notextile>
-The choice of subsystem ("memory" in this example) must correspond to one of the resource types enabled in SLURM's @cgroup.conf@. Limits for other resource types will also be respected. The specified subsystem is singled out only to let Crunch determine the name of the cgroup provided by SLURM.
+The choice of subsystem ("memory" in this example) must correspond to one of the resource types enabled in SLURM's @cgroup.conf@. Limits for other resource types will also be respected. The specified subsystem is singled out only to let Crunch determine the name of the cgroup provided by SLURM. When doing this, you should also set "ReserveExtraRAM":#ReserveExtraRAM .
{% include 'notebox_begin' %}
{% include 'notebox_end' %}
-h3. CrunchRunCommand: Using host networking for containers
+h3(#CrunchRunCommand-network). CrunchRunCommand: Using host networking for containers
Older Linux kernels (prior to 3.18) have bugs in network namespace handling which can lead to compute node lockups. This by is indicated by blocked kernel tasks in "Workqueue: netns cleanup_net". If you are experiencing this problem, as a workaround you can disable use of network namespaces by Docker across the cluster. Be aware this reduces container isolation, which may be a security risk.
</code></pre>
</notextile>
-h3. MinRetryPeriod: Rate-limit repeated attempts to start containers
+h3(#MinRetryPeriod). MinRetryPeriod: Rate-limit repeated attempts to start containers
If SLURM is unable to run a container, the dispatcher will submit it again after the next PollPeriod. If PollPeriod is very short, this can be excessive. If MinRetryPeriod is set, the dispatcher will avoid submitting the same container to SLURM more than once in the given time span.
</code></pre>
</notextile>
+h3(#ReserveExtraRAM). ReserveExtraRAM: Extra RAM for jobs
+
+Extra RAM to reserve (in bytes) on each SLURM job submitted by Arvados, which is added to the amount specified in the container's @runtime_constraints@. If not provided, the default value is zero. Helpful when using @-cgroup-parent-subsystem@, where @crunch-run@ and @arv-mount@ share the control group memory limit with the user process. In this situation, at least 256MiB is recommended to accomodate each container's @crunch-run@ and @arv-mount@ processes.
+
+<notextile>
+<pre><code class="userinput">ReserveExtraRAM: <b>268435456</b>
+</code></pre>
+</notextile>
+
h2. Restart the dispatcher
{% include 'notebox_begin' %}
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-h2(#slurm). Set up SLURM
-
On the API server, install SLURM and munge, and generate a munge key.
On Debian-based systems:
---
layout: default
navsection: installguide
-title: Installation overview
+title: Installation options
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Arvados components run on GNU/Linux systems, and do not depend on any particular cloud operating stack. Arvados supports Debian and derivatives such as Ubuntu, as well as Red Hat and derivatives such as CentOS.
+Arvados components run on GNU/Linux systems, and supports multiple cloud operating stacks. Arvados supports Debian and derivatives such as Ubuntu, as well as Red Hat and derivatives such as CentOS.
-Arvados components can be installed and configured in a number of different ways. Step-by-step instructions are available to perform a production installation from packages with manual configuration. This method assumes you have several (virtual) machines at your disposal for running the various Arvados components.
+Arvados components can be installed and configured in a number of different ways.
-* "Docker quick start":arvbox.html
-* "Manual installation":install-manual-prerequisites.html
+<div class="offset1">
+table(table table-bordered table-condensed).
+|||\5=. Appropriate for|
+||_. Ease of setup|_. Multiuser/networked access|_. Workflow Development and Testing|_. Large Scale Production|_. Development of Arvados|_. Arvados System Testing|
+|"Arvados-in-a-box":arvbox.html (arvbox)|Easy|no|yes|no|yes|yes|
+|"Arvados on Kubernetes":arvados-on-kubernetes.html|Easy ^1^|yes|yes ^2^|no ^2^|no|yes|
+|"Manual installation":install-manual-prerequisites.html|Complicated|yes|yes|yes|no|no|
+|"Arvados Playground":https://playground.arvados.org hosted by Veritas Genetics|N/A ^3^|yes|yes|no|no|no|
+|"Cluster Operation Subscription":https://curoverse.com/products supported by Veritas Genetics|N/A ^3^|yes|yes|yes|yes|yes|
+</div>
+
+* ^1^ Assumes a Kubernetes cluster is available
+* ^2^ Arvados on Kubernetes is under development and not yet ready for production use
+* ^3^ No installation necessary, Veritas Genetics run and managed
</code></pre>
</notextile>
+h3(#disable_api_methods). disable_api_methods
+
+Set the @disable_api_methods@ configuration option to disable the deprecated @jobs@ API. This will prevent users from accidentally submitting jobs that won't run. "All new installations should use the containers API.":crunch2-slurm/install-prerequisites.html
+
+<notextile>
+<pre><code>
+ disable_api_methods:
+ - jobs.create
+ - pipeline_instances.create
+ - pipeline_templates.create
+ - jobs.get
+ - pipeline_instances.get
+ - pipeline_templates.get
+ - jobs.list
+ - pipeline_instances.list
+ - pipeline_templates.list
+ - jobs.index
+ - pipeline_instances.index
+ - pipeline_templates.index
+ - jobs.update
+ - pipeline_instances.update
+ - pipeline_templates.update
+ - jobs.queue
+ - jobs.queue_size
+ - job_tasks.create
+ - job_tasks.get
+ - job_tasks.list
+ - job_tasks.index
+ - job_tasks.update
+ - jobs.show
+ - pipeline_instances.show
+ - pipeline_templates.show
+ - job_tasks.show
+</code></pre>
+</notextile>
+
h2(#set_up). Set up Nginx and Passenger
The Nginx server will serve API requests using Passenger. It will also be used to proxy SSL requests to other services which are covered later in this guide.
First, "Install Nginx and Phusion Passenger":https://www.phusionpassenger.com/library/walkthroughs/deploy/ruby/ownserver/nginx/oss/install_passenger_main.html.
-Edit the http section of your Nginx configuration to run the Passenger server, and serve SSL requests. Add a block like the following, adding SSL and logging parameters to taste:
+Edit the http section of your Nginx configuration to run the Passenger server. Add a block like the following, adding SSL and logging parameters to taste:
<notextile>
-<pre><code>server {
+<pre><code>
+server {
listen 127.0.0.1:8000;
server_name localhost-api;
default 1;
<span class="userinput">10.20.30.0/24</span> 0;
}
-
-server {
- listen <span class="userinput">[your public IP address]</span>:443 ssl;
- server_name <span class="userinput">uuid_prefix.your.domain</span>;
-
- ssl on;
- ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
- ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
-
- index index.html index.htm index.php;
-
- # Refer to the comment about this setting in the server section above.
- client_max_body_size 128m;
-
- location / {
- proxy_pass http://api;
- proxy_redirect off;
- proxy_connect_timeout 90s;
- proxy_read_timeout 300s;
-
- proxy_set_header X-Forwarded-Proto https;
- proxy_set_header Host $http_host;
- proxy_set_header X-External-Client $external_client;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- }
-}
</code></pre>
</notextile>
It is not strictly necessary to deploy _both_ SSH and HTTPS access, but we recommend deploying both:
* SSH is a more appropriate way to authenticate from a user's workstation because it does not require managing tokens on the client side;
* HTTPS is a more appropriate way to authenticate from a shell VM because it does not depend on SSH agent forwarding (SSH clients' agent forwarding features tend to behave as if the remote machine is fully trusted).
+* HTTPS is also used by Arvados Composer to access git repositories from the browser.
The HTTPS instructions given below will not work if you skip the SSH setup steps.
<pre><code>gitserver:~$ <span class="userinput">sudo nginx -s reload</span>
</code></pre>
</notextile>
+
+h2. Clone Arvados repository
+
+Here we create a repository object which will be used to set up a hosted clone of the arvados repository on this cluster.
+
+<notextile>
+<pre><code>~$ <span class="userinput">prefix=`arv --format=uuid user current | cut -d- -f1`</span>
+~$ <span class="userinput">echo "Site prefix is '$prefix'"</span>
+~$ <span class="userinput">all_users_group_uuid="$prefix-j7d0g-fffffffffffffff"</span>
+~$ <span class="userinput">repo_uuid=`arv --format=uuid repository create --repository "{\"owner_uuid\":\"$prefix-tpzed-000000000000000\", \"name\":\"arvados\"}"`</span>
+~$ <span class="userinput">echo "Arvados repository uuid is '$repo_uuid'"</span>
+</code></pre></notextile>
+
+Create a link object to make the repository object readable by the "All users" group, and therefore by every active user. This makes it possible for users to run the bundled Crunch scripts by specifying @"script_version":"master","repository":"arvados"@ rather than pulling the Arvados source tree into their own repositories.
+
+<notextile>
+<pre><code>~$ <span class="userinput">read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"</span>
+<span class="userinput">{
+ "tail_uuid":"$all_users_group_uuid",
+ "head_uuid":"$repo_uuid",
+ "link_class":"permission",
+ "name":"can_read"
+}
+EOF</span>
+</code></pre></notextile>
+
+In a couple of minutes, your arvados-git-sync cron job will create an empty repository on your git server. Seed it with the real arvados repository. If your git credential helpers were configured correctly when you "set up your shell server":install-shell-server.html, the "git push" command will use your API token instead of prompting you for a username and password.
+
+<notextile>
+<pre><code>~$ <span class="userinput">cd /tmp</span>
+/tmp$ <span class="userinput">git clone --bare https://github.com/curoverse/arvados.git</span>
+/tmp <span class="userinput">git --git-dir arvados.git push https://git.<b>uuid_prefix.your.domain</b>/arvados.git '*:*'</span>
+</code></pre>
+</notextile>
+
+If you did not set up a HTTPS service, you can push to <code>git@git.uuid_prefix.your.domain:arvados.git</code> using your SSH key, or by logging in to your git server and using sudo.
+
+<notextile>
+<pre><code>gitserver:~$ <span class="userinput">sudo -u git -i bash</span>
+git@gitserver:~$ <span class="userinput">git clone --bare https://github.com/curoverse/arvados.git /tmp/arvados.git</span>
+git@gitserver:~$ <span class="userinput">cd /tmp/arvados.git</span>
+git@gitserver:/tmp/arvados.git$ <span class="userinput">gitolite push /var/lib/arvados/git/repositories/<b>your_arvados_repo_uuid</b>.git '*:*'</span>
+</code></pre>
+</notextile>
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Choosing which components to install
+...
+
+Arvados consists of many components, some of which may be omitted (at the cost of reduced functionality.) It may also be helpful to review the "Arvados Architecture":{{site.baseurl}}/architecture to understand how these components interact.
+
+table(table table-bordered table-condensed).
+|\3=. *Core*|
+|"Postgres database":install-postgresql.html |Stores data for the API server.|Required.|
+|"API server":install-api-server.html |Core Arvados logic for managing users, groups, collections, containers, and enforcing permissions.|Required.|
+|\3=. *Keep (storage)*|
+|"Keepstore":install-keepstore.html |Stores content-addressed blocks in a variety of backends (local filesystem, cloud object storage).|Required.|
+|"Keepproxy":install-keepproxy.html |Gateway service to access keep servers from external networks.|Required to be able to use arv-put, arv-get, or arv-mount outside the private Arvados network.|
+|"Keep-web":install-keep-web.html |Gateway service providing read/write HTTP and WebDAV support on top of Keep.|Required to be able to download files from Keep over plain HTTP in Workbench.|
+|"Keep-balance":install-keep-balance.html |Storage cluster maintenance daemon responsible for moving blocks to their optimal server location, adjusting block replication levels, and trashing unreferenced blocks.|Required to free deleted data from underlying storage, and to ensure proper replication and block distribution (including support for storage classes).|
+|\3=. *User interface*|
+|"Single Sign On server":install-sso.html |Login server.|Required for web based login to Workbench.|
+|"Workbench":install-workbench-app.html |Primary graphical user interface for working with file collections and running containers.|Optional. Depends on API server, SSO server, keep-web, websockets server.|
+|"Workflow Composer":install-composer.html |Graphical user interface for editing Common Workflow Language workflows.|Optional. Depends on git server (arv-git-httpd).|
+|\3=. *Additional services*|
+|"Websockets server":install-ws.html |Event distribution server.|Required to view streaming container logs in Workbench.|
+|"Shell server":install-shell-server.html |Synchronize (create/delete/configure) Unix shell accounts with Arvados users.|Optional.|
+|"Git server":install-arv-git-httpd.html |Arvados-hosted git repositories, with Arvados-token based authentication.|Optional, but required by Workflow Composer.|
+|\3=. *Crunch (running containers)*|
+|"crunch-dispatch-slurm":crunch2-slurm/install-prerequisites.html |Run analysis workflows using Docker containers distributed across a SLURM cluster.|Optional if you wish to use Arvados for data management only.|
+|"Node Manager":install-nodemanager.html |Allocate and free cloud VM instances on demand based on workload.|Optional, not needed for a static SLURM cluster (such as on-premise HPC).|
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Install Composer
+...
+
+Arvados Composer is a single-page javascript application for building Common Workflow Languge (CWL) Workflows.
+
+h2. Prerequisites
+
+In addition to Arvados core services, Composer requires "Arvados hosted git repositories":install-arv-git-httpd.html which are used for storing workflow files.
+
+h2. Install
+
+Composer may be installed on the same host as Workbench, or on a different host. Composer communicates directly with the Arvados API server. It does not require its own backend and should be served as a static file.
+
+On a Debian-based system, install the following package:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo apt-get install arvados-composer</span>
+</code></pre>
+</notextile>
+
+On a Red Hat-based system, install the following package:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo yum install arvados-composer</span>
+</code></pre>
+</notextile>
+
+h2. Configure
+
+h3. composer.yml
+
+Edit @/etc/arvados/composer/composer.yml@ and set @apiEndPoint@ to your API server:
+
+<pre>
+apiEndPoint: https://zzzzz.arvadosapi.com
+</pre>
+
+h3. Nginx
+
+Add Composer to your Nginx configuration. This example will host Composer at @/composer@.
+
+<pre>
+location /composer {
+ root /var/www/arvados-composer
+ index index.html
+}
+</pre>
+
+h3. Workbench link to composer
+
+Edit the workbench @application.yml@ and set @composer_url@ to the location from which it is served.
+
+<pre>
+production:
+ composer_url: 'https://workbench.zzzzz.arvadosapi.com/composer'
+</pre>
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Install the controller
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+The arvados-controller service must be installed on your API server node.
+
+On Debian-based systems:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo apt-get install arvados-controller</span>
+</code></pre>
+</notextile>
+
+On Red Hat-based systems:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo yum install arvados-controller</span>
+</code></pre>
+</notextile>
+
+Verify the @arvados-controller@ program is functional:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arvados-controller -h</span>
+Usage:
+ -config file
+[...]
+</code></pre>
+</notextile>
+
+h3. Configure Nginx to route requests to the controller
+
+Add @upstream@ and @server@ definitions inside the @http@ section of your Nginx configuration using the following template.
+
+{% include 'notebox_begin' %}
+
+If you are adding arvados-controller to an existing system as part of the upgrade procedure, do not add a new "server" part here. Instead, add only the "upstream" part as shown here, and update your existing "server" section by changing its @proxy_pass@ directive from @http://api@ to @http://controller@.
+
+{% include 'notebox_end' %}
+
+<notextile>
+<pre><code>upstream controller {
+ server 127.0.0.1:9004 fail_timeout=10s;
+}
+
+server {
+ listen <span class="userinput">[your public IP address]</span>:443 ssl;
+ server_name <span class="userinput">uuid_prefix.your.domain</span>;
+
+ ssl on;
+ ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
+ ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
+
+ # Refer to the comment about this setting in the passenger (arvados
+ # api server) section of your Nginx configuration.
+ client_max_body_size 128m;
+
+ location / {
+ proxy_pass http://controller;
+ proxy_redirect off;
+ proxy_connect_timeout 90s;
+ proxy_read_timeout 300s;
+
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-External-Client $external_client;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ }
+}
+</code></pre>
+</notextile>
+
+Restart Nginx to apply the new configuration.
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo nginx -s reload</span>
+</code></pre>
+</notextile>
+
+h3. Configure arvados-controller
+
+Create the cluster configuration file @/etc/arvados/config.yml@ using the following template.
+
+<notextile>
+<pre><code>Clusters:
+ <span class="userinput">uuid_prefix</span>:
+ NodeProfiles:
+ apiserver:
+ arvados-controller:
+ Listen: ":<span class="userinput">9004</span>" # must match the "upstream controller" section of your Nginx config
+ arvados-api-server:
+ Listen: ":<span class="userinput">8000</span>" # must match the "upstream api" section of your Nginx config
+ PostgreSQL:
+ ConnectionPool: 128
+ Connection:
+ host: localhost
+ dbname: arvados_production
+ user: arvados
+ password: <span class="userinput">xxxxxxxx</span>
+ sslmode: require
+</code></pre>
+</notextile>
+
+Create the host configuration file @/etc/arvados/environment@.
+
+<notextile>
+<pre><code>ARVADOS_NODE_PROFILE=apiserver
+</code></pre>
+</notextile>
+
+h3. Start the service (option 1: systemd)
+
+If your system does not use systemd, skip this section and follow the "runit instructions":#runit instead.
+
+If your system uses systemd, the arvados-controller service should already be set up. Restart it to load the new configuration file, and check its status:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo systemctl restart arvados-controller</span>
+~$ <span class="userinput">sudo systemctl status arvados-controller</span>
+● arvados-controller.service - Arvados controller
+ Loaded: loaded (/lib/systemd/system/arvados-controller.service; enabled; vendor preset: enabled)
+ Active: active (running) since Tue 2018-07-31 13:17:44 UTC; 3s ago
+ Docs: https://doc.arvados.org/
+ Main PID: 25066 (arvados-control)
+ CGroup: /system.slice/arvados-controller.service
+ └─25066 /usr/bin/arvados-controller
+
+Jul 31 13:17:44 zzzzz systemd[1]: Starting Arvados controller...
+Jul 31 13:17:44 zzzzz arvados-controller[25191]: {"Listen":"[::]:9004","Service":"arvados-controller","level":"info","msg":"listening","time":"2018-07-31T13:17:44.521694195Z"}
+Jul 31 13:17:44 zzzzz systemd[1]: Started Arvados controller.
+</code></pre>
+</notextile>
+
+Skip ahead to "confirm the service is working":#confirm.
+
+h3(#runit). Start the service (option 2: runit)
+
+Install runit to supervise the arvados-controller daemon. {% include 'install_runit' %}
+
+Create a supervised service.
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo mkdir /etc/service/arvados-controller</span>
+~$ <span class="userinput">cd /etc/service/arvados-controller</span>
+~$ <span class="userinput">sudo mkdir log log/main</span>
+~$ <span class="userinput">printf '#!/bin/sh\nset -a\n. /etc/arvados/environment\nexec arvados-controller 2>&1\n' | sudo tee run</span>
+~$ <span class="userinput">printf '#!/bin/sh\nexec svlogd main\n' | sudo tee log/run</span>
+~$ <span class="userinput">sudo chmod +x run log/run</span>
+~$ <span class="userinput">sudo sv exit .</span>
+~$ <span class="userinput">cd -</span>
+</code></pre>
+</notextile>
+
+Use @sv stat@ and check the log file to verify the service is running.
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo sv stat /etc/service/arvados-controller</span>
+run: /etc/service/arvados-controller: (pid 12520) 2s; run: log: (pid 12519) 2s
+~$ <span class="userinput">tail /etc/service/arvados-controller/log/main/current</span>
+{"Listen":"[::]:9004","Service":"arvados-controller","level":"info","msg":"listening","time":"2018-07-31T13:17:44.521694195Z"}
+</code></pre>
+</notextile>
+
+h3(#confirm). Confirm the service is working
+
+Confirm the service is listening on its assigned port and responding to requests.
+
+<notextile>
+<pre><code>~$ <span class="userinput">curl -X OPTIONS http://0.0.0.0:<b>9004</b>/login</span>
+{"errors":["Forbidden"],"error_token":"1533044555+684b532c"}
+</code></pre>
+</notextile>
Create an Arvados superuser token for use by keep-balance. *On the API server*, run:
-<notextile>
-<pre><code>apiserver:~$ <span class="userinput">cd /var/www/arvados-api/current</span>
-apiserver:/var/www/arvados-api/current$ <span class="userinput">sudo -u <b>webserver-user</b> RAILS_ENV=production bundle exec script/create_superuser_token.rb</span>
-zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
-</code></pre>
-</notextile>
+{% include 'create_superuser_token' %}
h3. Update keepstore configuration files
h3. Tell the API server about the Keepproxy server
-The API server needs to be informed about the presence of your Keepproxy server. Please execute the following commands on your <strong>shell server</strong>.
+The API server needs to be informed about the presence of your Keepproxy server.
+
+First, if you don't already have an admin token, create a superuser token:
+
+{% include 'create_superuser_token' %}
+
+Configure your environment to run @arv@ using the output of create_superuser_token.rb:
+
+<pre>
+export ARVADOS_API_HOST=zzzzz.example.com
+export ARVADOS_API_TOKEN=zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+</pre>
<notextile>
<pre><code>~$ <span class="userinput">uuid_prefix=`arv --format=uuid user current | cut -d- -f1`</span>
}
EOF</span>
</code></pre></notextile>
+
+h3. Testing keepproxy
+
+Log into a host that is on an external network from your private Arvados network. The host should be able to contact your keepproxy server (eg keep.$uuid_prefix.arvadosapi.com), but not your keepstore servers (eg keep[0-9].$uuid_prefix.arvadosapi.com).
+
+Install the "Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html
+
+@ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ must be set in the environment.
+
+You should now be able to use @arv-put@ to upload collections and @arv-get@ to fetch collections, for an example see "Testing keep.":install-keepstore.html#testing on the keepstore install page.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-We are going to install two Keepstore servers. By convention, we use the following hostname pattern:
+Keepstore provides access to underlying storage for reading and writing content-addressed blocks, with enforcement of Arvados permissions. Keepstore supports a variety of cloud object storage and POSIX filesystems for its backing store.
+
+We recommend starting off with two Keepstore servers. Exact server specifications will be site and workload specific, but in general keepstore will be I/O bound and should be set up to maximize aggregate bandwidth with compute nodes. To increase capacity (either space or throughput) it is straightforward to add additional servers, or (in cloud environments) to increase the machine size of the existing servers.
+
+By convention, we use the following hostname pattern:
<div class="offset1">
table(table table-bordered table-condensed).
|keep1.@uuid_prefix@.your.domain|
</div>
-Because the Keepstore servers are not directly accessible from the internet, these hostnames only need to resolve on the local network.
+Keepstore servers should not be directly accessible from the Internet (they are accessed via "keepproxy":install-keepproxy.html), so the hostnames only need to resolve on the private network.
h2. Install Keepstore
Verify that Keepstore is functional:
<notextile>
-<pre><code>~$ <span class="userinput">keepstore -h</span>
-2016/07/01 14:06:21 keepstore starting, pid 32339
-Usage of ./keepstore:
- -azure-max-get-bytes int
- Maximum bytes to request in a single GET request. If smaller than 67108864, use multiple concurrent range requests to retrieve a block. (default 67108864)
- -azure-storage-account-key-file string
- File containing the account key used for subsequent --azure-storage-container-volume arguments.
- -azure-storage-account-name string
- Azure storage account name used for subsequent --azure-storage-container-volume arguments.
- -azure-storage-container-volume value
- Use the given container as a storage volume. Can be given multiple times. (default [])
- -azure-storage-replication int
- Replication level to report to clients when data is stored in an Azure container. (default 3)
- -blob-signature-ttl int
- Lifetime of blob permission signatures in seconds. Modifying the ttl will invalidate all existing signatures. See services/api/config/application.default.yml. (default 1209600)
- -blob-signing-key-file string
- File containing the secret key for generating and verifying blob permission signatures.
- -data-manager-token-file string
- File with the API token used by the Data Manager. All DELETE requests or GET /index requests must carry this token.
- -enforce-permissions
- Enforce permission signatures on requests.
- -listen string
- Listening address, in the form "host:port". e.g., 10.0.1.24:8000. Omit the host part to listen on all interfaces. (default ":25107")
- -max-buffers int
- Maximum RAM to use for data buffers, given in multiples of block size (64 MiB). When this limit is reached, HTTP requests requiring buffers (like GET and PUT) will wait for buffer space to be released. (default 128)
- -max-requests int
- Maximum concurrent requests. When this limit is reached, new requests will receive 503 responses. Note: this limit does not include idle connections from clients using HTTP keepalive, so it does not strictly limit the number of concurrent connections. (default 2 * max-buffers)
- -never-delete
- If true, nothing will be deleted. Warning: the relevant features in keepstore and data manager have not been extensively tested. You should leave this option alone unless you can afford to lose data. (default true)
- -permission-key-file string
- Synonym for -blob-signing-key-file.
- -permission-ttl int
- Synonym for -blob-signature-ttl.
- -pid fuser -k pidfile
- Path to write pid file during startup. This file is kept open and locked with LOCK_EX until keepstore exits, so fuser -k pidfile is one way to shut down. Exit immediately if there is an error opening, locking, or writing the pid file.
- -readonly
- Do not write, delete, or touch anything on the following volumes.
- -s3-access-key-file string
- File containing the access key used for subsequent -s3-bucket-volume arguments.
- -s3-bucket-volume value
- Use the given bucket as a storage volume. Can be given multiple times. (default [])
- -s3-endpoint string
- Endpoint URL used for subsequent -s3-bucket-volume arguments. If blank, use the AWS endpoint corresponding to the -s3-region argument. For Google Storage, use "https://storage.googleapis.com".
- -s3-region string
- AWS region used for subsequent -s3-bucket-volume arguments. Allowed values are ["ap-southeast-1" "eu-west-1" "us-gov-west-1" "sa-east-1" "cn-north-1" "ap-northeast-1" "ap-southeast-2" "eu-central-1" "us-east-1" "us-west-1" "us-west-2"].
- -s3-replication int
- Replication level reported to clients for subsequent -s3-bucket-volume arguments. (default 2)
- -s3-secret-key-file string
- File containing the secret key used for subsequent -s3-bucket-volume arguments.
- -s3-unsafe-delete
- EXPERIMENTAL. Enable deletion (garbage collection), even though there are known race conditions that can cause data loss.
- -serialize
- Serialize read and write operations on the following volumes.
- -trash-check-interval duration
- Time duration at which the emptyTrash goroutine will check and delete expired trashed blocks. Default is one day. (default 24h0m0s)
- -trash-lifetime duration
- Time duration after a block is trashed during which it can be recovered using an /untrash request
- -volume value
- Local storage directory. Can be given more than once to add multiple directories. If none are supplied, the default is to use all directories named "keep" that exist in the top level directory of a mount point at startup time. Can be a comma-separated list, but this is deprecated: use multiple -volume arguments instead. (default [])
- -volumes value
- Deprecated synonym for -volume. (default [])
+<pre><code>~$ <span class="userinput">keepstore --version</span>
</code></pre>
</notextile>
-h3. Prepare storage volumes
-
-{% include 'notebox_begin' %}
-This section uses a local filesystem as a backing store. If you are using Azure Storage, follow the setup instructions on the "Azure Blob Storage":configure-azure-blob-storage.html page instead.
-{% include 'notebox_end' %}
-
-There are two ways to specify a set of local directories where keepstore should store its data files.
-# Implicitly, by creating a directory called @keep@ at the top level of each filesystem you intend to use, and omitting @-volume@ arguments.
-# Explicitly, by providing a @-volume@ argument for each directory.
-
-For example, if there are filesystems mounted at @/mnt@ and @/mnt2@:
-
-<notextile>
-<pre><code>~$ <span class="userinput">mkdir /mnt/keep /mnt2/keep</span>
-~$ <span class="userinput">keepstore</span>
-2015/05/08 13:44:26 keepstore starting, pid 2765
-2015/05/08 13:44:26 Using volume [UnixVolume /mnt/keep] (writable=true)
-2015/05/08 13:44:26 Using volume [UnixVolume /mnt2/keep] (writable=true)
-2015/05/08 13:44:26 listening at :25107
-</code></pre>
-</notextile>
-
-Equivalently:
-
-<notextile>
-<pre><code>~$ <span class="userinput">mkdir /mnt/keep /mnt2/keep</span>
-~$ <span class="userinput">keepstore -volume=/mnt/keep -volume=/mnt2/keep</span>
-2015/05/08 13:44:26 keepstore starting, pid 2765
-2015/05/08 13:44:26 Using volume [UnixVolume /mnt/keep] (writable=true)
-2015/05/08 13:44:26 Using volume [UnixVolume /mnt2/keep] (writable=true)
-2015/05/08 13:44:26 listening at :25107
-</code></pre>
-</notextile>
+h3. Create config file
+
+By default, keepstore will look for its configuration file at @/etc/arvados/keepstore/keepstore.yml@
+
+You can override the configuration file location using the @-config@ command line option to keepstore.
+
+The following is a sample configuration file:
+
+<pre>
+# Duration for which new permission signatures (returned in PUT
+# responses) will be valid. This should be equal to the API
+# server's blob_signature_ttl configuration entry.
+BlobSignatureTTL: 336h0m0s
+
+# Local file containing the secret blob signing key (used to generate
+# and verify blob signatures). The contents of the key file must be
+# identical to the API server's blob_signing_key configuration entry.
+BlobSigningKeyFile: ""
+
+# Print extra debug logging
+Debug: false
+
+# Maximum number of concurrent block deletion operations (per
+# volume) when emptying trash. Default is 1.
+EmptyTrashWorkers: 1
+
+# Enable trash and delete features. If false, trash lists will be
+# accepted but blocks will not be trashed or deleted.
+# Keepstore does not delete data on its own. The keep-balance
+# service determines which blocks are candidates for deletion
+# and instructs the keepstore to move those blocks to the trash.
+EnableDelete: true
+
+# Local port to listen on. Can be 'address:port' or ':port', where
+# 'address' is a host IP address or name and 'port' is a port number
+# or name.
+Listen: :25107
+
+# Format of request/response and error logs: "json" or "text".
+LogFormat: json
+
+# The secret key that must be provided by monitoring services
+# wishing to access the health check endpoint (/_health).
+ManagementToken: ""
+
+# Maximum RAM to use for data buffers, given in multiples of block
+# size (64 MiB). When this limit is reached, HTTP requests requiring
+# buffers (like GET and PUT) will wait for buffer space to be
+# released.
+#
+# It should be set such that MaxBuffers * 64MiB + 10% fits
+# comfortably in memory. On a host dedicated to running keepstore,
+# divide total memory by 88MiB to suggest a suitable value. For example,
+# if grep MemTotal /proc/meminfo reports MemTotal: 7125440 kB,
+# compute 7125440 / (88 * 1024)=79 and configure MaxBuffers: 79
+MaxBuffers: 128
+
+# Maximum concurrent requests. When this limit is reached, new
+# requests will receive 503 responses. Note: this limit does not
+# include idle connections from clients using HTTP keepalive, so it
+# does not strictly limit the number of concurrent connections. If
+# omitted or zero, the default is 2 * MaxBuffers.
+MaxRequests: 0
+
+# Path to write PID file during startup. This file is kept open and
+# locked with LOCK_EX until keepstore exits, so "fuser -k pidfile" is
+# one way to shut down. Exit immediately if there is an error
+# opening, locking, or writing the PID file.
+PIDFile: ""
+
+# Maximum number of concurrent pull operations. Default is 1, i.e.,
+# pull lists are processed serially. A pull operation copies a block
+# from another keepstore server.
+PullWorkers: 1
+
+# Honor read requests only if a valid signature is provided. This
+# should be true, except for development use and when migrating from
+# a very old version.
+RequireSignatures: true
+
+# Local file containing the Arvados API token used by keep-balance
+# or data manager. Delete, trash, and index requests are honored
+# only for this token.
+SystemAuthTokenFile: ""
+
+# Path to server certificate file in X509 format. Enables TLS mode.
+#
+# Example: /var/lib/acme/live/keep0.example.com/fullchain
+TLSCertificateFile: ""
+
+# Path to server key file in X509 format. Enables TLS mode.
+#
+# The key pair is read from disk during startup, and whenever SIGHUP
+# is received.
+#
+# Example: /var/lib/acme/live/keep0.example.com/privkey
+TLSKeyFile: ""
+
+# How often to check for (and delete) trashed blocks whose
+# TrashLifetime has expired.
+TrashCheckInterval: 24h0m0s
+
+# Time duration after a block is trashed during which it can be
+# recovered using an /untrash request.
+TrashLifetime: 336h0m0s
+
+# Maximum number of concurrent trash operations (moving a block to the
+# trash, or permanently deleting it) . Default is 1, i.e., trash lists
+# are processed serially. If individual trash operations have high
+# latency (eg some cloud platforms) you should increase this.
+TrashWorkers: 1
+</pre>
+
+h3. Notes on storage management
+
+On its own, a keepstore server never deletes data. The "keep-balance":install-keep-balance.html service determines which blocks are candidates for deletion and instructs the keepstore to move those blocks to the trash.
+
+When a block is newly written, it is protected from deletion for the duration in @BlobSignatureTTL@. During this time, it cannot be trashed.
+
+If keep-balance instructs keepstore to trash a block which is older than @BlobSignatureTTL@, and @EnableDelete@ is true, the block will be moved to "trash". A block which is in the trash is no longer accessible by read requests, but has not yet been permanently deleted. Blocks which are in the trash may be recovered using the "untrash" API endpoint. Blocks are permanently deleted after they have been in the trash for the duration in @TrashLifetime@.
+
+Keep-balance is also responsible for balancing the distribution of blocks across keepstore servers by asking servers to pull blocks from other servers (as determined by their "storage class":{{site.baseurl}}/admin/storage-classes.html and "rendezvous hashing order":{{site.baseurl}}/api/storage.html). Pulling a block makes a copy. If a block is overreplicated (i.e. there are excess copies) after pulling, it will be subsequently trashed on the original server.
+
+h3. Configure storage volumes
+
+Available storage volume types include POSIX filesystems and cloud object storage.
+
+* To use a POSIX filesystem, including both local filesystems (ext4, xfs) and network file system such as GPFS or Lustre, follow the setup instructions on "Filesystem storage":configure-fs-storage.html
+* If you are using S3-compatible object storage (including Amazon S3, Google Cloud Storage, and Ceph RADOS), follow the setup instructions on "S3 Object Storage":configure-s3-object-storage.html
+* If you are using Azure Blob Storage, follow the setup instructions on "Azure Blob Storage":configure-azure-blob-storage.html
h3. Run keepstore as a supervised service
Install runit to supervise the keepstore daemon. {% include 'install_runit' %}
-Install this script as the run script for the keepstore service, modifying it as directed below.
+Install this script as the run script @/etc/sv/keepstore/run@ for the keepstore service:
<notextile>
<pre><code>#!/bin/sh
exec 2>&1
-GOGC=10 exec keepstore \
- -enforce-permissions=true \
- -blob-signing-key-file=<span class="userinput">/etc/keepstore/blob-signing.key</span> \
- -max-buffers=<span class="userinput">100</span> \
- -serialize=true \
- -never-delete=false \
- -volume=<span class="userinput">/mnt/keep</span> \
- -volume=<span class="userinput">/mnt2/keep</span>
+GOGC=10 exec keepstore -config /etc/arvados/keepstore/keepstore.yml
</code></pre>
</notextile>
-p(#max-buffers). The @-max-buffers@ argument limits keepstore's memory usage. It should be set such that @max-buffers * 64MiB + 10%@ fits comfortably in memory. On a host dedicated to running keepstore, divide total memory by 88MiB to suggest a suitable value. For example, if @grep MemTotal /proc/meminfo@ reports @MemTotal: 7125440 kB@, compute 7125440÷(88×1024)=79 and configure @-max-buffers=79@.
-
-If you want access control on your Keepstore server(s), you must specify the @-enforce-permissions@ flag and provide a signing key. The @-blob-signing-key-file@ argument should be a file containing a long random alphanumeric string with no internal line breaks (it is also possible to use a socket or FIFO: keepstore reads it only once, at startup). This key must be the same as the @blob_signing_key@ configured in the "API server's":install-api-server.html configuration file, @/etc/arvados/api/application.yml@.
-
-The @-serialize=true@ (default: @false@) argument limits keepstore to one reader/writer process per storage partition. This avoids thrashing by allowing the storage device underneath the storage partition to do read/write operations sequentially. Enabling @-serialize@ can improve Keepstore performance if the storage partitions map 1:1 to physical disks that are dedicated to Keepstore, particularly so for mechanical disks. In some cloud environments, enabling @-serialize@ has also also proven to be beneficial for performance, but YMMV. If your storage partition(s) are backed by network or RAID storage that can handle many simultaneous reader/writer processes without thrashing, you probably do not want to set @-serialize@.
-
h3. Set up additional servers
Repeat the above sections to prepare volumes and bring up supervised services on each Keepstore server you are setting up.
h3. Tell the API server about the Keepstore servers
-The API server needs to be informed about the presence of your Keepstore servers. For each of the Keepstore servers you have created, please execute the following commands on your <strong>shell server</strong>.
+The API server needs to be informed about the presence of your Keepstore servers.
+
+First, if you don't already have an admin token, create a superuser token:
+
+{% include 'create_superuser_token' %}
-Make sure to update the @service_host@ value to match each of your Keepstore servers.
+Configure your environment to run @arv@ using the output of create_superuser_token.rb:
+
+<pre>
+export ARVADOS_API_HOST=zzzzz.example.com
+export ARVADOS_API_TOKEN=zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+</pre>
+
+Use this command to register each keepstore server you have installed. Make sure to update the @service_host@ value.
<notextile>
<pre><code>~$ <span class="userinput">prefix=`arv --format=uuid user current | cut -d- -f1`</span>
~$ <span class="userinput">echo "Site prefix is '$prefix'"</span>
~$ <span class="userinput">read -rd $'\000' keepservice <<EOF; arv keep_service create --keep-service "$keepservice"</span>
<span class="userinput">{
- "service_host":"<strong>keep0.$prefix.your.domain</strong>",
+ "service_host":"<strong>keep0.$uuid_prefix.your.domain</strong>",
"service_port":25107,
"service_ssl_flag":false,
"service_type":"disk"
}
EOF</span>
</code></pre></notextile>
+
+h3(#testing). Testing keep
+
+Install the "Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html
+
+@ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ must be set in the environment.
+
+You should now be able to use @arv-put@ to upload collections and @arv-get@ to fetch collections:
+
+<pre>
+$ echo "hello world!" > hello.txt
+
+$ arv-put --portable-data-hash hello.txt
+2018-07-12 13:35:25 arvados.arv_put[28702] INFO: Creating new cache file at /home/example/.cache/arvados/arv-put/1571ec0adb397c6a18d5c74cc95b3a2a
+0M / 0M 100.0% 2018-07-12 13:35:27 arvados.arv_put[28702] INFO:
+
+2018-07-12 13:35:27 arvados.arv_put[28702] INFO: Collection saved as 'Saved at 2018-07-12 17:35:25 UTC by example@example'
+59389a8f9ee9d399be35462a0f92541c+53
+
+$ arv-get 59389a8f9ee9d399be35462a0f92541c+53/hello.txt
+hello world!
+</pre>
<div class="offset1">
table(table table-bordered table-condensed).
-|_Function_|_Number of nodes_|
+|_. Function|_. Number of nodes|
|Arvados API, Crunch dispatcher, Git, Websockets and Workbench|1|
|Arvados Compute node|1|
|Arvados Keepproxy and Keep-web server|1|
h2. Supported GNU/Linux distributions
table(table table-bordered table-condensed).
-|_Distribution_|_State_|_Last supported version_|
+|_. Distribution|_. State|_. Last supported version|
|CentOS 7|Supported|Latest|
|Debian 8 ("jessie")|Supported|Latest|
|Debian 9 ("stretch")|Supported|Latest|
Configure apt to retrieve packages from the Arvados package repository. This command depends on your OS vendor and version:
table(table table-bordered table-condensed).
-|OS version|Command|
+|_. OS version|_. Command|
|Debian 8 ("jessie")|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ jessie main" | sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
|Debian 9 ("stretch")|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ stretch main" | sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
|Ubuntu 14.04 ("trusty")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ trusty main" | sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
<div class="offset1">
table(table table-bordered table-condensed).
-|_Function_|_Hostname_|
+|_. Function|_. Hostname|
|Arvados API|@uuid_prefix@.your.domain|
|Arvados Git server|git.@uuid_prefix@.your.domain|
|Arvados Keepproxy server|keep.@uuid_prefix@.your.domain|
Arvados Node Manager provides elastic computing for Arvados and SLURM by creating and destroying virtual machines on demand. Node Manager currently supports Amazon Web Services (AWS), Google Cloud Platform (GCP) and Microsoft Azure.
-Note: node manager is only required for elastic computing cloud environments. Fixed size clusters do not require node manager.
+Note: node manager is only required for elastic computing cloud environments. Fixed size clusters (such as on-premise HPC) do not require node manager.
h2. Install
# an Arvados node that hasn't been updated for this long.
node_stale_after = 14400
+# Number of consecutive times a node must report as "idle" before it
+# will be considered eligible for shutdown. Node status is checked
+# each poll period, and node can go idle at any point during a poll
+# period (meaning a node could be reported as idle that has only been
+# idle for 1 second). With a 60 second poll period, three consecutive
+# status updates of "idle" suggests the node has been idle at least
+# 121 seconds.
+consecutive_idle_count = 3
+
# Scaling factor to be applied to nodes' available RAM size. Usually there's a
# variable discrepancy between the advertised RAM value on cloud nodes and the
# actual amount available.
# an Arvados node that hasn't been updated for this long.
node_stale_after = 14400
+# Number of consecutive times a node must report as "idle" before it
+# will be considered eligible for shutdown. Node status is checked
+# each poll period, and node can go idle at any point during a poll
+# period (meaning a node could be reported as idle that has only been
+# idle for 1 second). With a 60 second poll period, three consecutive
+# status updates of "idle" suggests the node has been idle at least
+# 121 seconds.
+consecutive_idle_count = 3
+
# Scaling factor to be applied to nodes' available RAM size. Usually there's a
# variable discrepancy between the advertised RAM value on cloud nodes and the
# actual amount available.
# an Arvados node that hasn't been updated for this long.
node_stale_after = 14400
+# Number of consecutive times a node must report as "idle" before it
+# will be considered eligible for shutdown. Node status is checked
+# each poll period, and node can go idle at any point during a poll
+# period (meaning a node could be reported as idle that has only been
+# idle for 1 second). With a 60 second poll period, three consecutive
+# status updates of "idle" suggests the node has been idle at least
+# 121 seconds.
+consecutive_idle_count = 3
+
# Scaling factor to be applied to nodes' available RAM size. Usually there's a
# variable discrepancy between the advertised RAM value on cloud nodes and the
# actual amount available.
h2. LobSTR v3
-In this quickstart guide, we'll run an existing pipeline with pre-existing data. Step-by-step instructions are shown below. You can follow along using your own local install or by using Curoverse's <a href="http://lp.curoverse.com/beta-signup/">hosted version of Arvados</a> (in public beta, any Google account can be used to login).
+In this quickstart guide, we'll run an existing pipeline with pre-existing data. Step-by-step instructions are shown below. You can follow along using your own local install or by using the <a href="https://playground.arvados.org/">Arvados Playground</a> (any Google account can be used to log in).
(For more information about this pipeline, see our <a href="https://dev.arvados.org/projects/arvados/wiki/LobSTR_tutorial">detailed lobSTR guide</a>).
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: Create a Workflow with Composer
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+The Arvados Workflow Composer is a graphical interface for building Common Workflow Language (CWL) workflows to run on Arvados.
+
+This tutorial will demonstrate:
+
+# Creating a new git repository through Arvados to store the workflow
+# Creating CommandLineTools for "sort" and "uniq"
+# Creating a Workflow which uses "sort" and "uniq" to remove duplicate lines from a text file
+# Submitting the Workflow to run on Arvados
+
+h3. 1. Access from workbench
+
+!(screenshot)c1.png!
+
+h3. 2. Composer starting page
+
+!(screenshot)c2.png!
+
+h3. 3. Manage git repositories (opens Workbench in new tab)
+
+!(screenshot)c2b.png!
+
+h3. 4. Add a new repository
+
+!(screenshot)c4.png!
+
+!(screenshot)c3.png!
+
+h3. 5. Return to Composer. Use refresh button to discover new repository (may take a few moments to show up).
+
+!(screenshot)c2c.png!
+
+h3. 6. Create a new Command Line Tool
+
+!(screenshot)c5.png!
+
+!(screenshot)c20.png!
+
+h3. 7. Set Docker image, base command, and input port for "sort" tool
+
+The "Docker Repository" is the name:tag of a "Docker image uploaded Arvados.":{{site.baseurl}}/user/topics/arv-docker.html (Use @arv-keepdocker --pull debian:8@) You can also find prepackaged bioinformatics tools on various sites, such as http://dockstore.org and http://biocontainers.pro/ .
+
+!(screenshot)c6.png!
+
+h3. 8. Redirect stdout to a file
+
+!(screenshot)c7.png!
+
+h3. 9. Capture output file
+
+!(screenshot)c8.png!
+
+h3. 10. Save Command Line Tool
+
+!(screenshot)c22.png!
+
+h3. 11. Repeat steps 6-10 for "uniq" tool
+
+Create a new tool with a "base command" of "uniq".
+
+h3. 12. Switch back to "Home" tab and create workflow
+
+!(screenshot)c24.png!
+
+!(screenshot)c9.png!
+
+!(screenshot)c10.png!
+
+h3. 13. Drag and drop tools into Workflow
+
+!(screenshot)c11.png!
+
+h3. 14. Drag from input port of "sort" to empty space to create workflow input
+
+!(screenshot)c21.png!
+
+h3. 15. Drag from output port of "sort" to input port of "uniq"
+
+!(screenshot)c13.png!
+
+h3. 16. Drag from output port of "uniq" to empty space to create workflow output
+
+!(screenshot)c14.png!
+
+h3. 17. Save Workflow
+
+!(screenshot)c23.png!
+
+h3. 18. Click on "Test" tab then click "Run"
+
+!(screenshot)c15.png!
+
+h3. 19. Choose input file
+
+You may need to "upload an input file":{{site.baseurl}}/user/tutorials/tutorial-keep.html
+
+!(screenshot)c16.png!
+
+h3. 20. Run the workflow
+
+!(screenshot)c17.png!
+
+h3. 21. Monitor progress (may take several minutes)
+
+!(screenshot)c18.png!
+
+h3. 22. Get workflow output
+
+!(screenshot)c19.png!
enableReuse: false
cwltool:Secrets:
secrets: [input1, input2]
+ cwltool:TimeLimit:
+ timelimit: 14400
+ arv:WorkflowRunnerResources:
+ ramMin: 2048
+ coresMin: 2
</pre>
The one exception to this is @arv:APIRequirement@, see note below.
table(table table-bordered table-condensed).
|_. Field |_. Type |_. Description |
|secrets|array<string>|Input parameters which are considered "secret". Must be strings.|
+
+
+h2. cwltool:TimeLimit
+
+Set an upper limit on the execution time of a CommandLineTool or ExpressionTool. A tool execution which exceeds the time limit may be preemptively terminated and considered failed. May also be used by batch systems to make scheduling decisions.
+
+table(table table-bordered table-condensed).
+|_. Field |_. Type |_. Description |
+|timelimit|int|Execution time limit in seconds. If set to zero, no limit is enforced.|
+
+h2. arv:WorkflowRunnerResources
+
+Specify resource requirements for the workflow runner process (arvados-cwl-runner) that manages a workflow run. Must be applied to the top level workflow. Will also be set implicitly when using @--submit-runner-ram@ on the command line along with @--create-workflow@ or @--update-workflow@. Use this to adjust the runner's allocation if the workflow runner is getting "out of memory" exceptions or being killed by the out-of-memory (OOM) killer.
+
+table(table table-bordered table-condensed).
+|_. Field |_. Type |_. Description |
+|ramMin|int|RAM, in mebibytes, to reserve for the arvados-cwl-runner process. Default 1 GiB|
+|coresMin|int|Number of cores to reserve to the arvados-cwl-runner process. Default 1 core.|
</code></pre>
</notextile>
-The tutorial data is hosted on "https://cloud.curoverse.com":https://cloud.curoverse.com (also referred to by the identifier *qr1hi*). If you are using a different Arvados instance, you may need to copy the data to your own instance. The easiest way to do this is with "arv-copy":{{site.baseurl}}/user/topics/arv-copy.html (this requires signing up for a free cloud.curoverse.com account).
+The tutorial data is hosted on "https://playground.arvados.org":https://playground.arvados.org (also referred to by the identifier *qr1hi*). If you are using a different Arvados instance, you may need to copy the data to your own instance. The easiest way to do this is with "arv-copy":{{site.baseurl}}/user/topics/arv-copy.html (this requires signing up for a free playground.arvados.org account).
<notextile>
<pre><code>~$ <span class="userinput">arv-copy --src qr1hi --dst settings 2463fa9efeb75e099685528b3b9071e0+438</span>
</code></pre>
</notextile>
-If you do not wish to create an account on "https://cloud.curoverse.com":https://cloud.curoverse.com, you may download the files anonymously and upload them to your local Arvados instance:
+If you do not wish to create an account on "https://playground.arvados.org":https://playground.arvados.org, you may download the files anonymously and upload them to your local Arvados instance:
-"https://cloud.curoverse.com/collections/2463fa9efeb75e099685528b3b9071e0+438":https://cloud.curoverse.com/collections/2463fa9efeb75e099685528b3b9071e0+438
+"https://playground.arvados.org/collections/2463fa9efeb75e099685528b3b9071e0+438":https://playground.arvados.org/collections/2463fa9efeb75e099685528b3b9071e0+438
-"https://cloud.curoverse.com/collections/ae480c5099b81e17267b7445e35b4bc7+180":https://cloud.curoverse.com/collections/ae480c5099b81e17267b7445e35b4bc7+180
+"https://playground.arvados.org/collections/ae480c5099b81e17267b7445e35b4bc7+180":https://playground.arvados.org/collections/ae480c5099b81e17267b7445e35b4bc7+180
-"https://cloud.curoverse.com/collections/655c6cd07550151b210961ed1d3852cf+57":https://cloud.curoverse.com/collections/655c6cd07550151b210961ed1d3852cf+57
+"https://playground.arvados.org/collections/655c6cd07550151b210961ed1d3852cf+57":https://playground.arvados.org/collections/655c6cd07550151b210961ed1d3852cf+57
h2. Submitting a workflow to an Arvados cluster
tmpdirMin: 90000
</pre>
+* Available compute nodes types vary over time and across different cloud providers, so try to limit the RAM requirement to what the program actually needs. However, if you need to target a specific compute node type, see this discussion on "calculating RAM request and choosing instance type for containers.":{{site.baseurl}}/api/execution.html#RAM
+
* Instead of scattering separate steps, prefer to scatter over a subworkflow.
With the following pattern, @step1@ has to wait for all samples to complete before @step2@ can start computing on any samples. This means a single long-running sample can prevent the rest of the workflow from moving on:
run: tool3.cwl
</pre>
-* When migrating from crunch v1 API (--api=jobs) to the crunch v2 API (--api=containers) there are a few differences in behavior:
+h2(#migrate). Migrating running CWL on jobs API to containers API
+
+* When migrating from jobs API (--api=jobs) (sometimes referred to as "crunch v1") to the containers API (--api=containers) ("crunch v2") there are a few differences in behavior:
** The tool is limited to accessing only collections which are explicitly listed in the input, and further limited to only the subdirectories of collections listed in input. For example, given an explicit file input @/dir/subdir/file1.txt@, a tool will not be able to implicitly access the file @/dir/file2.txt@. Use @secondaryFiles@ or a @Directory@ input to describe trees of files.
** Files listed in @InitialWorkDirRequirement@ appear in the output directory as normal files (not symlinks) but cannot be moved, renamed or deleted. These files will be added to the output collection but without any additional copies of the underlying data.
** Tools are disallowed network access by default. Tools which require network access must include @arv:APIRequirement: {}@ in their @requirements@ section.
@arv-copy@ allows users to copy collections and pipeline templates from one cluster to another. By default, @arv-copy@ will recursively go through a template and copy all dependencies associated with the object.
-For example, let's copy from the <a href="https://cloud.curoverse.com/">cloud instance *qr1hi*</a> to *dst_cluster*. The names *qr1hi* and *dst_cluster* are interchangable with any cluster name. You can find the cluster name from the prefix of the uuid of the object you want to copy. For example, in *qr1hi*-4zz18-tci4vn4fa95w0zx, the cluster name is qr1hi.
+For example, let's copy from the <a href="https://playground.arvados.org/">Arvados playground</a>, also known as *qr1hi*, to *dst_cluster*. The names *qr1hi* and *dst_cluster* are interchangable with any cluster name. You can find the cluster name from the prefix of the uuid of the object you want to copy. For example, in *qr1hi*-4zz18-tci4vn4fa95w0zx, the cluster name is qr1hi.
In order to communicate with both clusters, you must create custom configuration files for each cluster. In the Arvados Workbench, click on the dropdown menu icon <span class="fa fa-lg fa-user"></span> <span class="caret"></span> in the upper right corner of the top navigation menu to access the user settings menu, and click on the menu item *Current token*. Copy the @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ in both of your clusters. Then, create two configuration files, one for each cluster. The names of the files must have the format of *uuid_prefix.conf*. In our example, let's make two files, one for *qr1hi* and one for *dst_cluster*. From your *Current token* page in *qr1hi* and *dst_cluster*, copy the @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@.
First, select the uuid of the collection you want to copy from the source cluster. The uuid can be found in the collection display page in the collection summary area (top left box), or from the URL bar (the part after @collections/...@)
-Now copy the collection from *qr1hi* to *dst_cluster*. We will use the uuid @qr1hi-4zz18-tci4vn4fa95w0zx@ as an example. You can find this collection in the <a href="https://cloud.curoverse.com/collections/qr1hi-4zz18-tci4vn4fa95w0zx">lobSTR v.3 project on cloud.curoverse.com</a>.
+Now copy the collection from *qr1hi* to *dst_cluster*. We will use the uuid @qr1hi-4zz18-tci4vn4fa95w0zx@ as an example. You can find this collection in the <a href="https://playground.arvados.org/collections/qr1hi-4zz18-tci4vn4fa95w0zx">lobSTR v.3 project on playground.arvados.org</a>.
<notextile>
<pre><code>~$ <span class="userinput">arv-copy --src qr1hi --dst dst_cluster qr1hi-4zz18-tci4vn4fa95w0zx</span>
qr1hi-4zz18-tci4vn4fa95w0zx: 6.1M / 6.1M 100.0%
h2. Upload your image
-Finally, we are ready to upload the new Docker image to Arvados. Use @arv keep docker@ with the image repository name to upload the image. Without arguments, @arv keep docker@ will print out the list of Docker images in Arvados that are available to you.
+Finally, we are ready to upload the new Docker image to Arvados. Use @arv-keepdocker@ with the image repository name to upload the image. Without arguments, @arv-keepdocker@ will print out the list of Docker images in Arvados that are available to you.
<notextile>
-<pre><code>$ <span class="userinput">arv keep docker arvados/jobs-with-r</span>
+<pre><code>$ <span class="userinput">arv-keepdocker arvados/jobs-with-r</span>
703M / 703M 100.0%
Collection saved as 'Docker image arvados/jobs-with-r:latest 2818853ff9f9'
qr1hi-4zz18-abcdefghijklmno
-$ <span class="userinput">arv keep docker</span>
+$ <span class="userinput">arv-keepdocker</span>
REPOSITORY TAG IMAGE ID COLLECTION CREATED
arvados/jobs-with-r latest 2818853ff9f9 qr1hi-4zz18-abcdefghijklmno Tue Jan 17 20:35:53 2017
</code></pre>
h2. Share Docker images
-Docker images are subject to normal Arvados permissions. If wish to share your Docker image with others (or wish to share a pipeline template that uses your Docker image) you will need to use @arv keep docker@ with the @--project-uuid@ option to upload the image to a shared project.
+Docker images are subject to normal Arvados permissions. If wish to share your Docker image with others (or wish to share a pipeline template that uses your Docker image) you will need to use @arv-keepdocker@ with the @--project-uuid@ option to upload the image to a shared project.
<notextile>
-<pre><code>$ <span class="userinput">arv keep docker --project-uuid qr1hi-j7d0g-xxxxxxxxxxxxxxx arvados/jobs-with-r</span>
+<pre><code>$ <span class="userinput">arv-keepdocker --project-uuid qr1hi-j7d0g-xxxxxxxxxxxxxxx arvados/jobs-with-r</span>
</code></pre>
</notextile>
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: Using storage classes
+...
+
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+Storage classes (alternately known as "storage tiers") allow you to control which volumes should be used to store particular collection data blocks. This can be used to implement data storage policies such as moving data to archival storage.
+
+Names of storage classes are internal to the cluster and decided by the administrator. Aside from "default", Arvados currently does not define any standard storage class names.
+
+h3. arv-put
+
+You may specify the desired storage class for a collection uploaded using @arv-put@:
+
+<pre>
+$ arv-put --storage-classes=hot myfile.txt
+</pre>
+
+h3. arvados-cwl-runner
+
+You may also specify the desired storage class for the final output collection produced by @arvados-cwl-runner@:
+
+<pre>
+$ arvados-cwl-runner --storage-classes=hot myworkflow.cwl myinput.yml
+</pre>
+
+(Note: intermediate collections produced by a workflow run will have "default" storage class.)
+
+h3. arv command line
+
+You may set the storage class on an existing collection by setting the "storage_classes_desired" field of a Collection. For example, at the command line:
+
+<pre>
+$ arv collection update --uuid zzzzz-4zz18-dhhm0ay8k8cqkvg --collection '{"storage_classes_desired": ["archival"]}'
+</pre>
+
+By setting "storage_classes_desired" to "archival", the blocks that make up the collection will be preferentially moved to keepstore volumes which are configured with the "archival" storage class.
+
+h3. Storage class notes
+
+Collection blocks will be in the "default" storage class if not otherwise specified.
+
+Currently, a collection may only have one desired storage class.
+
+Any user with write access to a collection may set any storage class on that collection.
+
+Names of storage classes are internal to the cluster and decided by the administrator. Aside from "default", Arvados currently does not define any standard storage class names.
<pre><code>~$ <span class="userinput">git rev-parse HEAD</span></code></pre>
</notextile>
-* @"docker_image"@ : The docker image hash used is found on the "Collection page":https://cloud.curoverse.com/collections/qr1hi-4zz18-dov6im679g3jr1n as the *Content address*.
+* @"docker_image"@ : The docker image hash used is found on the "Collection page":https://playground.arvados.org/collections/qr1hi-4zz18-dov6im679g3jr1n as the *Content address*.
h2. Running your pipeline
<pre><code>~$ <span class="userinput">git rev-parse HEAD</span></code></pre>
</notextile>
-* @"docker_image"@ : This specifies the "Docker":https://www.docker.com/ runtime environment where jobs run their scripts. Docker version control is similar to git, and you can commit and push changes to your images. You must re-use the docker image hash from the previous run to use the same image. It can be found on the "Collection page":https://cloud.curoverse.com/collections/qr1hi-4zz18-dov6im679g3jr1n as the *Content address* or the *docker_image_locator* in a job's metadata.
+* @"docker_image"@ : This specifies the "Docker":https://www.docker.com/ runtime environment where jobs run their scripts. Docker version control is similar to git, and you can commit and push changes to your images. You must re-use the docker image hash from the previous run to use the same image. It can be found on the "Collection page":https://playground.arvados.org/collections/qr1hi-4zz18-dov6im679g3jr1n as the *Content address* or the *docker_image_locator* in a job's metadata.
h3. Steps
+notextile. <div class="spaced-out">
+
# Start from the *Workbench Dashboard*. You can access the Dashboard by clicking on *<i class="fa fa-lg fa-fw fa-dashboard"></i> Dashboard* in the upper left corner of any Workbench page.
# Click on the <span class="btn btn-sm btn-primary"><i class="fa fa-fw fa-gear"></i> Run a process...</span> button. This will open a dialog box titled *Choose a pipeline or workflow to run*.
# In the search box, type in *Tutorial bwa mem cwl*.
# After the process starts running, you can track the progress by watching log messages from the component(s). This page refreshes automatically. You will see a <span class="label label-success">complete</span> label when the process completes successfully.
# Click on the *Output* link to see the results of the process. This will load a new page listing the output files from this process. You'll see the output SAM file from the alignment tool under the *Files* tab.
# Click on the <span class="btn btn-sm btn-info"><i class="fa fa-download"></i></span> download button to the right of the SAM file to download your results.
+
+notextile. </div>
h2. Developing workflows
-For an introduction and and detailed documentation about writing CWL, see the "CWL User Guide":http://commonwl.org/v1.0/UserGuide.html and the "CWL Specification":http://commonwl.org/v1.0 .
+For an introduction and and detailed documentation about writing CWL, see the "CWL User Guide":https://www.commonwl.org/user_guide and the "CWL Specification":http://commonwl.org/v1.0 .
See "Best Practices for writing CWL":{{site.baseurl}}/user/cwl/cwl-style.html and "Arvados CWL Extensions":{{site.baseurl}}/user/cwl/cwl-extensions.html for additional information about using CWL on Arvados.
+h2. Using Composer
+
+You can create new workflows in the browser using "Arvados Composer":{{site.baseurl}}/user/composer/composer.html
+
h2. Registering a workflow to use in Workbench
Use @--create-workflow@ to register a CWL workflow with Arvados. This enables you to share workflows with other Arvados users, and run them by clicking the <span class="btn btn-sm btn-primary"><i class="fa fa-fw fa-gear"></i> Run a process...</span> button on the Workbench Dashboard and on the command line by UUID.
"fmt"
"io"
"io/ioutil"
+ "path/filepath"
+ "regexp"
+ "runtime"
"sort"
"strings"
)
return f(prog, args, stdin, stdout, stderr)
}
+type Version string
+
+func (v Version) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ prog = regexp.MustCompile(` -*version$`).ReplaceAllLiteralString(prog, "")
+ fmt.Fprintf(stdout, "%s %s (%s)\n", prog, v, runtime.Version())
+ return 0
+}
+
// Multi is a Handler that looks up its first argument in a map, and
// invokes the resulting Handler with the remaining args.
//
type Multi map[string]Handler
func (m Multi) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
- if len(args) < 1 {
+ _, basename := filepath.Split(prog)
+ basename = strings.TrimPrefix(basename, "arvados-")
+ basename = strings.TrimPrefix(basename, "crunch-")
+ if cmd, ok := m[basename]; ok {
+ return cmd.RunCommand(prog, args, stdin, stdout, stderr)
+ } else if len(args) < 1 {
fmt.Fprintf(stderr, "usage: %s command [args]\n", prog)
m.Usage(stderr)
return 2
- }
- if cmd, ok := m[args[0]]; !ok {
- fmt.Fprintf(stderr, "unrecognized command %q\n", args[0])
+ } else if cmd, ok = m[args[0]]; ok {
+ return cmd.RunCommand(prog+" "+args[0], args[1:], stdin, stdout, stderr)
+ } else {
+ fmt.Fprintf(stderr, "%s: unrecognized command %q\n", prog, args[0])
m.Usage(stderr)
return 2
- } else {
- return cmd.RunCommand(prog+" "+args[0], args[1:], stdin, stdout, stderr)
}
}
c.Check(stderr.String(), check.Equals, "")
}
+func (s *CmdSuite) TestHelloViaProg(c *check.C) {
+ defer cmdtest.LeakCheck(c)()
+ stdout := bytes.NewBuffer(nil)
+ stderr := bytes.NewBuffer(nil)
+ exited := testCmd.RunCommand("/usr/local/bin/echo", []string{"hello", "world"}, bytes.NewReader(nil), stdout, stderr)
+ c.Check(exited, check.Equals, 0)
+ c.Check(stdout.String(), check.Equals, "hello world\n")
+ c.Check(stderr.String(), check.Equals, "")
+}
+
func (s *CmdSuite) TestUsage(c *check.C) {
defer cmdtest.LeakCheck(c)()
stdout := bytes.NewBuffer(nil)
exited := testCmd.RunCommand("prog", []string{"nosuchcommand", "hi"}, bytes.NewReader(nil), stdout, stderr)
c.Check(exited, check.Equals, 2)
c.Check(stdout.String(), check.Equals, "")
- c.Check(stderr.String(), check.Matches, `(?ms)^unrecognized command "nosuchcommand"\n.*echo.*\n`)
+ c.Check(stderr.String(), check.Matches, `(?ms)^prog: unrecognized command "nosuchcommand"\n.*echo.*\n`)
}
func (s *CmdSuite) TestSubcommandToFront(c *check.C) {
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package controller
+
+import (
+ "git.curoverse.com/arvados.git/lib/cmd"
+ "git.curoverse.com/arvados.git/lib/service"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+)
+
+var Command cmd.Handler = service.Command(arvados.ServiceNameController, newHandler)
+
+func newHandler(cluster *arvados.Cluster, np *arvados.NodeProfile) service.Handler {
+ return &Handler{Cluster: cluster, NodeProfile: np}
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package controller
+
+import (
+ "bytes"
+ "database/sql"
+ "io/ioutil"
+ "net/http"
+ "net/url"
+ "regexp"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/auth"
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
+)
+
+var wfRe = regexp.MustCompile(`^/arvados/v1/workflows/([0-9a-z]{5})-[^/]+$`)
+
+func (h *Handler) proxyRemoteCluster(w http.ResponseWriter, req *http.Request, next http.Handler) {
+ m := wfRe.FindStringSubmatch(req.URL.Path)
+ if len(m) < 2 || m[1] == h.Cluster.ClusterID {
+ next.ServeHTTP(w, req)
+ return
+ }
+ remoteID := m[1]
+ remote, ok := h.Cluster.RemoteClusters[remoteID]
+ if !ok {
+ httpserver.Error(w, "no proxy available for cluster "+remoteID, http.StatusNotFound)
+ return
+ }
+ scheme := remote.Scheme
+ if scheme == "" {
+ scheme = "https"
+ }
+ err := h.saltAuthToken(req, remoteID)
+ if err != nil {
+ httpserver.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ urlOut := &url.URL{
+ Scheme: scheme,
+ Host: remote.Host,
+ Path: req.URL.Path,
+ RawPath: req.URL.RawPath,
+ RawQuery: req.URL.RawQuery,
+ }
+ client := h.secureClient
+ if remote.Insecure {
+ client = h.insecureClient
+ }
+ h.proxy.Do(w, req, urlOut, client)
+}
+
+// Extract the auth token supplied in req, and replace it with a
+// salted token for the remote cluster.
+func (h *Handler) saltAuthToken(req *http.Request, remote string) error {
+ creds := auth.NewCredentials()
+ creds.LoadTokensFromHTTPRequest(req)
+ if len(creds.Tokens) == 0 && req.Header.Get("Content-Type") == "application/x-www-form-encoded" {
+ // Override ParseForm's 10MiB limit by ensuring
+ // req.Body is a *http.maxBytesReader.
+ req.Body = http.MaxBytesReader(nil, req.Body, 1<<28) // 256MiB. TODO: use MaxRequestSize from discovery doc or config.
+ if err := creds.LoadTokensFromHTTPRequestBody(req); err != nil {
+ return err
+ }
+ // Replace req.Body with a buffer that re-encodes the
+ // form without api_token, in case we end up
+ // forwarding the request.
+ if req.PostForm != nil {
+ req.PostForm.Del("api_token")
+ }
+ req.Body = ioutil.NopCloser(bytes.NewBufferString(req.PostForm.Encode()))
+ }
+ if len(creds.Tokens) == 0 {
+ return nil
+ }
+ token, err := auth.SaltToken(creds.Tokens[0], remote)
+ if err == auth.ErrObsoleteToken {
+ // If the token exists in our own database, salt it
+ // for the remote. Otherwise, assume it was issued by
+ // the remote, and pass it through unmodified.
+ db, err := h.db(req)
+ if err != nil {
+ return err
+ }
+ aca := arvados.APIClientAuthorization{APIToken: creds.Tokens[0]}
+ err = db.QueryRowContext(req.Context(), `SELECT uuid FROM api_client_authorizations WHERE api_token=$1 AND (expires_at IS NULL OR expires_at > current_timestamp) LIMIT 1`, aca.APIToken).Scan(&aca.UUID)
+ if err == sql.ErrNoRows {
+ // Not ours; pass through unmodified.
+ token = aca.APIToken
+ } else if err != nil {
+ return err
+ } else {
+ // Found; make V2 version and salt it.
+ token, err = auth.SaltToken(aca.TokenV2(), remote)
+ if err != nil {
+ return err
+ }
+ }
+ } else if err != nil {
+ return err
+ }
+ req.Header.Set("Authorization", "Bearer "+token)
+
+ // Remove api_token=... from the the query string, in case we
+ // end up forwarding the request.
+ if values, err := url.ParseQuery(req.URL.RawQuery); err != nil {
+ return err
+ } else if _, ok := values["api_token"]; ok {
+ delete(values, "api_token")
+ req.URL.RawQuery = values.Encode()
+ }
+ return nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package controller
+
+import (
+ "encoding/json"
+ "io/ioutil"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strings"
+ "time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
+ "github.com/Sirupsen/logrus"
+ check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+var _ = check.Suite(&FederationSuite{})
+
+type FederationSuite struct {
+ log *logrus.Logger
+ // testServer and testHandler are the controller being tested,
+ // "zhome".
+ testServer *httpserver.Server
+ testHandler *Handler
+ // remoteServer ("zzzzz") forwards requests to the Rails API
+ // provided by the integration test environment.
+ remoteServer *httpserver.Server
+ // remoteMock ("zmock") appends each incoming request to
+ // remoteMockRequests, and returns an empty 200 response.
+ remoteMock *httpserver.Server
+ remoteMockRequests []http.Request
+}
+
+func (s *FederationSuite) SetUpTest(c *check.C) {
+ s.log = logrus.New()
+ s.log.Formatter = &logrus.JSONFormatter{}
+ s.log.Out = &logWriter{c.Log}
+
+ s.remoteServer = newServerFromIntegrationTestEnv(c)
+ c.Assert(s.remoteServer.Start(), check.IsNil)
+
+ s.remoteMock = newServerFromIntegrationTestEnv(c)
+ s.remoteMock.Server.Handler = http.HandlerFunc(s.remoteMockHandler)
+ c.Assert(s.remoteMock.Start(), check.IsNil)
+
+ nodeProfile := arvados.NodeProfile{
+ Controller: arvados.SystemServiceInstance{Listen: ":"},
+ RailsAPI: arvados.SystemServiceInstance{Listen: ":1"}, // local reqs will error "connection refused"
+ }
+ s.testHandler = &Handler{Cluster: &arvados.Cluster{
+ ClusterID: "zhome",
+ PostgreSQL: integrationTestCluster().PostgreSQL,
+ NodeProfiles: map[string]arvados.NodeProfile{
+ "*": nodeProfile,
+ },
+ }, NodeProfile: &nodeProfile}
+ s.testServer = newServerFromIntegrationTestEnv(c)
+ s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.log, s.testHandler))
+
+ s.testHandler.Cluster.RemoteClusters = map[string]arvados.RemoteCluster{
+ "zzzzz": {
+ Host: s.remoteServer.Addr,
+ Proxy: true,
+ Scheme: "http",
+ },
+ "zmock": {
+ Host: s.remoteMock.Addr,
+ Proxy: true,
+ Scheme: "http",
+ },
+ }
+
+ c.Assert(s.testServer.Start(), check.IsNil)
+
+ s.remoteMockRequests = nil
+}
+
+func (s *FederationSuite) remoteMockHandler(w http.ResponseWriter, req *http.Request) {
+ s.remoteMockRequests = append(s.remoteMockRequests, *req)
+}
+
+func (s *FederationSuite) TearDownTest(c *check.C) {
+ if s.remoteServer != nil {
+ s.remoteServer.Close()
+ }
+ if s.testServer != nil {
+ s.testServer.Close()
+ }
+}
+
+func (s *FederationSuite) testRequest(req *http.Request) *http.Response {
+ resp := httptest.NewRecorder()
+ s.testServer.Server.Handler.ServeHTTP(resp, req)
+ return resp.Result()
+}
+
+func (s *FederationSuite) TestLocalRequest(c *check.C) {
+ req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zhome-", 1), nil)
+ resp := s.testRequest(req)
+ s.checkHandledLocally(c, resp)
+}
+
+func (s *FederationSuite) checkHandledLocally(c *check.C, resp *http.Response) {
+ // Our "home" controller can't handle local requests because
+ // it doesn't have its own stub/test Rails API, so we rely on
+ // "connection refused" to indicate the controller tried to
+ // proxy the request to its local Rails API.
+ c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
+ s.checkJSONErrorMatches(c, resp, `.*connection refused`)
+}
+
+func (s *FederationSuite) TestNoAuth(c *check.C) {
+ req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
+ resp := s.testRequest(req)
+ c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
+ s.checkJSONErrorMatches(c, resp, `Not logged in`)
+}
+
+func (s *FederationSuite) TestBadAuth(c *check.C) {
+ req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
+ req.Header.Set("Authorization", "Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+ resp := s.testRequest(req)
+ c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
+ s.checkJSONErrorMatches(c, resp, `Not logged in`)
+}
+
+func (s *FederationSuite) TestNoAccess(c *check.C) {
+ req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
+ req.Header.Set("Authorization", "Bearer "+arvadostest.SpectatorToken)
+ resp := s.testRequest(req)
+ c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
+ s.checkJSONErrorMatches(c, resp, `.*not found`)
+}
+
+func (s *FederationSuite) TestGetUnknownRemote(c *check.C) {
+ req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zz404-", 1), nil)
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp := s.testRequest(req)
+ c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
+ s.checkJSONErrorMatches(c, resp, `.*no proxy available for cluster zz404`)
+}
+
+func (s *FederationSuite) TestRemoteError(c *check.C) {
+ rc := s.testHandler.Cluster.RemoteClusters["zzzzz"]
+ rc.Scheme = "https"
+ s.testHandler.Cluster.RemoteClusters["zzzzz"] = rc
+
+ req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp := s.testRequest(req)
+ c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
+ s.checkJSONErrorMatches(c, resp, `.*HTTP response to HTTPS client`)
+}
+
+func (s *FederationSuite) TestGetRemoteWorkflow(c *check.C) {
+ req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp := s.testRequest(req)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ var wf arvados.Workflow
+ c.Check(json.NewDecoder(resp.Body).Decode(&wf), check.IsNil)
+ c.Check(wf.UUID, check.Equals, arvadostest.WorkflowWithDefinitionYAMLUUID)
+ c.Check(wf.OwnerUUID, check.Equals, arvadostest.ActiveUserUUID)
+}
+
+func (s *FederationSuite) TestOptionsMethod(c *check.C) {
+ req := httptest.NewRequest("OPTIONS", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
+ req.Header.Set("Origin", "https://example.com")
+ resp := s.testRequest(req)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ body, err := ioutil.ReadAll(resp.Body)
+ c.Check(err, check.IsNil)
+ c.Check(string(body), check.Equals, "")
+ c.Check(resp.Header.Get("Access-Control-Allow-Origin"), check.Equals, "*")
+ for _, hdr := range []string{"Authorization", "Content-Type"} {
+ c.Check(resp.Header.Get("Access-Control-Allow-Headers"), check.Matches, ".*"+hdr+".*")
+ }
+ for _, method := range []string{"GET", "HEAD", "PUT", "POST", "DELETE"} {
+ c.Check(resp.Header.Get("Access-Control-Allow-Methods"), check.Matches, ".*"+method+".*")
+ }
+}
+
+func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
+ req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1)+"?api_token="+arvadostest.ActiveToken, nil)
+ s.testRequest(req)
+ c.Assert(len(s.remoteMockRequests), check.Equals, 1)
+ pr := s.remoteMockRequests[0]
+ // Token is salted and moved from query to Authorization header.
+ c.Check(pr.URL.String(), check.Not(check.Matches), `.*api_token=.*`)
+ c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
+}
+
+func (s *FederationSuite) TestLocalTokenSalted(c *check.C) {
+ req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ s.testRequest(req)
+ c.Assert(len(s.remoteMockRequests), check.Equals, 1)
+ pr := s.remoteMockRequests[0]
+ // The salted token here has a "zzzzz-" UUID instead of a
+ // "ztest-" UUID because ztest's local database has the
+ // "zzzzz-" test fixtures. The "secret" part is HMAC(sha1,
+ // arvadostest.ActiveToken, "zmock") = "7fd3...".
+ c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
+}
+
+func (s *FederationSuite) TestRemoteTokenNotSalted(c *check.C) {
+ // remoteToken can be any v1 token that doesn't appear in
+ // ztest's local db.
+ remoteToken := "abcdef00000000000000000000000000000000000000000000"
+ req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
+ req.Header.Set("Authorization", "Bearer "+remoteToken)
+ s.testRequest(req)
+ c.Assert(len(s.remoteMockRequests), check.Equals, 1)
+ pr := s.remoteMockRequests[0]
+ c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
+}
+
+func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
+ wf := arvados.Workflow{
+ Description: "TestCRUD",
+ }
+ {
+ body := &strings.Builder{}
+ json.NewEncoder(body).Encode(&wf)
+ req := httptest.NewRequest("POST", "/arvados/v1/workflows", strings.NewReader(url.Values{
+ "workflow": {body.String()},
+ }.Encode()))
+ req.Header.Set("Content-type", "application/x-www-form-urlencoded")
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ rec := httptest.NewRecorder()
+ s.remoteServer.Server.Handler.ServeHTTP(rec, req) // direct to remote -- can't proxy a create req because no uuid
+ resp := rec.Result()
+ s.checkResponseOK(c, resp)
+ json.NewDecoder(resp.Body).Decode(&wf)
+
+ defer func() {
+ req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ s.remoteServer.Server.Handler.ServeHTTP(httptest.NewRecorder(), req)
+ }()
+ c.Check(wf.UUID, check.Not(check.Equals), "")
+
+ c.Assert(wf.ModifiedAt, check.NotNil)
+ c.Logf("wf.ModifiedAt: %v", wf.ModifiedAt)
+ c.Check(time.Since(*wf.ModifiedAt) < time.Minute, check.Equals, true)
+ }
+ for _, method := range []string{"PATCH", "PUT", "POST"} {
+ form := url.Values{
+ "workflow": {`{"description": "Updated with ` + method + `"}`},
+ }
+ if method == "POST" {
+ form["_method"] = []string{"PATCH"}
+ }
+ req := httptest.NewRequest(method, "/arvados/v1/workflows/"+wf.UUID, strings.NewReader(form.Encode()))
+ req.Header.Set("Content-type", "application/x-www-form-urlencoded")
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp := s.testRequest(req)
+ s.checkResponseOK(c, resp)
+ err := json.NewDecoder(resp.Body).Decode(&wf)
+ c.Check(err, check.IsNil)
+
+ c.Check(wf.Description, check.Equals, "Updated with "+method)
+ }
+ {
+ req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp := s.testRequest(req)
+ s.checkResponseOK(c, resp)
+ err := json.NewDecoder(resp.Body).Decode(&wf)
+ c.Check(err, check.IsNil)
+ }
+ {
+ req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+wf.UUID, nil)
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp := s.testRequest(req)
+ c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
+ }
+}
+
+func (s *FederationSuite) checkResponseOK(c *check.C, resp *http.Response) {
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ if resp.StatusCode != http.StatusOK {
+ body, err := ioutil.ReadAll(resp.Body)
+ c.Logf("... response body = %q, %v\n", body, err)
+ }
+}
+
+func (s *FederationSuite) checkJSONErrorMatches(c *check.C, resp *http.Response, re string) {
+ var jresp httpserver.ErrorResponse
+ err := json.NewDecoder(resp.Body).Decode(&jresp)
+ c.Check(err, check.IsNil)
+ c.Assert(len(jresp.Errors), check.Equals, 1)
+ c.Check(jresp.Errors[0], check.Matches, re)
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package controller
+
+import (
+ "database/sql"
+ "errors"
+ "net"
+ "net/http"
+ "net/url"
+ "strings"
+ "sync"
+ "time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/health"
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
+ _ "github.com/lib/pq"
+)
+
+type Handler struct {
+ Cluster *arvados.Cluster
+ NodeProfile *arvados.NodeProfile
+
+ setupOnce sync.Once
+ handlerStack http.Handler
+ proxy *proxy
+ secureClient *http.Client
+ insecureClient *http.Client
+ pgdb *sql.DB
+ pgdbMtx sync.Mutex
+}
+
+func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+ h.setupOnce.Do(h.setup)
+ if req.Method != "GET" && req.Method != "HEAD" {
+ // http.ServeMux returns 301 with a cleaned path if
+ // the incoming request has a double slash. Some
+ // clients (including the Go standard library) change
+ // the request method to GET when following a 301
+ // redirect if the original method was not HEAD
+ // (RFC7231 6.4.2 specifically allows this in the case
+ // of POST). Thus "POST //foo" gets misdirected to
+ // "GET /foo". To avoid this, eliminate double slashes
+ // before passing the request to ServeMux.
+ for strings.Contains(req.URL.Path, "//") {
+ req.URL.Path = strings.Replace(req.URL.Path, "//", "/", -1)
+ }
+ }
+ h.handlerStack.ServeHTTP(w, req)
+}
+
+func (h *Handler) CheckHealth() error {
+ h.setupOnce.Do(h.setup)
+ _, _, err := findRailsAPI(h.Cluster, h.NodeProfile)
+ return err
+}
+
+func neverRedirect(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
+
+func (h *Handler) setup() {
+ mux := http.NewServeMux()
+ mux.Handle("/_health/", &health.Handler{
+ Token: h.Cluster.ManagementToken,
+ Prefix: "/_health/",
+ })
+ hs := http.NotFoundHandler()
+ hs = prepend(hs, h.proxyRailsAPI)
+ hs = prepend(hs, h.proxyRemoteCluster)
+ mux.Handle("/", hs)
+ h.handlerStack = mux
+
+ sc := *arvados.DefaultSecureClient
+ sc.Timeout = time.Duration(h.Cluster.HTTPRequestTimeout)
+ sc.CheckRedirect = neverRedirect
+ h.secureClient = &sc
+
+ ic := *arvados.InsecureHTTPClient
+ ic.Timeout = time.Duration(h.Cluster.HTTPRequestTimeout)
+ ic.CheckRedirect = neverRedirect
+ h.insecureClient = &ic
+
+ h.proxy = &proxy{
+ Name: "arvados-controller",
+ RequestTimeout: time.Duration(h.Cluster.HTTPRequestTimeout),
+ }
+}
+
+var errDBConnection = errors.New("database connection error")
+
+func (h *Handler) db(req *http.Request) (*sql.DB, error) {
+ h.pgdbMtx.Lock()
+ defer h.pgdbMtx.Unlock()
+ if h.pgdb != nil {
+ return h.pgdb, nil
+ }
+
+ db, err := sql.Open("postgres", h.Cluster.PostgreSQL.Connection.String())
+ if err != nil {
+ httpserver.Logger(req).WithError(err).Error("postgresql connect failed")
+ return nil, errDBConnection
+ }
+ if p := h.Cluster.PostgreSQL.ConnectionPool; p > 0 {
+ db.SetMaxOpenConns(p)
+ }
+ if err := db.Ping(); err != nil {
+ httpserver.Logger(req).WithError(err).Error("postgresql connect succeeded but ping failed")
+ return nil, errDBConnection
+ }
+ h.pgdb = db
+ return db, nil
+}
+
+type middlewareFunc func(http.ResponseWriter, *http.Request, http.Handler)
+
+func prepend(next http.Handler, middleware middlewareFunc) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ middleware(w, req, next)
+ })
+}
+
+func (h *Handler) proxyRailsAPI(w http.ResponseWriter, req *http.Request, next http.Handler) {
+ urlOut, insecure, err := findRailsAPI(h.Cluster, h.NodeProfile)
+ if err != nil {
+ httpserver.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ urlOut = &url.URL{
+ Scheme: urlOut.Scheme,
+ Host: urlOut.Host,
+ Path: req.URL.Path,
+ RawPath: req.URL.RawPath,
+ RawQuery: req.URL.RawQuery,
+ }
+ client := h.secureClient
+ if insecure {
+ client = h.insecureClient
+ }
+ h.proxy.Do(w, req, urlOut, client)
+}
+
+// For now, findRailsAPI always uses the rails API running on this
+// node.
+func findRailsAPI(cluster *arvados.Cluster, np *arvados.NodeProfile) (*url.URL, bool, error) {
+ hostport := np.RailsAPI.Listen
+ if len(hostport) > 1 && hostport[0] == ':' && strings.TrimRight(hostport[1:], "0123456789") == "" {
+ // ":12345" => connect to indicated port on localhost
+ hostport = "localhost" + hostport
+ } else if _, _, err := net.SplitHostPort(hostport); err == nil {
+ // "[::1]:12345" => connect to indicated address & port
+ } else {
+ return nil, false, err
+ }
+ proto := "http"
+ if np.RailsAPI.TLS {
+ proto = "https"
+ }
+ url, err := url.Parse(proto + "://" + hostport)
+ return url, np.RailsAPI.Insecure, err
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package controller
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "os"
+ "strings"
+ "testing"
+ "time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
+ check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&HandlerSuite{})
+
+type HandlerSuite struct {
+ cluster *arvados.Cluster
+ handler http.Handler
+}
+
+func (s *HandlerSuite) SetUpTest(c *check.C) {
+ s.cluster = &arvados.Cluster{
+ ClusterID: "zzzzz",
+ PostgreSQL: integrationTestCluster().PostgreSQL,
+ NodeProfiles: map[string]arvados.NodeProfile{
+ "*": {
+ Controller: arvados.SystemServiceInstance{Listen: ":"},
+ RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"), TLS: true, Insecure: true},
+ },
+ },
+ }
+ node := s.cluster.NodeProfiles["*"]
+ s.handler = newHandler(s.cluster, &node)
+}
+
+func (s *HandlerSuite) TestProxyDiscoveryDoc(c *check.C) {
+ req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ var dd arvados.DiscoveryDocument
+ err := json.Unmarshal(resp.Body.Bytes(), &dd)
+ c.Check(err, check.IsNil)
+ c.Check(dd.BlobSignatureTTL, check.Not(check.Equals), int64(0))
+ c.Check(dd.BlobSignatureTTL > 0, check.Equals, true)
+ c.Check(len(dd.Resources), check.Not(check.Equals), 0)
+ c.Check(len(dd.Schemas), check.Not(check.Equals), 0)
+}
+
+func (s *HandlerSuite) TestRequestTimeout(c *check.C) {
+ s.cluster.HTTPRequestTimeout = arvados.Duration(time.Nanosecond)
+ req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusBadGateway)
+ var jresp httpserver.ErrorResponse
+ err := json.Unmarshal(resp.Body.Bytes(), &jresp)
+ c.Check(err, check.IsNil)
+ c.Assert(len(jresp.Errors), check.Equals, 1)
+ c.Check(jresp.Errors[0], check.Matches, `.*context deadline exceeded.*`)
+}
+
+func (s *HandlerSuite) TestProxyWithoutToken(c *check.C) {
+ req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
+ jresp := map[string]interface{}{}
+ err := json.Unmarshal(resp.Body.Bytes(), &jresp)
+ c.Check(err, check.IsNil)
+ c.Check(jresp["errors"], check.FitsTypeOf, []interface{}{})
+}
+
+func (s *HandlerSuite) TestProxyWithToken(c *check.C) {
+ req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ var u arvados.User
+ err := json.Unmarshal(resp.Body.Bytes(), &u)
+ c.Check(err, check.IsNil)
+ c.Check(u.UUID, check.Equals, arvadostest.ActiveUserUUID)
+}
+
+func (s *HandlerSuite) TestProxyWithTokenInRequestBody(c *check.C) {
+ req := httptest.NewRequest("POST", "/arvados/v1/users/current", strings.NewReader(url.Values{
+ "_method": {"GET"},
+ "api_token": {arvadostest.ActiveToken},
+ }.Encode()))
+ req.Header.Set("Content-type", "application/x-www-form-urlencoded")
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ var u arvados.User
+ err := json.Unmarshal(resp.Body.Bytes(), &u)
+ c.Check(err, check.IsNil)
+ c.Check(u.UUID, check.Equals, arvadostest.ActiveUserUUID)
+}
+
+func (s *HandlerSuite) TestProxyNotFound(c *check.C) {
+ req := httptest.NewRequest("GET", "/arvados/v1/xyzzy", nil)
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusNotFound)
+ jresp := map[string]interface{}{}
+ err := json.Unmarshal(resp.Body.Bytes(), &jresp)
+ c.Check(err, check.IsNil)
+ c.Check(jresp["errors"], check.FitsTypeOf, []interface{}{})
+}
+
+func (s *HandlerSuite) TestProxyRedirect(c *check.C) {
+ req := httptest.NewRequest("GET", "https://0.0.0.0:1/login?return_to=foo", nil)
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusFound)
+ c.Check(resp.Header().Get("Location"), check.Matches, `https://0.0.0.0:1/auth/joshid\?return_to=foo&?`)
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package controller
+
+import (
+ "context"
+ "io"
+ "net/http"
+ "net/url"
+ "time"
+
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
+)
+
+type proxy struct {
+ Name string // to use in Via header
+ RequestTimeout time.Duration
+}
+
+// headers that shouldn't be forwarded when proxying. See
+// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers
+var dropHeaders = map[string]bool{
+ "Connection": true,
+ "Keep-Alive": true,
+ "Proxy-Authenticate": true,
+ "Proxy-Authorization": true,
+ "TE": true,
+ "Trailer": true,
+ "Transfer-Encoding": true,
+ "Upgrade": true,
+}
+
+func (p *proxy) Do(w http.ResponseWriter, reqIn *http.Request, urlOut *url.URL, client *http.Client) {
+ // Copy headers from incoming request, then add/replace proxy
+ // headers like Via and X-Forwarded-For.
+ hdrOut := http.Header{}
+ for k, v := range reqIn.Header {
+ if !dropHeaders[k] {
+ hdrOut[k] = v
+ }
+ }
+ xff := reqIn.RemoteAddr
+ if xffIn := reqIn.Header.Get("X-Forwarded-For"); xffIn != "" {
+ xff = xffIn + "," + xff
+ }
+ hdrOut.Set("X-Forwarded-For", xff)
+ if hdrOut.Get("X-Forwarded-Proto") == "" {
+ hdrOut.Set("X-Forwarded-Proto", reqIn.URL.Scheme)
+ }
+ hdrOut.Add("Via", reqIn.Proto+" arvados-controller")
+
+ ctx := reqIn.Context()
+ if p.RequestTimeout > 0 {
+ var cancel context.CancelFunc
+ ctx, cancel = context.WithDeadline(ctx, time.Now().Add(time.Duration(p.RequestTimeout)))
+ defer cancel()
+ }
+
+ reqOut := (&http.Request{
+ Method: reqIn.Method,
+ URL: urlOut,
+ Host: reqIn.Host,
+ Header: hdrOut,
+ Body: reqIn.Body,
+ }).WithContext(ctx)
+ resp, err := client.Do(reqOut)
+ if err != nil {
+ httpserver.Error(w, err.Error(), http.StatusBadGateway)
+ return
+ }
+ for k, v := range resp.Header {
+ for _, v := range v {
+ w.Header().Add(k, v)
+ }
+ }
+ w.WriteHeader(resp.StatusCode)
+ n, err := io.Copy(w, resp.Body)
+ if err != nil {
+ httpserver.Logger(reqIn).WithError(err).WithField("bytesCopied", n).Error("error copying response body")
+ }
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package controller
+
+import (
+ "bytes"
+ "net/http"
+ "os"
+ "path/filepath"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
+ "github.com/Sirupsen/logrus"
+ check "gopkg.in/check.v1"
+)
+
+// logWriter is an io.Writer that writes by calling a "write log"
+// function, typically (*check.C)Log().
+type logWriter struct {
+ logfunc func(...interface{})
+}
+
+func (tl *logWriter) Write(buf []byte) (int, error) {
+ tl.logfunc(string(bytes.TrimRight(buf, "\n")))
+ return len(buf), nil
+}
+
+func integrationTestCluster() *arvados.Cluster {
+ cfg, err := arvados.GetConfig(filepath.Join(os.Getenv("WORKSPACE"), "tmp", "arvados.yml"))
+ if err != nil {
+ panic(err)
+ }
+ cc, err := cfg.GetCluster("zzzzz")
+ if err != nil {
+ panic(err)
+ }
+ return cc
+}
+
+// Return a new unstarted controller server, using the Rails API
+// provided by the integration-testing environment.
+func newServerFromIntegrationTestEnv(c *check.C) *httpserver.Server {
+ log := logrus.New()
+ log.Formatter = &logrus.JSONFormatter{}
+ log.Out = &logWriter{c.Log}
+
+ nodeProfile := arvados.NodeProfile{
+ Controller: arvados.SystemServiceInstance{Listen: ":"},
+ RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"), TLS: true, Insecure: true},
+ }
+ handler := &Handler{Cluster: &arvados.Cluster{
+ ClusterID: "zzzzz",
+ PostgreSQL: integrationTestCluster().PostgreSQL,
+ NodeProfiles: map[string]arvados.NodeProfile{
+ "*": nodeProfile,
+ },
+ }, NodeProfile: &nodeProfile}
+
+ srv := &httpserver.Server{
+ Server: http.Server{
+ Handler: httpserver.AddRequestIDs(httpserver.LogRequests(log, handler)),
+ },
+ Addr: nodeProfile.Controller.Listen,
+ }
+ return srv
+}
"os"
"strconv"
"strings"
+ "syscall"
"time"
)
// Interval between samples. Must be positive.
PollPeriod time.Duration
+ // Temporary directory, will be monitored for available, used & total space.
+ TempDir string
+
// Where to write statistics. Must not be nil.
Logger *log.Logger
- reportedStatFile map[string]string
- lastNetSample map[string]ioSample
- lastDiskSample map[string]ioSample
- lastCPUSample cpuSample
+ reportedStatFile map[string]string
+ lastNetSample map[string]ioSample
+ lastDiskIOSample map[string]ioSample
+ lastCPUSample cpuSample
+ lastDiskSpaceSample diskSpaceSample
done chan struct{} // closed when we should stop reporting
flushed chan struct{} // closed when we have made our last report
continue
}
delta := ""
- if prev, ok := r.lastDiskSample[dev]; ok {
+ if prev, ok := r.lastDiskIOSample[dev]; ok {
delta = fmt.Sprintf(" -- interval %.4f seconds %d write %d read",
sample.sampleTime.Sub(prev.sampleTime).Seconds(),
sample.txBytes-prev.txBytes,
sample.rxBytes-prev.rxBytes)
}
r.Logger.Printf("blkio:%s %d write %d read%s\n", dev, sample.txBytes, sample.rxBytes, delta)
- r.lastDiskSample[dev] = sample
+ r.lastDiskIOSample[dev] = sample
}
}
}
}
+type diskSpaceSample struct {
+ hasData bool
+ sampleTime time.Time
+ total uint64
+ used uint64
+ available uint64
+}
+
+func (r *Reporter) doDiskSpaceStats() {
+ s := syscall.Statfs_t{}
+ err := syscall.Statfs(r.TempDir, &s)
+ if err != nil {
+ return
+ }
+ bs := uint64(s.Bsize)
+ nextSample := diskSpaceSample{
+ hasData: true,
+ sampleTime: time.Now(),
+ total: s.Blocks * bs,
+ used: (s.Blocks - s.Bfree) * bs,
+ available: s.Bavail * bs,
+ }
+
+ var delta string
+ if r.lastDiskSpaceSample.hasData {
+ prev := r.lastDiskSpaceSample
+ interval := nextSample.sampleTime.Sub(prev.sampleTime).Seconds()
+ delta = fmt.Sprintf(" -- interval %.4f seconds %d used",
+ interval,
+ int64(nextSample.used-prev.used))
+ }
+ r.Logger.Printf("statfs %d available %d used %d total%s\n",
+ nextSample.available, nextSample.used, nextSample.total, delta)
+ r.lastDiskSpaceSample = nextSample
+}
+
type cpuSample struct {
hasData bool // to distinguish the zero value from real data
sampleTime time.Time
}
r.lastNetSample = make(map[string]ioSample)
- r.lastDiskSample = make(map[string]ioSample)
+ r.lastDiskIOSample = make(map[string]ioSample)
+
+ if len(r.TempDir) == 0 {
+ // Temporary dir not provided, try to get it from the environment.
+ r.TempDir = os.Getenv("TMPDIR")
+ }
+ if len(r.TempDir) > 0 {
+ r.Logger.Printf("notice: monitoring temp dir %s\n", r.TempDir)
+ }
ticker := time.NewTicker(r.PollPeriod)
for {
r.doCPUStats()
r.doBlkIOStats()
r.doNetworkStats()
+ r.doDiskSpaceStats()
select {
case <-r.done:
return
"errors"
"log"
"os/exec"
+ "sort"
"strings"
"time"
)
var (
- ErrConstraintsNotSatisfiable = errors.New("constraints not satisfiable by any configured instance type")
ErrInstanceTypesNotConfigured = errors.New("site configuration does not list any instance types")
discountConfiguredRAMPercent = 5
)
+// ConstraintsNotSatisfiableError includes a list of available instance types
+// to be reported back to the user.
+type ConstraintsNotSatisfiableError struct {
+ error
+ AvailableTypes []arvados.InstanceType
+}
+
// ChooseInstanceType returns the cheapest available
// arvados.InstanceType big enough to run ctr.
func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvados.InstanceType, err error) {
needRAM := ctr.RuntimeConstraints.RAM + ctr.RuntimeConstraints.KeepCacheRAM
needRAM = (needRAM * 100) / int64(100-discountConfiguredRAMPercent)
- err = ErrConstraintsNotSatisfiable
+ ok := false
for _, it := range cc.InstanceTypes {
switch {
- case err == nil && it.Price > best.Price:
- case it.Scratch < needScratch:
- case it.RAM < needRAM:
+ case ok && it.Price > best.Price:
+ case int64(it.Scratch) < needScratch:
+ case int64(it.RAM) < needRAM:
case it.VCPUs < needVCPUs:
+ case it.Preemptible != ctr.SchedulingParameters.Preemptible:
case it.Price == best.Price && (it.RAM < best.RAM || it.VCPUs < best.VCPUs):
// Equal price, but worse specs
default:
// Lower price || (same price && better specs)
best = it
- err = nil
+ ok = true
+ }
+ }
+ if !ok {
+ availableTypes := make([]arvados.InstanceType, 0, len(cc.InstanceTypes))
+ for _, t := range cc.InstanceTypes {
+ availableTypes = append(availableTypes, t)
+ }
+ sort.Slice(availableTypes, func(a, b int) bool {
+ return availableTypes[a].Price < availableTypes[b].Price
+ })
+ err = ConstraintsNotSatisfiableError{
+ errors.New("constraints not satisfiable by any configured instance type"),
+ availableTypes,
}
+ return
}
return
}
var _ = check.Suite(&NodeSizeSuite{})
-const GiB = int64(1 << 30)
+const GiB = arvados.ByteSize(1 << 30)
type NodeSizeSuite struct{}
func (*NodeSizeSuite) TestChooseUnsatisfiable(c *check.C) {
checkUnsatisfiable := func(ctr *arvados.Container) {
- _, err := ChooseInstanceType(&arvados.Cluster{InstanceTypes: []arvados.InstanceType{
- {Price: 1.1, RAM: 1000000000, VCPUs: 2, Name: "small1"},
- {Price: 2.2, RAM: 2000000000, VCPUs: 4, Name: "small2"},
- {Price: 4.4, RAM: 4000000000, VCPUs: 8, Name: "small4", Scratch: GiB},
+ _, err := ChooseInstanceType(&arvados.Cluster{InstanceTypes: map[string]arvados.InstanceType{
+ "small1": {Price: 1.1, RAM: 1000000000, VCPUs: 2, Name: "small1"},
+ "small2": {Price: 2.2, RAM: 2000000000, VCPUs: 4, Name: "small2"},
+ "small4": {Price: 4.4, RAM: 4000000000, VCPUs: 8, Name: "small4", Scratch: GiB},
}}, ctr)
- c.Check(err, check.Equals, ErrConstraintsNotSatisfiable)
+ c.Check(err, check.FitsTypeOf, ConstraintsNotSatisfiableError{})
}
for _, rc := range []arvados.RuntimeConstraints{
checkUnsatisfiable(&arvados.Container{RuntimeConstraints: rc})
}
checkUnsatisfiable(&arvados.Container{
- Mounts: map[string]arvados.Mount{"/tmp": {Kind: "tmp", Capacity: 2 * GiB}},
+ Mounts: map[string]arvados.Mount{"/tmp": {Kind: "tmp", Capacity: int64(2 * GiB)}},
RuntimeConstraints: arvados.RuntimeConstraints{RAM: 12345, VCPUs: 1},
})
}
func (*NodeSizeSuite) TestChoose(c *check.C) {
- for _, menu := range [][]arvados.InstanceType{
+ for _, menu := range []map[string]arvados.InstanceType{
{
- {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly"},
- {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
- {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Name: "small"},
+ "costly": {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly"},
+ "best": {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
+ "small": {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Name: "small"},
},
{
- {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly"},
- {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "goodenough"},
- {Price: 2.2, RAM: 4000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
- {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Name: "small"},
+ "costly": {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly"},
+ "goodenough": {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "goodenough"},
+ "best": {Price: 2.2, RAM: 4000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
+ "small": {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Name: "small"},
},
{
- {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Name: "small"},
- {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "goodenough"},
- {Price: 2.2, RAM: 4000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
- {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly"},
+ "small": {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Name: "small"},
+ "goodenough": {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "goodenough"},
+ "best": {Price: 2.2, RAM: 4000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
+ "costly": {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly"},
},
{
- {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: GiB, Name: "small"},
- {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: GiB, Name: "nearly"},
- {Price: 3.3, RAM: 4000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
- {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly"},
+ "small": {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: GiB, Name: "small"},
+ "nearly": {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: GiB, Name: "nearly"},
+ "best": {Price: 3.3, RAM: 4000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
+ "costly": {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly"},
},
} {
best, err := ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu}, &arvados.Container{
Mounts: map[string]arvados.Mount{
- "/tmp": {Kind: "tmp", Capacity: 2 * GiB},
+ "/tmp": {Kind: "tmp", Capacity: 2 * int64(GiB)},
},
RuntimeConstraints: arvados.RuntimeConstraints{
VCPUs: 2,
c.Check(best.Scratch >= 2*GiB, check.Equals, true)
}
}
+
+func (*NodeSizeSuite) TestChoosePreemptable(c *check.C) {
+ menu := map[string]arvados.InstanceType{
+ "costly": {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Preemptible: true, Name: "costly"},
+ "almost best": {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "almost best"},
+ "best": {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Preemptible: true, Name: "best"},
+ "small": {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Preemptible: true, Name: "small"},
+ }
+ best, err := ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu}, &arvados.Container{
+ Mounts: map[string]arvados.Mount{
+ "/tmp": {Kind: "tmp", Capacity: 2 * int64(GiB)},
+ },
+ RuntimeConstraints: arvados.RuntimeConstraints{
+ VCPUs: 2,
+ RAM: 987654321,
+ KeepCacheRAM: 123456789,
+ },
+ SchedulingParameters: arvados.SchedulingParameters{
+ Preemptible: true,
+ },
+ })
+ c.Check(err, check.IsNil)
+ c.Check(best.Name, check.Equals, "best")
+ c.Check(best.RAM >= 1234567890, check.Equals, true)
+ c.Check(best.VCPUs >= 2, check.Equals, true)
+ c.Check(best.Scratch >= 2*GiB, check.Equals, true)
+ c.Check(best.Preemptible, check.Equals, true)
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// package service provides a cmd.Handler that brings up a system service.
+package service
+
+import (
+ "flag"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+
+ "git.curoverse.com/arvados.git/lib/cmd"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
+ "github.com/Sirupsen/logrus"
+ "github.com/coreos/go-systemd/daemon"
+)
+
+type Handler interface {
+ http.Handler
+ CheckHealth() error
+}
+
+type NewHandlerFunc func(*arvados.Cluster, *arvados.NodeProfile) Handler
+
+type command struct {
+ newHandler NewHandlerFunc
+ svcName arvados.ServiceName
+}
+
+// Command returns a cmd.Handler that loads site config, calls
+// newHandler with the current cluster and node configs, and brings up
+// an http server with the returned handler.
+//
+// The handler is wrapped with server middleware (adding X-Request-ID
+// headers, logging requests/responses, etc).
+func Command(svcName arvados.ServiceName, newHandler NewHandlerFunc) cmd.Handler {
+ return &command{
+ newHandler: newHandler,
+ svcName: svcName,
+ }
+}
+
+func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ log := logrus.New()
+ log.Formatter = &logrus.JSONFormatter{
+ TimestampFormat: rfc3339NanoFixed,
+ }
+ log.Out = stderr
+
+ var err error
+ defer func() {
+ if err != nil {
+ log.WithError(err).Info("exiting")
+ }
+ }()
+ flags := flag.NewFlagSet("", flag.ContinueOnError)
+ flags.SetOutput(stderr)
+ configFile := flags.String("config", arvados.DefaultConfigFile, "Site configuration `file`")
+ nodeProfile := flags.String("node-profile", "", "`Name` of NodeProfiles config entry to use (if blank, use $ARVADOS_NODE_PROFILE or hostname reported by OS)")
+ err = flags.Parse(args)
+ if err == flag.ErrHelp {
+ err = nil
+ return 0
+ } else if err != nil {
+ return 2
+ }
+ cfg, err := arvados.GetConfig(*configFile)
+ if err != nil {
+ return 1
+ }
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return 1
+ }
+ profileName := *nodeProfile
+ if profileName == "" {
+ profileName = os.Getenv("ARVADOS_NODE_PROFILE")
+ }
+ profile, err := cluster.GetNodeProfile(profileName)
+ if err != nil {
+ return 1
+ }
+ listen := profile.ServicePorts()[c.svcName]
+ if listen == "" {
+ err = fmt.Errorf("configuration does not enable the %s service on this host", c.svcName)
+ return 1
+ }
+ handler := c.newHandler(cluster, profile)
+ if err = handler.CheckHealth(); err != nil {
+ return 1
+ }
+ srv := &httpserver.Server{
+ Server: http.Server{
+ Handler: httpserver.AddRequestIDs(httpserver.LogRequests(log, handler)),
+ },
+ Addr: listen,
+ }
+ err = srv.Start()
+ if err != nil {
+ return 1
+ }
+ log.WithFields(logrus.Fields{
+ "Listen": srv.Addr,
+ "Service": c.svcName,
+ }).Info("listening")
+ if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
+ log.WithError(err).Errorf("error notifying init daemon")
+ }
+ err = srv.Wait()
+ if err != nil {
+ return 1
+ }
+ return 0
+}
+
+const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
Package: ArvadosR
Type: Package
Title: Arvados R SDK
-Version: 0.0.2
+Version: 0.0.5
Authors@R: person("Fuad", "Muhic", role = c("aut", "cre"), email = "fmuhic@capeannenterprises.com")
Maintainer: Ward Vandewege <wvandewege@veritasgenetics.com>
Description: This is the Arvados R SDK
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
#' users.get
#'
#' users.get is a method defined in Arvados class.
#' @name users.update_uuid
NULL
+#' users.merge
+#'
+#' users.merge is a method defined in Arvados class.
+#'
+#' @usage arv$users.merge(new_owner_uuid,
+#' new_user_token, redirect_to_new_user = NULL)
+#' @param new_owner_uuid
+#' @param new_user_token
+#' @param redirect_to_new_user
+#' @return User object.
+#' @name users.merge
+NULL
+
#' users.list
#'
#' users.list is a method defined in Arvados class.
#' @name api_clients.list
NULL
-#' authorized_keys.get
+#' container_requests.get
#'
-#' authorized_keys.get is a method defined in Arvados class.
+#' container_requests.get is a method defined in Arvados class.
#'
-#' @usage arv$authorized_keys.get(uuid)
-#' @param uuid The UUID of the AuthorizedKey in question.
-#' @return AuthorizedKey object.
-#' @name authorized_keys.get
+#' @usage arv$container_requests.get(uuid)
+#' @param uuid The UUID of the ContainerRequest in question.
+#' @return ContainerRequest object.
+#' @name container_requests.get
NULL
-#' authorized_keys.create
+#' container_requests.create
#'
-#' authorized_keys.create is a method defined in Arvados class.
+#' container_requests.create is a method defined in Arvados class.
#'
-#' @usage arv$authorized_keys.create(authorizedkey,
+#' @usage arv$container_requests.create(containerrequest,
#' ensure_unique_name = "false")
-#' @param authorizedKey AuthorizedKey object.
+#' @param containerRequest ContainerRequest object.
#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision.
-#' @return AuthorizedKey object.
-#' @name authorized_keys.create
+#' @return ContainerRequest object.
+#' @name container_requests.create
NULL
-#' authorized_keys.update
+#' container_requests.update
#'
-#' authorized_keys.update is a method defined in Arvados class.
+#' container_requests.update is a method defined in Arvados class.
#'
-#' @usage arv$authorized_keys.update(authorizedkey,
+#' @usage arv$container_requests.update(containerrequest,
#' uuid)
-#' @param authorizedKey AuthorizedKey object.
-#' @param uuid The UUID of the AuthorizedKey in question.
-#' @return AuthorizedKey object.
-#' @name authorized_keys.update
+#' @param containerRequest ContainerRequest object.
+#' @param uuid The UUID of the ContainerRequest in question.
+#' @return ContainerRequest object.
+#' @name container_requests.update
NULL
-#' authorized_keys.delete
+#' container_requests.delete
#'
-#' authorized_keys.delete is a method defined in Arvados class.
+#' container_requests.delete is a method defined in Arvados class.
#'
-#' @usage arv$authorized_keys.delete(uuid)
-#' @param uuid The UUID of the AuthorizedKey in question.
-#' @return AuthorizedKey object.
-#' @name authorized_keys.delete
+#' @usage arv$container_requests.delete(uuid)
+#' @param uuid The UUID of the ContainerRequest in question.
+#' @return ContainerRequest object.
+#' @name container_requests.delete
NULL
-#' authorized_keys.list
+#' container_requests.list
#'
-#' authorized_keys.list is a method defined in Arvados class.
+#' container_requests.list is a method defined in Arvados class.
#'
-#' @usage arv$authorized_keys.list(filters = NULL,
+#' @usage arv$container_requests.list(filters = NULL,
#' where = NULL, order = NULL, select = NULL,
#' distinct = NULL, limit = "100", offset = "0",
#' count = "exact")
#' @param limit
#' @param offset
#' @param count
-#' @return AuthorizedKeyList object.
-#' @name authorized_keys.list
+#' @return ContainerRequestList object.
+#' @name container_requests.list
NULL
-#' container_requests.get
+#' authorized_keys.get
#'
-#' container_requests.get is a method defined in Arvados class.
+#' authorized_keys.get is a method defined in Arvados class.
#'
-#' @usage arv$container_requests.get(uuid)
-#' @param uuid The UUID of the ContainerRequest in question.
-#' @return ContainerRequest object.
-#' @name container_requests.get
+#' @usage arv$authorized_keys.get(uuid)
+#' @param uuid The UUID of the AuthorizedKey in question.
+#' @return AuthorizedKey object.
+#' @name authorized_keys.get
NULL
-#' container_requests.create
+#' authorized_keys.create
#'
-#' container_requests.create is a method defined in Arvados class.
+#' authorized_keys.create is a method defined in Arvados class.
#'
-#' @usage arv$container_requests.create(containerrequest,
+#' @usage arv$authorized_keys.create(authorizedkey,
#' ensure_unique_name = "false")
-#' @param containerRequest ContainerRequest object.
+#' @param authorizedKey AuthorizedKey object.
#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision.
-#' @return ContainerRequest object.
-#' @name container_requests.create
+#' @return AuthorizedKey object.
+#' @name authorized_keys.create
NULL
-#' container_requests.update
+#' authorized_keys.update
#'
-#' container_requests.update is a method defined in Arvados class.
+#' authorized_keys.update is a method defined in Arvados class.
#'
-#' @usage arv$container_requests.update(containerrequest,
+#' @usage arv$authorized_keys.update(authorizedkey,
#' uuid)
-#' @param containerRequest ContainerRequest object.
-#' @param uuid The UUID of the ContainerRequest in question.
-#' @return ContainerRequest object.
-#' @name container_requests.update
+#' @param authorizedKey AuthorizedKey object.
+#' @param uuid The UUID of the AuthorizedKey in question.
+#' @return AuthorizedKey object.
+#' @name authorized_keys.update
NULL
-#' container_requests.delete
+#' authorized_keys.delete
#'
-#' container_requests.delete is a method defined in Arvados class.
+#' authorized_keys.delete is a method defined in Arvados class.
#'
-#' @usage arv$container_requests.delete(uuid)
-#' @param uuid The UUID of the ContainerRequest in question.
-#' @return ContainerRequest object.
-#' @name container_requests.delete
+#' @usage arv$authorized_keys.delete(uuid)
+#' @param uuid The UUID of the AuthorizedKey in question.
+#' @return AuthorizedKey object.
+#' @name authorized_keys.delete
NULL
-#' container_requests.list
+#' authorized_keys.list
#'
-#' container_requests.list is a method defined in Arvados class.
+#' authorized_keys.list is a method defined in Arvados class.
#'
-#' @usage arv$container_requests.list(filters = NULL,
+#' @usage arv$authorized_keys.list(filters = NULL,
#' where = NULL, order = NULL, select = NULL,
#' distinct = NULL, limit = "100", offset = "0",
#' count = "exact")
#' @param limit
#' @param offset
#' @param count
-#' @return ContainerRequestList object.
-#' @name container_requests.list
+#' @return AuthorizedKeyList object.
+#' @name authorized_keys.list
NULL
#' collections.get
#' @name job_tasks.list
NULL
-#' links.get
-#'
-#' links.get is a method defined in Arvados class.
-#'
-#' @usage arv$links.get(uuid)
-#' @param uuid The UUID of the Link in question.
-#' @return Link object.
-#' @name links.get
-NULL
-
-#' links.create
-#'
-#' links.create is a method defined in Arvados class.
-#'
-#' @usage arv$links.create(link, ensure_unique_name = "false")
-#' @param link Link object.
-#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision.
-#' @return Link object.
-#' @name links.create
-NULL
-
-#' links.update
-#'
-#' links.update is a method defined in Arvados class.
-#'
-#' @usage arv$links.update(link, uuid)
-#' @param link Link object.
-#' @param uuid The UUID of the Link in question.
-#' @return Link object.
-#' @name links.update
-NULL
-
-#' links.delete
-#'
-#' links.delete is a method defined in Arvados class.
-#'
-#' @usage arv$links.delete(uuid)
-#' @param uuid The UUID of the Link in question.
-#' @return Link object.
-#' @name links.delete
-NULL
-
-#' links.list
-#'
-#' links.list is a method defined in Arvados class.
-#'
-#' @usage arv$links.list(filters = NULL,
-#' where = NULL, order = NULL, select = NULL,
-#' distinct = NULL, limit = "100", offset = "0",
-#' count = "exact")
-#' @param filters
-#' @param where
-#' @param order
-#' @param select
-#' @param distinct
-#' @param limit
-#' @param offset
-#' @param count
-#' @return LinkList object.
-#' @name links.list
-NULL
-
-#' links.get_permissions
-#'
-#' links.get_permissions is a method defined in Arvados class.
-#'
-#' @usage arv$links.get_permissions(uuid)
-#' @param uuid
-#' @return Link object.
-#' @name links.get_permissions
-NULL
-
#' jobs.get
#'
#' jobs.get is a method defined in Arvados class.
#' @name keep_disks.list
NULL
-#' keep_services.get
+#' nodes.get
#'
-#' keep_services.get is a method defined in Arvados class.
+#' nodes.get is a method defined in Arvados class.
#'
-#' @usage arv$keep_services.get(uuid)
-#' @param uuid The UUID of the KeepService in question.
-#' @return KeepService object.
-#' @name keep_services.get
+#' @usage arv$nodes.get(uuid)
+#' @param uuid The UUID of the Node in question.
+#' @return Node object.
+#' @name nodes.get
NULL
-#' keep_services.create
+#' nodes.create
#'
-#' keep_services.create is a method defined in Arvados class.
+#' nodes.create is a method defined in Arvados class.
#'
-#' @usage arv$keep_services.create(keepservice,
-#' ensure_unique_name = "false")
-#' @param keepService KeepService object.
+#' @usage arv$nodes.create(node, ensure_unique_name = "false",
+#' assign_slot = NULL)
+#' @param node Node object.
#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision.
-#' @return KeepService object.
-#' @name keep_services.create
+#' @param assign_slot assign slot and hostname
+#' @return Node object.
+#' @name nodes.create
NULL
-#' keep_services.update
+#' nodes.update
#'
-#' keep_services.update is a method defined in Arvados class.
+#' nodes.update is a method defined in Arvados class.
#'
-#' @usage arv$keep_services.update(keepservice,
-#' uuid)
-#' @param keepService KeepService object.
-#' @param uuid The UUID of the KeepService in question.
-#' @return KeepService object.
-#' @name keep_services.update
+#' @usage arv$nodes.update(node, uuid, assign_slot = NULL)
+#' @param node Node object.
+#' @param uuid The UUID of the Node in question.
+#' @param assign_slot assign slot and hostname
+#' @return Node object.
+#' @name nodes.update
NULL
-#' keep_services.delete
+#' nodes.delete
#'
-#' keep_services.delete is a method defined in Arvados class.
+#' nodes.delete is a method defined in Arvados class.
#'
-#' @usage arv$keep_services.delete(uuid)
-#' @param uuid The UUID of the KeepService in question.
-#' @return KeepService object.
-#' @name keep_services.delete
+#' @usage arv$nodes.delete(uuid)
+#' @param uuid The UUID of the Node in question.
+#' @return Node object.
+#' @name nodes.delete
NULL
-#' keep_services.accessible
+#' nodes.ping
#'
-#' keep_services.accessible is a method defined in Arvados class.
+#' nodes.ping is a method defined in Arvados class.
+#'
+#' @usage arv$nodes.ping(uuid, ping_secret)
+#' @param uuid
+#' @param ping_secret
+#' @return Node object.
+#' @name nodes.ping
+NULL
+
+#' nodes.list
+#'
+#' nodes.list is a method defined in Arvados class.
+#'
+#' @usage arv$nodes.list(filters = NULL,
+#' where = NULL, order = NULL, select = NULL,
+#' distinct = NULL, limit = "100", offset = "0",
+#' count = "exact")
+#' @param filters
+#' @param where
+#' @param order
+#' @param select
+#' @param distinct
+#' @param limit
+#' @param offset
+#' @param count
+#' @return NodeList object.
+#' @name nodes.list
+NULL
+
+#' links.get
+#'
+#' links.get is a method defined in Arvados class.
+#'
+#' @usage arv$links.get(uuid)
+#' @param uuid The UUID of the Link in question.
+#' @return Link object.
+#' @name links.get
+NULL
+
+#' links.create
+#'
+#' links.create is a method defined in Arvados class.
+#'
+#' @usage arv$links.create(link, ensure_unique_name = "false")
+#' @param link Link object.
+#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision.
+#' @return Link object.
+#' @name links.create
+NULL
+
+#' links.update
+#'
+#' links.update is a method defined in Arvados class.
+#'
+#' @usage arv$links.update(link, uuid)
+#' @param link Link object.
+#' @param uuid The UUID of the Link in question.
+#' @return Link object.
+#' @name links.update
+NULL
+
+#' links.delete
+#'
+#' links.delete is a method defined in Arvados class.
+#'
+#' @usage arv$links.delete(uuid)
+#' @param uuid The UUID of the Link in question.
+#' @return Link object.
+#' @name links.delete
+NULL
+
+#' links.list
+#'
+#' links.list is a method defined in Arvados class.
+#'
+#' @usage arv$links.list(filters = NULL,
+#' where = NULL, order = NULL, select = NULL,
+#' distinct = NULL, limit = "100", offset = "0",
+#' count = "exact")
+#' @param filters
+#' @param where
+#' @param order
+#' @param select
+#' @param distinct
+#' @param limit
+#' @param offset
+#' @param count
+#' @return LinkList object.
+#' @name links.list
+NULL
+
+#' links.get_permissions
+#'
+#' links.get_permissions is a method defined in Arvados class.
+#'
+#' @usage arv$links.get_permissions(uuid)
+#' @param uuid
+#' @return Link object.
+#' @name links.get_permissions
+NULL
+
+#' keep_services.get
+#'
+#' keep_services.get is a method defined in Arvados class.
+#'
+#' @usage arv$keep_services.get(uuid)
+#' @param uuid The UUID of the KeepService in question.
+#' @return KeepService object.
+#' @name keep_services.get
+NULL
+
+#' keep_services.create
+#'
+#' keep_services.create is a method defined in Arvados class.
+#'
+#' @usage arv$keep_services.create(keepservice,
+#' ensure_unique_name = "false")
+#' @param keepService KeepService object.
+#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision.
+#' @return KeepService object.
+#' @name keep_services.create
+NULL
+
+#' keep_services.update
+#'
+#' keep_services.update is a method defined in Arvados class.
+#'
+#' @usage arv$keep_services.update(keepservice,
+#' uuid)
+#' @param keepService KeepService object.
+#' @param uuid The UUID of the KeepService in question.
+#' @return KeepService object.
+#' @name keep_services.update
+NULL
+
+#' keep_services.delete
+#'
+#' keep_services.delete is a method defined in Arvados class.
+#'
+#' @usage arv$keep_services.delete(uuid)
+#' @param uuid The UUID of the KeepService in question.
+#' @return KeepService object.
+#' @name keep_services.delete
+NULL
+
+#' keep_services.accessible
+#'
+#' keep_services.accessible is a method defined in Arvados class.
#'
#' @usage arv$keep_services.accessible(NULL)
#' @return KeepService object.
#' @name pipeline_instances.list
NULL
-#' nodes.get
-#'
-#' nodes.get is a method defined in Arvados class.
-#'
-#' @usage arv$nodes.get(uuid)
-#' @param uuid The UUID of the Node in question.
-#' @return Node object.
-#' @name nodes.get
-NULL
-
-#' nodes.create
-#'
-#' nodes.create is a method defined in Arvados class.
-#'
-#' @usage arv$nodes.create(node, ensure_unique_name = "false",
-#' assign_slot = NULL)
-#' @param node Node object.
-#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision.
-#' @param assign_slot assign slot and hostname
-#' @return Node object.
-#' @name nodes.create
-NULL
-
-#' nodes.update
-#'
-#' nodes.update is a method defined in Arvados class.
-#'
-#' @usage arv$nodes.update(node, uuid, assign_slot = NULL)
-#' @param node Node object.
-#' @param uuid The UUID of the Node in question.
-#' @param assign_slot assign slot and hostname
-#' @return Node object.
-#' @name nodes.update
-NULL
-
-#' nodes.delete
-#'
-#' nodes.delete is a method defined in Arvados class.
-#'
-#' @usage arv$nodes.delete(uuid)
-#' @param uuid The UUID of the Node in question.
-#' @return Node object.
-#' @name nodes.delete
-NULL
-
-#' nodes.ping
-#'
-#' nodes.ping is a method defined in Arvados class.
-#'
-#' @usage arv$nodes.ping(uuid, ping_secret)
-#' @param uuid
-#' @param ping_secret
-#' @return Node object.
-#' @name nodes.ping
-NULL
-
-#' nodes.list
-#'
-#' nodes.list is a method defined in Arvados class.
-#'
-#' @usage arv$nodes.list(filters = NULL,
-#' where = NULL, order = NULL, select = NULL,
-#' distinct = NULL, limit = "100", offset = "0",
-#' count = "exact")
-#' @param filters
-#' @param where
-#' @param order
-#' @param select
-#' @param distinct
-#' @param limit
-#' @param offset
-#' @param count
-#' @return NodeList object.
-#' @name nodes.list
-NULL
-
#' repositories.get
#'
#' repositories.get is a method defined in Arvados class.
#' \item{}{\code{\link{users.delete}}}
#' \item{}{\code{\link{users.get}}}
#' \item{}{\code{\link{users.list}}}
+#' \item{}{\code{\link{users.merge}}}
#' \item{}{\code{\link{users.setup}}}
#' \item{}{\code{\link{users.system}}}
#' \item{}{\code{\link{users.unsetup}}}
resource
},
+ users.merge = function(new_owner_uuid, new_user_token,
+ redirect_to_new_user = NULL)
+ {
+ endPoint <- stringr::str_interp("users/merge")
+ url <- paste0(private$host, endPoint)
+ headers <- list(Authorization = paste("OAuth2", private$token),
+ "Content-Type" = "application/json")
+ queryArgs <- list(new_owner_uuid = new_owner_uuid,
+ new_user_token = new_user_token, redirect_to_new_user = redirect_to_new_user)
+
+ body <- NULL
+
+ response <- private$REST$http$exec("POST", url, headers, body,
+ queryArgs, private$numRetries)
+ resource <- private$REST$httpParser$parseJSONResponse(response)
+
+ if(!is.null(resource$errors))
+ stop(resource$errors)
+
+ resource
+ },
+
users.list = function(filters = NULL, where = NULL,
order = NULL, select = NULL, distinct = NULL,
limit = "100", offset = "0", count = "exact")
resource
},
- authorized_keys.get = function(uuid)
+ container_requests.get = function(uuid)
{
- endPoint <- stringr::str_interp("authorized_keys/${uuid}")
+ endPoint <- stringr::str_interp("container_requests/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- authorized_keys.create = function(authorizedkey,
+ container_requests.create = function(containerrequest,
ensure_unique_name = "false")
{
- endPoint <- stringr::str_interp("authorized_keys")
+ endPoint <- stringr::str_interp("container_requests")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
queryArgs <- list(ensure_unique_name = ensure_unique_name)
- if(length(authorizedkey) > 0)
- body <- jsonlite::toJSON(list(authorizedkey = authorizedkey),
+ if(length(containerrequest) > 0)
+ body <- jsonlite::toJSON(list(containerrequest = containerrequest),
auto_unbox = TRUE)
else
body <- NULL
resource
},
- authorized_keys.update = function(authorizedkey, uuid)
+ container_requests.update = function(containerrequest, uuid)
{
- endPoint <- stringr::str_interp("authorized_keys/${uuid}")
+ endPoint <- stringr::str_interp("container_requests/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
queryArgs <- NULL
- if(length(authorizedkey) > 0)
- body <- jsonlite::toJSON(list(authorizedkey = authorizedkey),
+ if(length(containerrequest) > 0)
+ body <- jsonlite::toJSON(list(containerrequest = containerrequest),
auto_unbox = TRUE)
else
body <- NULL
resource
},
- authorized_keys.delete = function(uuid)
+ container_requests.delete = function(uuid)
{
- endPoint <- stringr::str_interp("authorized_keys/${uuid}")
+ endPoint <- stringr::str_interp("container_requests/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- authorized_keys.list = function(filters = NULL,
+ container_requests.list = function(filters = NULL,
where = NULL, order = NULL, select = NULL,
distinct = NULL, limit = "100", offset = "0",
count = "exact")
{
- endPoint <- stringr::str_interp("authorized_keys")
+ endPoint <- stringr::str_interp("container_requests")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- container_requests.get = function(uuid)
+ authorized_keys.get = function(uuid)
{
- endPoint <- stringr::str_interp("container_requests/${uuid}")
+ endPoint <- stringr::str_interp("authorized_keys/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- container_requests.create = function(containerrequest,
+ authorized_keys.create = function(authorizedkey,
ensure_unique_name = "false")
{
- endPoint <- stringr::str_interp("container_requests")
+ endPoint <- stringr::str_interp("authorized_keys")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
queryArgs <- list(ensure_unique_name = ensure_unique_name)
- if(length(containerrequest) > 0)
- body <- jsonlite::toJSON(list(containerrequest = containerrequest),
+ if(length(authorizedkey) > 0)
+ body <- jsonlite::toJSON(list(authorizedkey = authorizedkey),
auto_unbox = TRUE)
else
body <- NULL
resource
},
- container_requests.update = function(containerrequest, uuid)
+ authorized_keys.update = function(authorizedkey, uuid)
{
- endPoint <- stringr::str_interp("container_requests/${uuid}")
+ endPoint <- stringr::str_interp("authorized_keys/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
queryArgs <- NULL
- if(length(containerrequest) > 0)
- body <- jsonlite::toJSON(list(containerrequest = containerrequest),
+ if(length(authorizedkey) > 0)
+ body <- jsonlite::toJSON(list(authorizedkey = authorizedkey),
auto_unbox = TRUE)
else
body <- NULL
resource
},
- container_requests.delete = function(uuid)
+ authorized_keys.delete = function(uuid)
{
- endPoint <- stringr::str_interp("container_requests/${uuid}")
+ endPoint <- stringr::str_interp("authorized_keys/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- container_requests.list = function(filters = NULL,
+ authorized_keys.list = function(filters = NULL,
where = NULL, order = NULL, select = NULL,
distinct = NULL, limit = "100", offset = "0",
count = "exact")
{
- endPoint <- stringr::str_interp("container_requests")
+ endPoint <- stringr::str_interp("authorized_keys")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- links.get = function(uuid)
+ jobs.get = function(uuid)
{
- endPoint <- stringr::str_interp("links/${uuid}")
+ endPoint <- stringr::str_interp("jobs/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- links.create = function(link, ensure_unique_name = "false")
+ jobs.create = function(job, ensure_unique_name = "false",
+ find_or_create = "false", filters = NULL,
+ minimum_script_version = NULL, exclude_script_versions = NULL)
{
- endPoint <- stringr::str_interp("links")
+ endPoint <- stringr::str_interp("jobs")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
- queryArgs <- list(ensure_unique_name = ensure_unique_name)
+ queryArgs <- list(ensure_unique_name = ensure_unique_name,
+ find_or_create = find_or_create, filters = filters,
+ minimum_script_version = minimum_script_version,
+ exclude_script_versions = exclude_script_versions)
- if(length(link) > 0)
- body <- jsonlite::toJSON(list(link = link),
+ if(length(job) > 0)
+ body <- jsonlite::toJSON(list(job = job),
auto_unbox = TRUE)
else
body <- NULL
resource
},
- links.update = function(link, uuid)
+ jobs.update = function(job, uuid)
{
- endPoint <- stringr::str_interp("links/${uuid}")
+ endPoint <- stringr::str_interp("jobs/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
queryArgs <- NULL
- if(length(link) > 0)
- body <- jsonlite::toJSON(list(link = link),
+ if(length(job) > 0)
+ body <- jsonlite::toJSON(list(job = job),
auto_unbox = TRUE)
else
body <- NULL
resource
},
- links.delete = function(uuid)
+ jobs.delete = function(uuid)
{
- endPoint <- stringr::str_interp("links/${uuid}")
+ endPoint <- stringr::str_interp("jobs/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- links.list = function(filters = NULL, where = NULL,
+ jobs.queue = function(filters = NULL, where = NULL,
order = NULL, select = NULL, distinct = NULL,
limit = "100", offset = "0", count = "exact")
{
- endPoint <- stringr::str_interp("links")
+ endPoint <- stringr::str_interp("jobs/queue")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- links.get_permissions = function(uuid)
+ jobs.queue_size = function()
{
- endPoint <- stringr::str_interp("permissions/${uuid}")
+ endPoint <- stringr::str_interp("jobs/queue_size")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- jobs.get = function(uuid)
+ jobs.cancel = function(uuid)
{
- endPoint <- stringr::str_interp("jobs/${uuid}")
+ endPoint <- stringr::str_interp("jobs/${uuid}/cancel")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
body <- NULL
- response <- private$REST$http$exec("GET", url, headers, body,
+ response <- private$REST$http$exec("POST", url, headers, body,
queryArgs, private$numRetries)
resource <- private$REST$httpParser$parseJSONResponse(response)
resource
},
- jobs.create = function(job, ensure_unique_name = "false",
- find_or_create = "false", filters = NULL,
- minimum_script_version = NULL, exclude_script_versions = NULL)
+ jobs.lock = function(uuid)
{
- endPoint <- stringr::str_interp("jobs")
+ endPoint <- stringr::str_interp("jobs/${uuid}/lock")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
- queryArgs <- list(ensure_unique_name = ensure_unique_name,
- find_or_create = find_or_create, filters = filters,
- minimum_script_version = minimum_script_version,
- exclude_script_versions = exclude_script_versions)
+ queryArgs <- NULL
- if(length(job) > 0)
- body <- jsonlite::toJSON(list(job = job),
- auto_unbox = TRUE)
- else
- body <- NULL
+ body <- NULL
response <- private$REST$http$exec("POST", url, headers, body,
queryArgs, private$numRetries)
resource
},
- jobs.update = function(job, uuid)
+ jobs.list = function(filters = NULL, where = NULL,
+ order = NULL, select = NULL, distinct = NULL,
+ limit = "100", offset = "0", count = "exact")
{
- endPoint <- stringr::str_interp("jobs/${uuid}")
+ endPoint <- stringr::str_interp("jobs")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
- queryArgs <- NULL
+ queryArgs <- list(filters = filters, where = where,
+ order = order, select = select, distinct = distinct,
+ limit = limit, offset = offset, count = count)
- if(length(job) > 0)
- body <- jsonlite::toJSON(list(job = job),
- auto_unbox = TRUE)
- else
- body <- NULL
+ body <- NULL
- response <- private$REST$http$exec("PUT", url, headers, body,
+ response <- private$REST$http$exec("GET", url, headers, body,
queryArgs, private$numRetries)
resource <- private$REST$httpParser$parseJSONResponse(response)
resource
},
- jobs.delete = function(uuid)
+ keep_disks.get = function(uuid)
{
- endPoint <- stringr::str_interp("jobs/${uuid}")
+ endPoint <- stringr::str_interp("keep_disks/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
body <- NULL
- response <- private$REST$http$exec("DELETE", url, headers, body,
+ response <- private$REST$http$exec("GET", url, headers, body,
queryArgs, private$numRetries)
resource <- private$REST$httpParser$parseJSONResponse(response)
resource
},
- jobs.queue = function(filters = NULL, where = NULL,
- order = NULL, select = NULL, distinct = NULL,
- limit = "100", offset = "0", count = "exact")
+ keep_disks.create = function(keepdisk, ensure_unique_name = "false")
{
- endPoint <- stringr::str_interp("jobs/queue")
+ endPoint <- stringr::str_interp("keep_disks")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
- queryArgs <- list(filters = filters, where = where,
- order = order, select = select, distinct = distinct,
- limit = limit, offset = offset, count = count)
+ queryArgs <- list(ensure_unique_name = ensure_unique_name)
- body <- NULL
+ if(length(keepdisk) > 0)
+ body <- jsonlite::toJSON(list(keepdisk = keepdisk),
+ auto_unbox = TRUE)
+ else
+ body <- NULL
- response <- private$REST$http$exec("GET", url, headers, body,
+ response <- private$REST$http$exec("POST", url, headers, body,
queryArgs, private$numRetries)
resource <- private$REST$httpParser$parseJSONResponse(response)
resource
},
- jobs.queue_size = function()
+ keep_disks.update = function(keepdisk, uuid)
{
- endPoint <- stringr::str_interp("jobs/queue_size")
+ endPoint <- stringr::str_interp("keep_disks/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
queryArgs <- NULL
- body <- NULL
+ if(length(keepdisk) > 0)
+ body <- jsonlite::toJSON(list(keepdisk = keepdisk),
+ auto_unbox = TRUE)
+ else
+ body <- NULL
- response <- private$REST$http$exec("GET", url, headers, body,
+ response <- private$REST$http$exec("PUT", url, headers, body,
queryArgs, private$numRetries)
resource <- private$REST$httpParser$parseJSONResponse(response)
resource
},
- jobs.cancel = function(uuid)
+ keep_disks.delete = function(uuid)
{
- endPoint <- stringr::str_interp("jobs/${uuid}/cancel")
+ endPoint <- stringr::str_interp("keep_disks/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
body <- NULL
- response <- private$REST$http$exec("POST", url, headers, body,
+ response <- private$REST$http$exec("DELETE", url, headers, body,
queryArgs, private$numRetries)
resource <- private$REST$httpParser$parseJSONResponse(response)
resource
},
- jobs.lock = function(uuid)
+ keep_disks.ping = function(uuid = NULL, ping_secret,
+ node_uuid = NULL, filesystem_uuid = NULL,
+ service_host = NULL, service_port, service_ssl_flag)
{
- endPoint <- stringr::str_interp("jobs/${uuid}/lock")
+ endPoint <- stringr::str_interp("keep_disks/ping")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
- queryArgs <- NULL
+ queryArgs <- list(uuid = uuid, ping_secret = ping_secret,
+ node_uuid = node_uuid, filesystem_uuid = filesystem_uuid,
+ service_host = service_host, service_port = service_port,
+ service_ssl_flag = service_ssl_flag)
body <- NULL
resource
},
- jobs.list = function(filters = NULL, where = NULL,
- order = NULL, select = NULL, distinct = NULL,
- limit = "100", offset = "0", count = "exact")
+ keep_disks.list = function(filters = NULL,
+ where = NULL, order = NULL, select = NULL,
+ distinct = NULL, limit = "100", offset = "0",
+ count = "exact")
{
- endPoint <- stringr::str_interp("jobs")
+ endPoint <- stringr::str_interp("keep_disks")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- keep_disks.get = function(uuid)
+ nodes.get = function(uuid)
{
- endPoint <- stringr::str_interp("keep_disks/${uuid}")
+ endPoint <- stringr::str_interp("nodes/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- keep_disks.create = function(keepdisk, ensure_unique_name = "false")
+ nodes.create = function(node, ensure_unique_name = "false",
+ assign_slot = NULL)
{
- endPoint <- stringr::str_interp("keep_disks")
+ endPoint <- stringr::str_interp("nodes")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
- queryArgs <- list(ensure_unique_name = ensure_unique_name)
+ queryArgs <- list(ensure_unique_name = ensure_unique_name,
+ assign_slot = assign_slot)
- if(length(keepdisk) > 0)
- body <- jsonlite::toJSON(list(keepdisk = keepdisk),
+ if(length(node) > 0)
+ body <- jsonlite::toJSON(list(node = node),
auto_unbox = TRUE)
else
body <- NULL
resource
},
- keep_disks.update = function(keepdisk, uuid)
+ nodes.update = function(node, uuid, assign_slot = NULL)
{
- endPoint <- stringr::str_interp("keep_disks/${uuid}")
+ endPoint <- stringr::str_interp("nodes/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
- queryArgs <- NULL
+ queryArgs <- list(assign_slot = assign_slot)
- if(length(keepdisk) > 0)
- body <- jsonlite::toJSON(list(keepdisk = keepdisk),
+ if(length(node) > 0)
+ body <- jsonlite::toJSON(list(node = node),
auto_unbox = TRUE)
else
body <- NULL
resource
},
- keep_disks.delete = function(uuid)
+ nodes.delete = function(uuid)
{
- endPoint <- stringr::str_interp("keep_disks/${uuid}")
+ endPoint <- stringr::str_interp("nodes/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- keep_disks.ping = function(uuid = NULL, ping_secret,
- node_uuid = NULL, filesystem_uuid = NULL,
- service_host = NULL, service_port, service_ssl_flag)
+ nodes.ping = function(uuid, ping_secret)
{
- endPoint <- stringr::str_interp("keep_disks/ping")
+ endPoint <- stringr::str_interp("nodes/${uuid}/ping")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
- queryArgs <- list(uuid = uuid, ping_secret = ping_secret,
- node_uuid = node_uuid, filesystem_uuid = filesystem_uuid,
- service_host = service_host, service_port = service_port,
- service_ssl_flag = service_ssl_flag)
+ queryArgs <- list(ping_secret = ping_secret)
body <- NULL
resource
},
- keep_disks.list = function(filters = NULL,
- where = NULL, order = NULL, select = NULL,
- distinct = NULL, limit = "100", offset = "0",
- count = "exact")
+ nodes.list = function(filters = NULL, where = NULL,
+ order = NULL, select = NULL, distinct = NULL,
+ limit = "100", offset = "0", count = "exact")
{
- endPoint <- stringr::str_interp("keep_disks")
+ endPoint <- stringr::str_interp("nodes")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- keep_services.get = function(uuid)
+ links.get = function(uuid)
{
- endPoint <- stringr::str_interp("keep_services/${uuid}")
+ endPoint <- stringr::str_interp("links/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- keep_services.create = function(keepservice,
- ensure_unique_name = "false")
+ links.create = function(link, ensure_unique_name = "false")
{
- endPoint <- stringr::str_interp("keep_services")
+ endPoint <- stringr::str_interp("links")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
queryArgs <- list(ensure_unique_name = ensure_unique_name)
- if(length(keepservice) > 0)
- body <- jsonlite::toJSON(list(keepservice = keepservice),
+ if(length(link) > 0)
+ body <- jsonlite::toJSON(list(link = link),
auto_unbox = TRUE)
else
body <- NULL
resource
},
- keep_services.update = function(keepservice, uuid)
+ links.update = function(link, uuid)
{
- endPoint <- stringr::str_interp("keep_services/${uuid}")
+ endPoint <- stringr::str_interp("links/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
queryArgs <- NULL
- if(length(keepservice) > 0)
- body <- jsonlite::toJSON(list(keepservice = keepservice),
+ if(length(link) > 0)
+ body <- jsonlite::toJSON(list(link = link),
auto_unbox = TRUE)
else
body <- NULL
resource
},
- keep_services.delete = function(uuid)
+ links.delete = function(uuid)
{
- endPoint <- stringr::str_interp("keep_services/${uuid}")
+ endPoint <- stringr::str_interp("links/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- keep_services.accessible = function()
+ links.list = function(filters = NULL, where = NULL,
+ order = NULL, select = NULL, distinct = NULL,
+ limit = "100", offset = "0", count = "exact")
{
- endPoint <- stringr::str_interp("keep_services/accessible")
+ endPoint <- stringr::str_interp("links")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
- queryArgs <- NULL
+ queryArgs <- list(filters = filters, where = where,
+ order = order, select = select, distinct = distinct,
+ limit = limit, offset = offset, count = count)
body <- NULL
resource
},
- keep_services.list = function(filters = NULL,
- where = NULL, order = NULL, select = NULL,
- distinct = NULL, limit = "100", offset = "0",
- count = "exact")
+ links.get_permissions = function(uuid)
{
- endPoint <- stringr::str_interp("keep_services")
+ endPoint <- stringr::str_interp("permissions/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
- queryArgs <- list(filters = filters, where = where,
- order = order, select = select, distinct = distinct,
- limit = limit, offset = offset, count = count)
+ queryArgs <- NULL
body <- NULL
resource
},
- pipeline_templates.get = function(uuid)
+ keep_services.get = function(uuid)
{
- endPoint <- stringr::str_interp("pipeline_templates/${uuid}")
+ endPoint <- stringr::str_interp("keep_services/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- pipeline_templates.create = function(pipelinetemplate,
+ keep_services.create = function(keepservice,
ensure_unique_name = "false")
{
- endPoint <- stringr::str_interp("pipeline_templates")
+ endPoint <- stringr::str_interp("keep_services")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
queryArgs <- list(ensure_unique_name = ensure_unique_name)
- if(length(pipelinetemplate) > 0)
- body <- jsonlite::toJSON(list(pipelinetemplate = pipelinetemplate),
+ if(length(keepservice) > 0)
+ body <- jsonlite::toJSON(list(keepservice = keepservice),
auto_unbox = TRUE)
else
body <- NULL
resource
},
- pipeline_templates.update = function(pipelinetemplate, uuid)
+ keep_services.update = function(keepservice, uuid)
{
- endPoint <- stringr::str_interp("pipeline_templates/${uuid}")
+ endPoint <- stringr::str_interp("keep_services/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
queryArgs <- NULL
- if(length(pipelinetemplate) > 0)
- body <- jsonlite::toJSON(list(pipelinetemplate = pipelinetemplate),
+ if(length(keepservice) > 0)
+ body <- jsonlite::toJSON(list(keepservice = keepservice),
auto_unbox = TRUE)
else
body <- NULL
resource
},
- pipeline_templates.delete = function(uuid)
+ keep_services.delete = function(uuid)
{
- endPoint <- stringr::str_interp("pipeline_templates/${uuid}")
+ endPoint <- stringr::str_interp("keep_services/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- pipeline_templates.list = function(filters = NULL,
- where = NULL, order = NULL, select = NULL,
- distinct = NULL, limit = "100", offset = "0",
- count = "exact")
+ keep_services.accessible = function()
{
- endPoint <- stringr::str_interp("pipeline_templates")
+ endPoint <- stringr::str_interp("keep_services/accessible")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
- queryArgs <- list(filters = filters, where = where,
- order = order, select = select, distinct = distinct,
- limit = limit, offset = offset, count = count)
+ queryArgs <- NULL
body <- NULL
resource
},
- pipeline_instances.get = function(uuid)
+ keep_services.list = function(filters = NULL,
+ where = NULL, order = NULL, select = NULL,
+ distinct = NULL, limit = "100", offset = "0",
+ count = "exact")
{
- endPoint <- stringr::str_interp("pipeline_instances/${uuid}")
+ endPoint <- stringr::str_interp("keep_services")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
- queryArgs <- NULL
+ queryArgs <- list(filters = filters, where = where,
+ order = order, select = select, distinct = distinct,
+ limit = limit, offset = offset, count = count)
body <- NULL
resource
},
- pipeline_instances.create = function(pipelineinstance,
- ensure_unique_name = "false")
+ pipeline_templates.get = function(uuid)
{
- endPoint <- stringr::str_interp("pipeline_instances")
+ endPoint <- stringr::str_interp("pipeline_templates/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
- queryArgs <- list(ensure_unique_name = ensure_unique_name)
+ queryArgs <- NULL
- if(length(pipelineinstance) > 0)
- body <- jsonlite::toJSON(list(pipelineinstance = pipelineinstance),
- auto_unbox = TRUE)
- else
- body <- NULL
+ body <- NULL
- response <- private$REST$http$exec("POST", url, headers, body,
+ response <- private$REST$http$exec("GET", url, headers, body,
queryArgs, private$numRetries)
resource <- private$REST$httpParser$parseJSONResponse(response)
resource
},
- pipeline_instances.update = function(pipelineinstance, uuid)
+ pipeline_templates.create = function(pipelinetemplate,
+ ensure_unique_name = "false")
{
- endPoint <- stringr::str_interp("pipeline_instances/${uuid}")
+ endPoint <- stringr::str_interp("pipeline_templates")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
- queryArgs <- NULL
+ queryArgs <- list(ensure_unique_name = ensure_unique_name)
- if(length(pipelineinstance) > 0)
- body <- jsonlite::toJSON(list(pipelineinstance = pipelineinstance),
+ if(length(pipelinetemplate) > 0)
+ body <- jsonlite::toJSON(list(pipelinetemplate = pipelinetemplate),
auto_unbox = TRUE)
else
body <- NULL
- response <- private$REST$http$exec("PUT", url, headers, body,
+ response <- private$REST$http$exec("POST", url, headers, body,
queryArgs, private$numRetries)
resource <- private$REST$httpParser$parseJSONResponse(response)
resource
},
- pipeline_instances.delete = function(uuid)
+ pipeline_templates.update = function(pipelinetemplate, uuid)
{
- endPoint <- stringr::str_interp("pipeline_instances/${uuid}")
+ endPoint <- stringr::str_interp("pipeline_templates/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
queryArgs <- NULL
- body <- NULL
+ if(length(pipelinetemplate) > 0)
+ body <- jsonlite::toJSON(list(pipelinetemplate = pipelinetemplate),
+ auto_unbox = TRUE)
+ else
+ body <- NULL
- response <- private$REST$http$exec("DELETE", url, headers, body,
+ response <- private$REST$http$exec("PUT", url, headers, body,
queryArgs, private$numRetries)
resource <- private$REST$httpParser$parseJSONResponse(response)
resource
},
- pipeline_instances.cancel = function(uuid)
+ pipeline_templates.delete = function(uuid)
{
- endPoint <- stringr::str_interp("pipeline_instances/${uuid}/cancel")
+ endPoint <- stringr::str_interp("pipeline_templates/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
body <- NULL
- response <- private$REST$http$exec("POST", url, headers, body,
+ response <- private$REST$http$exec("DELETE", url, headers, body,
queryArgs, private$numRetries)
resource <- private$REST$httpParser$parseJSONResponse(response)
resource
},
- pipeline_instances.list = function(filters = NULL,
+ pipeline_templates.list = function(filters = NULL,
where = NULL, order = NULL, select = NULL,
distinct = NULL, limit = "100", offset = "0",
count = "exact")
{
- endPoint <- stringr::str_interp("pipeline_instances")
+ endPoint <- stringr::str_interp("pipeline_templates")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- nodes.get = function(uuid)
+ pipeline_instances.get = function(uuid)
{
- endPoint <- stringr::str_interp("nodes/${uuid}")
+ endPoint <- stringr::str_interp("pipeline_instances/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- nodes.create = function(node, ensure_unique_name = "false",
- assign_slot = NULL)
+ pipeline_instances.create = function(pipelineinstance,
+ ensure_unique_name = "false")
{
- endPoint <- stringr::str_interp("nodes")
+ endPoint <- stringr::str_interp("pipeline_instances")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
- queryArgs <- list(ensure_unique_name = ensure_unique_name,
- assign_slot = assign_slot)
+ queryArgs <- list(ensure_unique_name = ensure_unique_name)
- if(length(node) > 0)
- body <- jsonlite::toJSON(list(node = node),
+ if(length(pipelineinstance) > 0)
+ body <- jsonlite::toJSON(list(pipelineinstance = pipelineinstance),
auto_unbox = TRUE)
else
body <- NULL
resource
},
- nodes.update = function(node, uuid, assign_slot = NULL)
+ pipeline_instances.update = function(pipelineinstance, uuid)
{
- endPoint <- stringr::str_interp("nodes/${uuid}")
+ endPoint <- stringr::str_interp("pipeline_instances/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
- queryArgs <- list(assign_slot = assign_slot)
+ queryArgs <- NULL
- if(length(node) > 0)
- body <- jsonlite::toJSON(list(node = node),
+ if(length(pipelineinstance) > 0)
+ body <- jsonlite::toJSON(list(pipelineinstance = pipelineinstance),
auto_unbox = TRUE)
else
body <- NULL
resource
},
- nodes.delete = function(uuid)
+ pipeline_instances.delete = function(uuid)
{
- endPoint <- stringr::str_interp("nodes/${uuid}")
+ endPoint <- stringr::str_interp("pipeline_instances/${uuid}")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
resource
},
- nodes.ping = function(uuid, ping_secret)
+ pipeline_instances.cancel = function(uuid)
{
- endPoint <- stringr::str_interp("nodes/${uuid}/ping")
+ endPoint <- stringr::str_interp("pipeline_instances/${uuid}/cancel")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
- queryArgs <- list(ping_secret = ping_secret)
+ queryArgs <- NULL
body <- NULL
resource
},
- nodes.list = function(filters = NULL, where = NULL,
- order = NULL, select = NULL, distinct = NULL,
- limit = "100", offset = "0", count = "exact")
+ pipeline_instances.list = function(filters = NULL,
+ where = NULL, order = NULL, select = NULL,
+ distinct = NULL, limit = "100", offset = "0",
+ count = "exact")
{
- endPoint <- stringr::str_interp("nodes")
+ endPoint <- stringr::str_interp("pipeline_instances")
url <- paste0(private$host, endPoint)
headers <- list(Authorization = paste("OAuth2", private$token),
"Content-Type" = "application/json")
source("./R/util.R")
#' ArvadosFile
-#'
+#'
#' ArvadosFile class represents a file inside Arvados collection.
-#'
+#'
#' @section Usage:
#' \preformatted{file = ArvadosFile$new(name)}
#'
#' \describe{
#' \item{name}{Name of the file.}
#' }
-#'
+#'
#' @section Methods:
#' \describe{
#' \item{getName()}{Returns name of the file.}
#' \item{flush()}{Write connections content to a file (override current content of the file).}
#' \item{remove(name)}{Removes ArvadosFile or Subcollection specified by name from the subcollection.}
#' \item{getSizeInBytes()}{Returns file size in bytes.}
-#' \item{move(newLocation)}{Moves file to a new location inside collection.}
+#' \item{move(destination)}{Moves file to a new location inside collection.}
+#' \item{copy(destination)}{Copies file to a new location inside collection.}
#' }
#'
#' @name ArvadosFile
#' myFile$write("This is new file content")
#' fileContent <- myFile$read()
#' fileContent <- myFile$read("text")
-#' fileContent <- myFile$read("raw", offset = 8, length = 4)
+#' fileContent <- myFile$read("raw", offset = 8, length = 4)
#'
#' #Write a table:
#' arvConnection <- myFile$connection("w")
#' mytable <- read.table(arvConnection)
#'
#' myFile$move("newFolder/myFile")
+#' myFile$copy("newFolder/myFile")
#' }
NULL
fileSize <- REST$getResourceSize(self$getRelativePath(),
private$collection$uuid)
-
fileSize
},
getCollection = function() private$collection,
- setCollection = function(collection)
+ setCollection = function(collection, setRecursively = TRUE)
{
private$collection <- collection
},
connection = function(rw)
{
- if (rw == "r" || rw == "rb")
+ if (rw == "r" || rw == "rb")
{
REST <- private$collection$getRESTService()
- return(REST$getConnection(private$collection$uuid,
- self$getRelativePath(),
+ return(REST$getConnection(self$getRelativePath(),
+ private$collection$uuid,
rw))
}
- else if (rw == "w")
+ else if (rw == "w")
{
private$buffer <- textConnection(NULL, "w")
}
},
- flush = function()
+ flush = function()
{
v <- textConnectionValue(private$buffer)
close(private$buffer)
writeResult
},
- move = function(newLocation)
+ move = function(destination)
{
if(is.null(private$collection))
- stop("ArvadosFile doesn't belong to any collection")
+ stop("ArvadosFile doesn't belong to any collection.")
- newLocation <- trimFromEnd(newLocation, "/")
- nameAndPath <- splitToPathAndName(newLocation)
+ destination <- trimFromEnd(destination, "/")
+ nameAndPath <- splitToPathAndName(destination)
newParent <- private$collection$get(nameAndPath$path)
if(is.null(newParent))
- {
- stop("Unable to get destination subcollection")
- }
+ stop("Unable to get destination subcollection.")
childWithSameName <- newParent$get(nameAndPath$name)
private$collection$uuid)
private$dettachFromCurrentParent()
- private$attachToNewParent(newParent)
+ private$attachToNewParent(self, newParent)
+ private$parent <- newParent
private$name <- nameAndPath$name
- "Content moved successfully."
+ self
+ },
+
+ copy = function(destination)
+ {
+ if(is.null(private$collection))
+ stop("ArvadosFile doesn't belong to any collection.")
+
+ destination <- trimFromEnd(destination, "/")
+ nameAndPath <- splitToPathAndName(destination)
+
+ newParent <- private$collection$get(nameAndPath$path)
+
+ if(is.null(newParent))
+ stop("Unable to get destination subcollection.")
+
+ childWithSameName <- newParent$get(nameAndPath$name)
+
+ if(!is.null(childWithSameName))
+ stop("Destination already contains content with same name.")
+
+ REST <- private$collection$getRESTService()
+ REST$copy(self$getRelativePath(),
+ paste0(newParent$getRelativePath(), "/", nameAndPath$name),
+ private$collection$uuid)
+
+ newFile <- self$duplicate(nameAndPath$name)
+ newFile$setCollection(self$getCollection())
+ private$attachToNewParent(newFile, newParent)
+ newFile$setParent(newParent)
+
+ newFile
+ },
+
+ duplicate = function(newName = NULL)
+ {
+ name <- if(!is.null(newName)) newName else private$name
+ newFile <- ArvadosFile$new(name)
+ newFile
}
),
collection = NULL,
buffer = NULL,
- attachToNewParent = function(newParent)
+ attachToNewParent = function(content, newParent)
{
- #Note: We temporary set parents collection to NULL. This will ensure that
- # add method doesn't post file on REST.
+ # We temporary set parents collection to NULL. This will ensure that
+ # add method doesn't post this file on REST.
+ # We also need to set content's collection to NULL because
+ # add method throws exception if we try to add content that already
+ # belongs to a collection.
parentsCollection <- newParent$getCollection()
+ content$setCollection(NULL, setRecursively = FALSE)
newParent$setCollection(NULL, setRecursively = FALSE)
-
- newParent$add(self)
-
+ newParent$add(content)
+ content$setCollection(parentsCollection, setRecursively = FALSE)
newParent$setCollection(parentsCollection, setRecursively = FALSE)
-
- private$parent <- newParent
},
dettachFromCurrentParent = function()
{
- #Note: We temporary set parents collection to NULL. This will ensure that
- # remove method doesn't remove this subcollection from REST.
+ # We temporary set parents collection to NULL. This will ensure that
+ # remove method doesn't remove this file from REST.
parent <- private$parent
parentsCollection <- parent$getCollection()
parent$setCollection(NULL, setRecursively = FALSE)
-
parent$remove(private$name)
-
parent$setCollection(parentsCollection, setRecursively = FALSE)
}
),
#'
#' @param x Instance of ArvadosFile class
#' @param ... Optional arguments.
-#' @export
+#' @export
print.ArvadosFile = function(x, ...)
{
collection <- NULL
relativePath <- paste0("/", relativePath)
}
- cat(paste0("Type: ", "\"", "ArvadosFile", "\""), sep = "\n")
- cat(paste0("Name: ", "\"", x$getName(), "\""), sep = "\n")
- cat(paste0("Relative path: ", "\"", relativePath, "\""), sep = "\n")
- cat(paste0("Collection: ", "\"", collection, "\""), sep = "\n")
+ cat(paste0("Type: ", "\"", "ArvadosFile", "\""), sep = "\n")
+ cat(paste0("Name: ", "\"", x$getName(), "\""), sep = "\n")
+ cat(paste0("Relative path: ", "\"", relativePath, "\""), sep = "\n")
+ cat(paste0("Collection: ", "\"", collection, "\""), sep = "\n")
}
source("./R/util.R")
#' Collection
-#'
+#'
#' Collection class provides interface for working with Arvados collections.
-#'
+#'
#' @section Usage:
#' \preformatted{collection = Collection$new(arv, uuid)}
#'
#' \item{arv}{Arvados object.}
#' \item{uuid}{UUID of a collection.}
#' }
-#'
+#'
#' @section Methods:
#' \describe{
#' \item{add(content)}{Adds ArvadosFile or Subcollection specified by content to the collection.}
-#' \item{create(fileNames, relativePath = "")}{Creates one or more ArvadosFiles and adds them to the collection at specified path.}
+#' \item{create(files)}{Creates one or more ArvadosFiles and adds them to the collection at specified path.}
#' \item{remove(fileNames)}{Remove one or more files from the collection.}
-#' \item{move(content, newLocation)}{Moves ArvadosFile or Subcollection to another location in the collection.}
+#' \item{move(content, destination)}{Moves ArvadosFile or Subcollection to another location in the collection.}
+#' \item{copy(content, destination)}{Copies ArvadosFile or Subcollection to another location in the collection.}
#' \item{getFileListing()}{Returns collections file content as character vector.}
#' \item{get(relativePath)}{If relativePath is valid, returns ArvadosFile or Subcollection specified by relativePath, else returns NULL.}
#' }
#' arv <- Arvados$new("your Arvados token", "example.arvadosapi.com")
#' collection <- Collection$new(arv, "uuid")
#'
-#' newFile <- ArvadosFile$new("myFile")
-#' collection$add(newFile, "myFolder")
-#'
#' createdFiles <- collection$create(c("main.cpp", lib.dll), "cpp/src/")
#'
#' collection$remove("location/to/my/file.cpp")
public = list(
uuid = NULL,
- # api = NULL,
- initialize = function(api, uuid)
+ initialize = function(api, uuid)
{
- # self$api <- api
private$REST <- api$getRESTService()
-
self$uuid <- uuid
-
- private$fileContent <- private$REST$getCollectionContent(uuid)
- private$tree <- CollectionTree$new(private$fileContent, self)
},
add = function(content, relativePath = "")
{
if(is.null(private$tree))
- private$genereateCollectionTreeStructure()
+ private$generateCollectionTreeStructure()
if(relativePath == "" ||
relativePath == "." ||
if("ArvadosFile" %in% class(content) ||
"Subcollection" %in% class(content))
{
+ if(!is.null(content$getCollection()))
+ stop("Content already belongs to a collection.")
+
if(content$getName() == "")
stop("Content has invalid name.")
}
},
- create = function(fileNames, relativePath = "")
+ create = function(files)
{
if(is.null(private$tree))
- private$genereateCollectionTreeStructure()
+ private$generateCollectionTreeStructure()
- if(relativePath == "" ||
- relativePath == "." ||
- relativePath == "./")
+ if(is.character(files))
{
- subcollection <- private$tree$getTree()
- }
- else
- {
- relativePath <- trimFromEnd(relativePath, "/")
- subcollection <- self$get(relativePath)
- }
-
- if(is.null(subcollection))
- stop(paste("Subcollection", relativePath, "doesn't exist."))
-
- if(is.character(fileNames))
- {
- arvadosFiles <- NULL
- sapply(fileNames, function(fileName)
+ sapply(files, function(file)
{
- childWithSameName <- subcollection$get(fileName)
+ childWithSameName <- self$get(file)
if(!is.null(childWithSameName))
stop("Destination already contains file with same name.")
- newFile <- ArvadosFile$new(fileName)
- subcollection$add(newFile)
+ newTreeBranch <- private$tree$createBranch(file)
+ private$tree$addBranch(private$tree$getTree(), newTreeBranch)
- arvadosFiles <<- c(arvadosFiles, newFile)
+ private$REST$create(file, self$uuid)
+ newTreeBranch$setCollection(self)
})
- if(length(arvadosFiles) == 1)
- return(arvadosFiles[[1]])
- else
- return(arvadosFiles)
+ "Created"
}
- else
+ else
{
stop(paste0("Expected character vector, got ",
- paste0("(", paste0(class(fileNames), collapse = ", "), ")"),
+ paste0("(", paste0(class(files), collapse = ", "), ")"),
"."))
}
},
remove = function(paths)
{
if(is.null(private$tree))
- private$genereateCollectionTreeStructure()
+ private$generateCollectionTreeStructure()
if(is.character(paths))
{
"Content removed"
}
- else
+ else
{
stop(paste0("Expected character vector, got ",
paste0("(", paste0(class(paths), collapse = ", "), ")"),
}
},
- move = function(content, newLocation)
+ move = function(content, destination)
{
if(is.null(private$tree))
- private$genereateCollectionTreeStructure()
+ private$generateCollectionTreeStructure()
content <- trimFromEnd(content, "/")
if(is.null(elementToMove))
stop("Content you want to move doesn't exist in the collection.")
- elementToMove$move(newLocation)
+ elementToMove$move(destination)
+ },
+
+ copy = function(content, destination)
+ {
+ if(is.null(private$tree))
+ private$generateCollectionTreeStructure()
+
+ content <- trimFromEnd(content, "/")
+
+ elementToCopy <- self$get(content)
+
+ if(is.null(elementToCopy))
+ stop("Content you want to copy doesn't exist in the collection.")
+
+ elementToCopy$copy(destination)
+ },
+
+ refresh = function()
+ {
+ if(!is.null(private$tree))
+ {
+ private$tree$getTree()$setCollection(NULL, setRecursively = TRUE)
+ private$tree <- NULL
+ }
},
getFileListing = function()
{
if(is.null(private$tree))
- private$genereateCollectionTreeStructure()
+ private$generateCollectionTreeStructure()
content <- private$REST$getCollectionContent(self$uuid)
content[order(tolower(content))]
get = function(relativePath)
{
if(is.null(private$tree))
- private$genereateCollectionTreeStructure()
+ private$generateCollectionTreeStructure()
private$tree$getElement(relativePath)
},
- toJSON = function()
- {
- fields <- sapply(private$classFields, function(field)
- {
- self[[field]]
- }, USE.NAMES = TRUE)
-
- jsonlite::toJSON(list("collection" =
- Filter(Negate(is.null), fields)), auto_unbox = TRUE)
- },
-
- isEmpty = function() {
- fields <- sapply(private$classFields,
- function(field) self[[field]])
-
- if(any(sapply(fields, function(field) !is.null(field) && field != "")))
- FALSE
- else
- TRUE
- },
-
getRESTService = function() private$REST,
setRESTService = function(newRESTService) private$REST <- newRESTService
),
REST = NULL,
tree = NULL,
fileContent = NULL,
- classFields = NULL,
- genereateCollectionTreeStructure = function()
+ generateCollectionTreeStructure = function()
{
if(is.null(self$uuid))
stop("Collection uuid is not defined.")
#'
#' @param x Instance of Collection class
#' @param ... Optional arguments.
-#' @export
+#' @export
print.Collection = function(x, ...)
{
cat(paste0("Type: ", "\"", "Arvados Collection", "\""), sep = "\n")
initialize = function(fileContent, collection)
{
self$pathsList <- fileContent
-
- treeBranches <- sapply(fileContent, function(filePath)
- {
- splitPath <- unlist(strsplit(filePath, "/", fixed = TRUE))
- branch <- private$createBranch(splitPath)
- })
-
+ treeBranches <- sapply(fileContent, function(filePath) self$createBranch(filePath))
root <- Subcollection$new("")
-
- sapply(treeBranches, function(branch)
- {
- private$addBranch(root, branch)
- })
-
+ sapply(treeBranches, function(branch) self$addBranch(root, branch))
root$setCollection(collection)
private$tree <- root
},
- getElement = function(relativePath)
- {
- relativePath <- trimFromStart(relativePath, "./")
- relativePath <- trimFromEnd(relativePath, "/")
-
- if(endsWith(relativePath, "/"))
- relativePath <- substr(relativePath, 0, nchar(relativePath) - 1)
-
- splitPath <- unlist(strsplit(relativePath, "/", fixed = TRUE))
- returnElement <- private$tree
-
- for(pathFragment in splitPath)
- {
- returnElement <- returnElement$get(pathFragment)
-
- if(is.null(returnElement))
- return(NULL)
- }
-
- returnElement
- },
-
- getTree = function() private$tree
- ),
-
- private = list(
-
- tree = NULL,
-
- createBranch = function(splitPath)
+ createBranch = function(filePath)
{
+ splitPath <- unlist(strsplit(filePath, "/", fixed = TRUE))
branch <- NULL
lastElementIndex <- length(splitPath)
branch <- newFolder
}
}
-
+
branch
},
if(is.null(child))
{
+ # Make sure we are don't make any REST call while adding child
+ collection <- container$getCollection()
+ container$setCollection(NULL, setRecursively = FALSE)
container$add(node)
+ container$setCollection(collection, setRecursively = FALSE)
}
else
{
- # Note: REST always returns folder name alone before other folder
+ # Note: REST always returns folder name alone before other folder
# content, so in first iteration we don't know if it's a file
- # or folder since its just a name, so we assume it's a file.
- # If we encounter that same name again we know
+ # or folder since its just a name, so we assume it's a file.
+ # If we encounter that same name again we know
# it's a folder so we need to replace ArvadosFile with Subcollection.
if("ArvadosFile" %in% class(child))
- {
child = private$replaceFileWithSubcollection(child)
- }
- private$addBranch(child, node$getFirst())
+ self$addBranch(child, node$getFirst())
}
},
+ getElement = function(relativePath)
+ {
+ relativePath <- trimFromStart(relativePath, "./")
+ relativePath <- trimFromEnd(relativePath, "/")
+
+ if(endsWith(relativePath, "/"))
+ relativePath <- substr(relativePath, 0, nchar(relativePath) - 1)
+
+ splitPath <- unlist(strsplit(relativePath, "/", fixed = TRUE))
+ returnElement <- private$tree
+
+ for(pathFragment in splitPath)
+ {
+ returnElement <- returnElement$get(pathFragment)
+
+ if(is.null(returnElement))
+ return(NULL)
+ }
+
+ returnElement
+ },
+
+ getTree = function() private$tree
+ ),
+
+ private = list(
+
+ tree = NULL,
+
replaceFileWithSubcollection = function(arvadosFile)
{
subcollection <- Subcollection$new(arvadosFile$getName())
validContentTypes = NULL,
- initialize = function()
+ initialize = function()
{
self$validContentTypes <- c("text", "raw")
},
- parseJSONResponse = function(serverResponse)
+ parseJSONResponse = function(serverResponse)
{
parsed_response <- httr::content(serverResponse,
as = "parsed",
result[-1]
},
- getFileSizesFromResponse = function(response, uri)
+ getFileSizesFromResponse = function(response, uri)
{
text <- rawToChar(response$content)
doc <- XML::xmlParse(text, asText=TRUE)
validContentTypes = NULL,
validVerbs = NULL,
- initialize = function()
+ initialize = function()
{
self$validContentTypes <- c("text", "raw")
- self$validVerbs <- c("GET", "POST", "PUT", "DELETE", "PROPFIND", "MOVE")
+ self$validVerbs <- c("GET", "POST", "PUT", "DELETE", "PROPFIND", "MOVE", "COPY")
},
exec = function(verb, url, headers = NULL, body = NULL, queryParams = NULL,
config <- httr::add_headers(unlist(headers))
if(toString(Sys.getenv("ARVADOS_API_HOST_INSECURE") == "TRUE"))
- config$options = list(ssl_verifypeer = FALSE)
+ config$options = list(ssl_verifypeer = 0L)
# times = 1 regular call + numberOfRetries
response <- httr::RETRY(verb, url = url, body = body,
}
return("")
+ },
+
+ getConnection = function(url, headers, openMode)
+ {
+ h <- curl::new_handle()
+ curl::handle_setheaders(h, .list = headers)
+
+ if(toString(Sys.getenv("ARVADOS_API_HOST_INSECURE") == "TRUE"))
+ curl::handle_setopt(h, ssl_verifypeer = 0L)
+
+ conn <- curl::curl(url = url, open = openMode, handle = h)
}
),
{
fileURL <- paste0(self$getWebDavHostName(), "c=",
uuid, "/", relativePath);
- headers <- list(Authorization = paste("OAuth2", self$token))
+ headers <- list(Authorization = paste("OAuth2", self$token))
serverResponse <- self$http$exec("DELETE", fileURL, headers,
retryTimes = self$numRetries)
{
collectionURL <- paste0(self$getWebDavHostName(), "c=", uuid, "/")
fromURL <- paste0(collectionURL, from)
- toURL <- paste0(collectionURL, to)
+ toURL <- paste0(collectionURL, trimFromStart(to, "/"))
headers <- list("Authorization" = paste("OAuth2", self$token),
- "Destination" = toURL)
+ "Destination" = toURL)
serverResponse <- self$http$exec("MOVE", fromURL, headers,
retryTimes = self$numRetries)
serverResponse
},
+ copy = function(from, to, uuid)
+ {
+ collectionURL <- paste0(self$getWebDavHostName(), "c=", uuid, "/")
+ fromURL <- paste0(collectionURL, from)
+ toURL <- paste0(collectionURL, trimFromStart(to, "/"))
+
+ headers <- list("Authorization" = paste("OAuth2", self$token),
+ "Destination" = toURL)
+
+ serverResponse <- self$http$exec("COPY", fromURL, headers,
+ retryTimes = self$numRetries)
+
+ if(serverResponse$status_code < 200 || serverResponse$status_code >= 300)
+ stop(paste("Server code:", serverResponse$status_code))
+
+ serverResponse
+ },
+
getCollectionContent = function(uuid)
{
collectionURL <- URLencode(paste0(self$getWebDavHostName(),
self$httpParser$parseResponse(serverResponse, "text")
},
- getConnection = function(uuid, relativePath, openMode)
+ getConnection = function(relativePath, uuid, openMode)
{
- fileURL <- paste0(self$getWebDavHostName(),
+ fileURL <- paste0(self$getWebDavHostName(),
"c=", uuid, "/", relativePath);
headers <- list(Authorization = paste("OAuth2", self$token))
- h <- curl::new_handle()
- curl::handle_setheaders(h, .list = headers)
-
- conn <- curl::curl(url = fileURL, open = openMode, handle = h)
-
- conn
+ conn <- self$http$getConnection(fileURL, headers, openMode)
}
),
{
fileURL <- paste0(self$getWebDavHostName(), "c=",
uuid, "/", relativePath)
- headers <- list(Authorization = paste("OAuth2", self$token),
+ headers <- list(Authorization = paste("OAuth2", self$token),
"Content-Type" = contentType)
body <- NULL
source("./R/util.R")
#' Subcollection
-#'
+#'
#' Subcollection class represents a folder inside Arvados collection.
#' It is essentially a composite of arvadosFiles and other subcollections.
-#'
+#'
#' @section Usage:
#' \preformatted{subcollection = Subcollection$new(name)}
#'
#' \describe{
#' \item{name}{Name of the subcollection.}
#' }
-#'
+#'
#' @section Methods:
#' \describe{
#' \item{getName()}{Returns name of the subcollection.}
#' \item{get(relativePath)}{If relativePath is valid, returns ArvadosFile or Subcollection specified by relativePath, else returns NULL.}
#' \item{getFileListing()}{Returns subcollections file content as character vector.}
#' \item{getSizeInBytes()}{Returns subcollections content size in bytes.}
-#' \item{move(newLocation)}{Moves subcollection to a new location inside collection.}
+#' \item{move(destination)}{Moves subcollection to a new location inside collection.}
+#' \item{copy(destination)}{Copies subcollection to a new location inside collection.}
#' }
#'
#' @name Subcollection
#' myFolder$remove("myFile")
#'
#' myFolder$move("newLocation/myFolder")
+#' myFolder$copy("newLocation/myFolder")
#' }
NULL
},
getName = function() private$name,
-
+
getRelativePath = function()
{
relativePath <- c(private$name)
if("ArvadosFile" %in% class(content) ||
"Subcollection" %in% class(content))
{
+ if(!is.null(content$getCollection()))
+ stop("Content already belongs to a collection.")
+
if(content$getName() == "")
stop("Content has invalid name.")
"or Subcollection with same name."))
if(!is.null(private$collection))
- {
+ {
if(self$getRelativePath() != "")
contentPath <- paste0(self$getRelativePath(),
"/", content$getFileListing())
return(sum(fileSizes))
},
- move = function(newLocation)
+ move = function(destination)
{
if(is.null(private$collection))
- stop("Subcollection doesn't belong to any collection")
+ stop("Subcollection doesn't belong to any collection.")
- newLocation <- trimFromEnd(newLocation, "/")
- nameAndPath <- splitToPathAndName(newLocation)
+ destination <- trimFromEnd(destination, "/")
+ nameAndPath <- splitToPathAndName(destination)
newParent <- private$collection$get(nameAndPath$path)
if(is.null(newParent))
- {
- stop("Unable to get destination subcollection")
- }
+ stop("Unable to get destination subcollection.")
childWithSameName <- newParent$get(nameAndPath$name)
private$collection$uuid)
private$dettachFromCurrentParent()
- private$attachToNewParent(newParent)
+ private$attachToNewParent(self, newParent)
+ private$parent <- newParent
private$name <- nameAndPath$name
- "Content moved successfully."
+ self
+ },
+
+ copy = function(destination)
+ {
+ if(is.null(private$collection))
+ stop("Subcollection doesn't belong to any collection.")
+
+ destination <- trimFromEnd(destination, "/")
+ nameAndPath <- splitToPathAndName(destination)
+
+ newParent <- private$collection$get(nameAndPath$path)
+
+ if(is.null(newParent) || !("Subcollection" %in% class(newParent)))
+ stop("Unable to get destination subcollection.")
+
+ childWithSameName <- newParent$get(nameAndPath$name)
+
+ if(!is.null(childWithSameName))
+ stop("Destination already contains content with same name.")
+
+ REST <- private$collection$getRESTService()
+ REST$copy(self$getRelativePath(),
+ paste0(newParent$getRelativePath(), "/", nameAndPath$name),
+ private$collection$uuid)
+
+ newContent <- self$duplicate(nameAndPath$name)
+ newContent$setCollection(self$getCollection(), setRecursively = TRUE)
+ newContent$setParent(newParent)
+ private$attachToNewParent(newContent, newParent)
+
+ newContent
+ },
+
+ duplicate = function(newName = NULL)
+ {
+ name <- if(!is.null(newName)) newName else private$name
+ root <- Subcollection$new(name)
+ for(child in private$children)
+ root$add(child$duplicate())
+
+ root
},
get = function(name)
}
},
- attachToNewParent = function(newParent)
+ attachToNewParent = function(content, newParent)
{
- #Note: We temporary set parents collection to NULL. This will ensure that
- # add method doesn't post file on REST.
+ # We temporary set parents collection to NULL. This will ensure that
+ # add method doesn't post this subcollection to REST.
+ # We also need to set content's collection to NULL because
+ # add method throws exception if we try to add content that already
+ # belongs to a collection.
parentsCollection <- newParent$getCollection()
+ content$setCollection(NULL, setRecursively = FALSE)
newParent$setCollection(NULL, setRecursively = FALSE)
-
- newParent$add(self)
-
+ newParent$add(content)
+ content$setCollection(parentsCollection, setRecursively = FALSE)
newParent$setCollection(parentsCollection, setRecursively = FALSE)
-
- private$parent <- newParent
},
dettachFromCurrentParent = function()
{
- #Note: We temporary set parents collection to NULL. This will ensure that
- # remove method doesn't remove this subcollection from REST.
+ # We temporary set parents collection to NULL. This will ensure that
+ # remove method doesn't remove this subcollection from REST.
parent <- private$parent
parentsCollection <- parent$getCollection()
parent$setCollection(NULL, setRecursively = FALSE)
-
parent$remove(private$name)
-
parent$setCollection(parentsCollection, setRecursively = FALSE)
},
content
}
),
-
+
cloneable = FALSE
)
#'
#' @param x Instance of Subcollection class
#' @param ... Optional arguments.
-#' @export
+#' @export
print.Subcollection = function(x, ...)
{
collection <- NULL
}, methodResources, resourceNames)))
projectDoc <- genProjectMethodsDoc()
-
+
c(methodsDoc, projectDoc)
}
methodNames[!(methodNames %in% c("index", "show", "destroy"))])
}, methodResources, resourceNames)))
-
+
hardcodedMethods <- c("projects.create", "projects.get",
"projects.list", "projects.update", "projects.delete")
- paste0("#' \t\\item{}{\\code{\\link{", sort(c(methodList, hardcodedMethods)), "}}}")
+ paste0("#' \t\\item{}{\\code{\\link{", sort(c(methodList, hardcodedMethods)), "}}}")
}
getMethodDoc <- function(methodName, methodMetaData)
className <- sapply(prop, function(ref) ref)
objectName <- paste0(tolower(substr(className, 1, 1)),
substr(className, 2, nchar(className)))
- paste("#' @param", objectName, className, "object.")
+ paste("#' @param", objectName, className, "object.")
})))
}
{
arg <- methodMetaData$parameters[[argName]]
argDescription <- arg$description
- paste("#' @param", argName, argDescription)
+ paste("#' @param", argName, argDescription)
})))
c(requestDoc, argsDoc)
{
if(length(args) > 1)
{
- args[1:(length(args) - 1)] <- paste0(args[1:(length(args) - 1)], ",")
+ args[1:(length(args) - 1)] <- paste0(args[1:(length(args) - 1)], ",")
}
args[1] <- paste0(prependAtStart, args[1])
argLines <- c(argLines, line)
}
-
+
argLines <- unlist(argLines)
argLinesLen <- length(argLines)
if(argLinesLen > 1)
- argLines[2:argLinesLen] <- paste0(prependToEachSplit, argLines[2:argLinesLen])
+ argLines[2:argLinesLen] <- paste0(prependToEachSplit, argLines[2:argLinesLen])
argLines
}
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+.onLoad <- function(libName, pkgName)
+{
+ minAllowedRVersion <- "3.3.0"
+ currentRVersion <- getRversion()
+
+ if(currentRVersion < minAllowedRVersion)
+ print(paste0("Minimum R version required to run ", pkgName, " is ",
+ minAllowedRVersion, ". Your current version is ",
+ toString(currentRVersion), ". Please update R and try again."))
+}
apt-get install build-essential libxml2-dev libssl-dev libcurl4-gnutls-dev
```
+Minimum R version required to run ArvadosR is 3.3.0.
+
### Usage
arvadosFile <- collection$get("location/to/my/file.cpp")
```
- or
+or
```{r}
arvadosSubcollection <- collection$get("location/to/my/directory/")
size <- arvadosFile$getSizeInBytes()
```
- or
+or
```{r}
size <- arvadosSubcollection$getSizeInBytes()
* Create new file in a collection:
```{r}
-collection$create(fileNames, optionalRelativePath)
+collection$create(files)
```
- Example:
+Example:
```{r}
-mainFile <- collection$create("main.cpp", "cpp/src/")
-fileList <- collection$create(c("main.cpp", lib.dll), "cpp/src/")
+mainFile <- collection$create("cpp/src/main.cpp")
+fileList <- collection$create(c("cpp/src/main.cpp", "cpp/src/util.h"))
```
-* Add existing ArvadosFile or Subcollection to a collection:
-
-```{r}
-folder <- Subcollection$new("src")
-file <- ArvadosFile$new("main.cpp")
-folder$add(file)
-```
-
-```{r}
-collection$add(folder, "cpp")
-```
-
-This examples will add file "main.cpp" in "./cpp/src/" folder if folder exists.
-If subcollection contains more files or folders they will be added recursively.
-
* Delete file from a collection:
```{r}
subcollection$remove("folderInsideSubcollection/")
```
-* Move file or folder inside collection:
+* Move or rename a file or folder within a collection (moving between collections is currently not supported):
-Directley from collection
+Directly from collection
```{r}
collection$move("folder/file.cpp", "file.cpp")
Make sure to include new file name in destination.
In second example file$move("newDestination/") will not work.
+* Copy file or folder within a collection (copying between collections is currently not supported):
+
+Directly from collection
+
+```{r}
+collection$copy("folder/file.cpp", "file.cpp")
+```
+
+Or from file
+
+```{r}
+file <- collection$get("location/to/my/file.cpp")
+file$copy("destination/file.cpp")
+```
+
+Or from subcollection
+
+```{r}
+subcollection <- collection$get("location/to/folder")
+subcollection$copy("destination/folder")
+```
+
#### Working with Aravdos projects
* Get a project:
\item{}{\code{\link{users.delete}}}
\item{}{\code{\link{users.get}}}
\item{}{\code{\link{users.list}}}
+ \item{}{\code{\link{users.merge}}}
\item{}{\code{\link{users.setup}}}
\item{}{\code{\link{users.system}}}
\item{}{\code{\link{users.unsetup}}}
\item{flush()}{Write connections content to a file (override current content of the file).}
\item{remove(name)}{Removes ArvadosFile or Subcollection specified by name from the subcollection.}
\item{getSizeInBytes()}{Returns file size in bytes.}
- \item{move(newLocation)}{Moves file to a new location inside collection.}
+ \item{move(destination)}{Moves file to a new location inside collection.}
+ \item{copy(destination)}{Copies file to a new location inside collection.}
}
}
myFile$write("This is new file content")
fileContent <- myFile$read()
fileContent <- myFile$read("text")
-fileContent <- myFile$read("raw", offset = 8, length = 4)
+fileContent <- myFile$read("raw", offset = 8, length = 4)
#Write a table:
arvConnection <- myFile$connection("w")
mytable <- read.table(arvConnection)
myFile$move("newFolder/myFile")
+myFile$copy("newFolder/myFile")
}
}
\describe{
\item{add(content)}{Adds ArvadosFile or Subcollection specified by content to the collection.}
- \item{create(fileNames, relativePath = "")}{Creates one or more ArvadosFiles and adds them to the collection at specified path.}
+ \item{create(files)}{Creates one or more ArvadosFiles and adds them to the collection at specified path.}
\item{remove(fileNames)}{Remove one or more files from the collection.}
- \item{move(content, newLocation)}{Moves ArvadosFile or Subcollection to another location in the collection.}
+ \item{move(content, destination)}{Moves ArvadosFile or Subcollection to another location in the collection.}
+ \item{copy(content, destination)}{Copies ArvadosFile or Subcollection to another location in the collection.}
\item{getFileListing()}{Returns collections file content as character vector.}
\item{get(relativePath)}{If relativePath is valid, returns ArvadosFile or Subcollection specified by relativePath, else returns NULL.}
}
arv <- Arvados$new("your Arvados token", "example.arvadosapi.com")
collection <- Collection$new(arv, "uuid")
-newFile <- ArvadosFile$new("myFile")
-collection$add(newFile, "myFolder")
-
createdFiles <- collection$create(c("main.cpp", lib.dll), "cpp/src/")
collection$remove("location/to/my/file.cpp")
\title{Subcollection}
\description{
Subcollection class represents a folder inside Arvados collection.
-It is essentially a composite of ArvadosFiles and other Subcollections.
+It is essentially a composite of arvadosFiles and other subcollections.
}
\section{Usage}{
\item{get(relativePath)}{If relativePath is valid, returns ArvadosFile or Subcollection specified by relativePath, else returns NULL.}
\item{getFileListing()}{Returns subcollections file content as character vector.}
\item{getSizeInBytes()}{Returns subcollections content size in bytes.}
- \item{move(newLocation)}{Moves subcollection to a new location inside collection.}
+ \item{move(destination)}{Moves subcollection to a new location inside collection.}
+ \item{copy(destination)}{Copies subcollection to a new location inside collection.}
}
}
myFolder$remove("myFile")
myFolder$move("newLocation/myFolder")
+myFolder$copy("newLocation/myFolder")
}
}
--- /dev/null
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/Arvados.R
+\name{users.merge}
+\alias{users.merge}
+\title{users.merge}
+\usage{
+arv$users.merge(new_owner_uuid,
+ new_user_token, redirect_to_new_user = NULL)
+}
+\arguments{
+\item{new_owner_uuid}{}
+
+\item{new_user_token}{}
+
+\item{redirect_to_new_user}{}
+}
+\value{
+User object.
+}
+\description{
+users.merge is a method defined in Arvados class.
+}
validContentTypes = NULL,
parserCallCount = NULL,
- initialize = function()
+ initialize = function()
{
self$parserCallCount <- 0
self$validContentTypes <- c("text", "raw")
},
- parseJSONResponse = function(serverResponse)
+ parseJSONResponse = function(serverResponse)
{
self$parserCallCount <- self$parserCallCount + 1
serverResponse
},
- getFileSizesFromResponse = function(serverResponse, uri)
+ getFileSizesFromResponse = function(serverResponse, uri)
{
self$parserCallCount <- self$parserCallCount + 1
JSONEncodedBodyIsProvided = NULL,
requestBodyIsProvided = NULL,
- numberOfGETRequests = NULL,
- numberOfDELETERequests = NULL,
- numberOfPUTRequests = NULL,
- numberOfPOSTRequests = NULL,
- numberOfMOVERequests = NULL,
+ numberOfGETRequests = NULL,
+ numberOfDELETERequests = NULL,
+ numberOfPUTRequests = NULL,
+ numberOfPOSTRequests = NULL,
+ numberOfMOVERequests = NULL,
+ numberOfCOPYRequests = NULL,
+ numberOfgetConnectionCalls = NULL,
initialize = function(expectedURL = NULL,
serverResponse = NULL,
self$numberOfPUTRequests <- 0
self$numberOfPOSTRequests <- 0
self$numberOfMOVERequests <- 0
+ self$numberOfCOPYRequests <- 0
+
+ self$numberOfgetConnectionCalls <- 0
self$serverMaxElementsPerRequest <- 5
},
self$numberOfDELETERequests <- self$numberOfDELETERequests + 1
else if(verb == "MOVE")
self$numberOfMOVERequests <- self$numberOfMOVERequests + 1
+ else if(verb == "COPY")
+ self$numberOfCOPYRequests <- self$numberOfCOPYRequests + 1
else if(verb == "PROPFIND")
{
return(self$content)
return(private$getElements(offset, limit))
else
return(self$content)
+ },
+
+ getConnection = function(url, headers, openMode)
+ {
+ self$numberOfgetConnectionCalls <- self$numberOfgetConnectionCalls + 1
+ c(url, headers, openMode)
}
),
private = list(
- validateURL = function(url)
+ validateURL = function(url)
{
if(!is.null(self$expectedURL) && url == self$expectedURL)
self$URLIsProperlyConfigured <- TRUE
},
- validateHeaders = function(headers)
+ validateHeaders = function(headers)
{
if(!is.null(headers$Authorization))
self$requestHeaderContainsAuthorizationField <- TRUE
validateBody = function(body)
{
- if(!is.null(body))
+ if(!is.null(body))
{
self$requestBodyIsProvided <- TRUE
- if(class(body) == "json")
+ if(class(body) == "json")
self$JSONEncodedBodyIsProvided <- TRUE
}
},
{
if(offset > self$content$items_available)
stop("Invalid offset")
-
+
start <- offset + 1
}
createCallCount = NULL,
deleteCallCount = NULL,
moveCallCount = NULL,
+ copyCallCount = NULL,
getCollectionContentCallCount = NULL,
getResourceSizeCallCount = NULL,
readCallCount = NULL,
collectionContent = NULL,
returnContent = NULL,
- initialize = function(collectionContent = NULL, returnContent = NULL,
+ initialize = function(collectionContent = NULL, returnContent = NULL,
expectedFilterContent = NULL)
{
self$getResourceCallCount <- 0
self$createCallCount <- 0
self$deleteCallCount <- 0
self$moveCallCount <- 0
+ self$copyCallCount <- 0
self$getCollectionContentCallCount <- 0
self$getResourceSizeCallCount <- 0
self$readCallCount <- 0
self$returnContent
},
+ copy = function(from, to, uuid)
+ {
+ self$copyCallCount <- self$copyCallCount + 1
+ self$returnContent
+ },
+
getCollectionContent = function(uuid)
{
self$getCollectionContentCallCount <- self$getCollectionContentCallCount + 1
self$getResourceSizeCallCount <- self$getResourceSizeCallCount + 1
self$returnContent
},
-
+
read = function(relativePath, uuid, contentType = "text", offset = 0, length = 0)
{
self$readCallCount <- self$readCallCount + 1
self$returnContent
},
- write = function(uuid, relativePath, content, contentType)
+ write = function(relativePath, uuid, content, contentType)
{
self$writeBuffer <- content
self$writeCallCount <- self$writeCallCount + 1
self$returnContent
},
- getConnection = function(relativePath, uuid, openMode)
+ getConnection = function(uuid, relativePath, openMode)
{
self$getConnectionCallCount <- self$getConnectionCallCount + 1
self$returnContent
test_that("constructor raises error if file name is empty string", {
expect_that(ArvadosFile$new(""), throws_error("Invalid name."))
-})
+})
test_that("getFileListing always returns file name", {
dog <- ArvadosFile$new("dog")
expect_that(dog$getFileListing(), equals("dog"))
-})
+})
test_that("get always returns NULL", {
dog <- ArvadosFile$new("dog")
-
+
responseIsNull <- is.null(dog$get("something"))
expect_that(responseIsNull, is_true())
-})
+})
test_that("getFirst always returns NULL", {
dog <- ArvadosFile$new("dog")
-
+
responseIsNull <- is.null(dog$getFirst())
expect_that(responseIsNull, is_true())
-})
+})
test_that(paste("getSizeInBytes returns zero if arvadosFile",
"is not part of a collection"), {
dog <- ArvadosFile$new("dog")
expect_that(dog$getSizeInBytes(), equals(0))
-})
+})
test_that(paste("getSizeInBytes delegates size calculation",
"to REST service class"), {
resourceSize <- fish$getSizeInBytes()
expect_that(resourceSize, equals(100))
-})
+})
test_that("getRelativePath returns path relative to the tree root", {
fish$add(shark)
expect_that(shark$getRelativePath(), equals("animal/fish/shark"))
-})
+})
test_that("read raises exception if file doesn't belong to a collection", {
expect_that(dog$read(),
throws_error("ArvadosFile doesn't belong to any collection."))
-})
+})
test_that("read raises exception offset or length is negative number", {
-
collectionContent <- c("animal", "animal/fish")
fakeREST <- FakeRESTService$new(collectionContent)
throws_error("Offset and length must be positive values."))
expect_that(fish$read(contentType = "text", offset = -1, length = -1),
throws_error("Offset and length must be positive values."))
-})
+})
test_that("read delegates reading operation to REST service class", {
api$setRESTService(fakeREST)
collection <- Collection$new(api, "myUUID")
fish <- collection$get("animal/fish")
-
+
fileContent <- fish$read("text")
expect_that(fileContent, equals("my file"))
expect_that(fakeREST$readCallCount, equals(1))
-})
+})
test_that(paste("connection delegates connection creation ro RESTService class",
- "which returns curl connection opened in read mode when",
+ "which returns curl connection opened in read mode when",
"'r' of 'rb' is passed as argument"), {
collectionContent <- c("animal", "animal/fish")
connection <- fish$connection("r")
expect_that(fakeREST$getConnectionCallCount, equals(1))
-})
+})
test_that(paste("connection returns textConnection opened",
"in write mode when 'w' is passed as argument"), {
expect_that(writeResult[1], equals("file"))
expect_that(writeResult[2], equals("content"))
-})
+})
test_that("flush sends data stored in a connection to a REST server", {
-
collectionContent <- c("animal", "animal/fish")
fakeREST <- FakeRESTService$new(collectionContent)
fish$flush()
expect_that(fakeREST$writeBuffer, equals("file content"))
-})
+})
test_that("write raises exception if file doesn't belong to a collection", {
expect_that(dog$write(),
throws_error("ArvadosFile doesn't belong to any collection."))
-})
+})
test_that("write delegates writing operation to REST service class", {
api$setRESTService(fakeREST)
collection <- Collection$new(api, "myUUID")
fish <- collection$get("animal/fish")
-
+
fileContent <- fish$write("new file content")
expect_that(fakeREST$writeBuffer, equals("new file content"))
-})
+})
test_that(paste("move raises exception if arvados file",
"doesn't belong to any collection"), {
animal <- ArvadosFile$new("animal")
expect_that(animal$move("new/location"),
- throws_error("ArvadosFile doesn't belong to any collection"))
-})
+ throws_error("ArvadosFile doesn't belong to any collection."))
+})
test_that(paste("move raises exception if newLocationInCollection",
"parameter is invalid"), {
-
collectionContent <- c("animal",
"animal/fish",
"animal/dog",
dog <- collection$get("animal/dog")
expect_that(dog$move("objects/dog"),
- throws_error("Unable to get destination subcollection"))
-})
+ throws_error("Unable to get destination subcollection."))
+})
test_that("move raises exception if new location contains content with the same name", {
expect_that(dog$move("dog"),
throws_error("Destination already contains content with same name."))
-})
+})
test_that("move moves arvados file inside collection tree", {
-
collectionContent <- c("animal",
"animal/fish",
"animal/dog",
expect_that(dogIsNullOnOldLocation, is_true())
expect_that(dogExistsOnNewLocation, is_true())
})
+
+test_that(paste("copy raises exception if arvados file",
+ "doesn't belong to any collection"), {
+
+ animal <- ArvadosFile$new("animal")
+
+ expect_that(animal$copy("new/location"),
+ throws_error("ArvadosFile doesn't belong to any collection."))
+})
+
+test_that(paste("copy raises exception if location parameter is invalid"), {
+
+ collectionContent <- c("animal",
+ "animal/fish",
+ "animal/dog",
+ "animal/fish/shark",
+ "ball")
+
+ fakeREST <- FakeRESTService$new(collectionContent)
+
+ api <- Arvados$new("myToken", "myHostName")
+ api$setRESTService(fakeREST)
+
+ collection <- Collection$new(api, "myUUID")
+ dog <- collection$get("animal/dog")
+
+ expect_that(dog$copy("objects/dog"),
+ throws_error("Unable to get destination subcollection."))
+})
+
+test_that("copy raises exception if new location contains content with the same name", {
+
+
+ collectionContent <- c("animal",
+ "animal/fish",
+ "animal/dog",
+ "animal/fish/shark",
+ "dog")
+
+ fakeREST <- FakeRESTService$new(collectionContent)
+
+ api <- Arvados$new("myToken", "myHostName")
+ api$setRESTService(fakeREST)
+ collection <- Collection$new(api, "myUUID")
+ dog <- collection$get("animal/dog")
+
+ expect_that(dog$copy("dog"),
+ throws_error("Destination already contains content with same name."))
+
+})
+
+test_that("copy copies arvados file inside collection tree", {
+
+ collectionContent <- c("animal",
+ "animal/fish",
+ "animal/dog",
+ "animal/fish/shark",
+ "ball")
+
+ fakeREST <- FakeRESTService$new(collectionContent)
+
+ api <- Arvados$new("myToken", "myHostName")
+ api$setRESTService(fakeREST)
+ collection <- Collection$new(api, "myUUID")
+ dog <- collection$get("animal/dog")
+
+ dog$copy("dog")
+ dogExistsOnOldLocation <- !is.null(collection$get("animal/dog"))
+ dogExistsOnNewLocation <- !is.null(collection$get("dog"))
+
+ expect_that(dogExistsOnOldLocation, is_true())
+ expect_that(dogExistsOnNewLocation, is_true())
+})
+
+test_that("duplicate performs deep cloning of Arvados file", {
+ arvFile <- ArvadosFile$new("foo")
+ newFile1 <- arvFile$duplicate()
+ newFile2 <- arvFile$duplicate("bar")
+
+ expect_that(newFile1$getFileListing(), equals(arvFile$getFileListing()))
+ expect_that(newFile2$getFileListing(), equals(c("bar")))
+})
collection <- Collection$new(api, "myUUID")
expect_that(collection$create(10),
- throws_error("Expected character vector, got (numeric).",
+ throws_error("Expected character vector, got (numeric).",
fixed = TRUE))
})
-test_that("create raises exception if relative path is not valid", {
-
- collectionContent <- c("animal",
- "animal/fish",
- "ball")
-
- fakeREST <- FakeRESTService$new(collectionContent)
-
- api <- Arvados$new("myToken", "myHostName")
- api$setRESTService(fakeREST)
- collection <- Collection$new(api, "myUUID")
-
- newPen <- ArvadosFile$new("pen")
-
- expect_that(collection$create(newPen, "objects"),
- throws_error("Subcollection objects doesn't exist.",
- fixed = TRUE))
-})
-
test_that(paste("create adds files specified by fileNames",
"to local tree structure and remote REST service"), {
- collectionContent <- c("animal", "animal/fish", "ball")
- fakeREST <- FakeRESTService$new(collectionContent)
-
+ fakeREST <- FakeRESTService$new()
api <- Arvados$new("myToken", "myHostName")
api$setRESTService(fakeREST)
collection <- Collection$new(api, "myUUID")
- files <- c("dog", "cat")
- collection$create(files, "animal")
+ collection$create(c("animal/dog", "animal/cat"))
dog <- collection$get("animal/dog")
cat <- collection$get("animal/cat")
collection <- Collection$new(api, "myUUID")
expect_that(collection$remove(10),
- throws_error("Expected character vector, got (numeric).",
+ throws_error("Expected character vector, got (numeric).",
fixed = TRUE))
})
api$setRESTService(fakeREST)
collection <- Collection$new(api, "myUUID")
- contentMatchExpected <- all(collection$getFileListing() ==
+ contentMatchExpected <- all(collection$getFileListing() ==
c("animal", "animal/fish", "ball"))
expect_that(contentMatchExpected, is_true())
expect_that(fishIsNotNull, is_true())
expect_that(fish$getName(), equals("fish"))
})
+
+test_that(paste("copy copies content to a new location inside file tree",
+ "and on REST service"), {
+
+ collectionContent <- c("animal", "animal/dog", "ball")
+ fakeREST <- FakeRESTService$new(collectionContent)
+
+ api <- Arvados$new("myToken", "myHostName")
+ api$setRESTService(fakeREST)
+ collection <- Collection$new(api, "myUUID")
+
+ collection$copy("animal/dog", "dog")
+
+ dogExistsOnOldLocation <- !is.null(collection$get("animal/dog"))
+ dogExistsOnNewLocation <- !is.null(collection$get("dog"))
+
+ expect_that(dogExistsOnOldLocation, is_true())
+ expect_that(dogExistsOnNewLocation, is_true())
+ expect_that(fakeREST$copyCallCount, equals(1))
+})
+
+test_that("copy raises exception if new location is not valid", {
+
+ collectionContent <- c("animal", "animal/fish", "ball")
+ fakeREST <- FakeRESTService$new(collectionContent)
+
+ api <- Arvados$new("myToken", "myHostName")
+ api$setRESTService(fakeREST)
+ collection <- Collection$new(api, "myUUID")
+
+ expect_that(collection$copy("fish", "object"),
+ throws_error("Content you want to copy doesn't exist in the collection.",
+ fixed = TRUE))
+})
+
+test_that("refresh invalidates current tree structure", {
+
+ collectionContent <- c("animal", "animal/fish", "ball")
+ fakeREST <- FakeRESTService$new(collectionContent)
+
+ api <- Arvados$new("myToken", "myHostName")
+ api$setRESTService(fakeREST)
+ collection <- Collection$new(api, "aaaaa-j7d0g-ccccccccccccccc")
+
+ # Before refresh
+ fish <- collection$get("animal/fish")
+ expect_that(fish$getName(), equals("fish"))
+ expect_that(fish$getCollection()$uuid, equals("aaaaa-j7d0g-ccccccccccccccc"))
+
+ collection$refresh()
+
+ # After refresh
+ expect_that(fish$getName(), equals("fish"))
+ expect_true(is.null(fish$getCollection()))
+})
test_that("constructor creates file tree from character array properly", {
collection <- "myCollection"
- characterArray <- c("animal",
+ characterArray <- c("animal",
"animal/dog",
"boat")
expect_that(boatIsOfTypeArvadosFile, is_true())
expect_that(boatsParentIsRoot, is_true())
expect_that(allElementsBelongToSameCollection, is_true())
-})
+})
test_that("getElement returns element from tree if element exists on specified path", {
collection <- "myCollection"
- characterArray <- c("animal",
+ characterArray <- c("animal",
"animal/dog",
"boat")
dog <- collectionTree$getElement("animal/dog")
expect_that(dog$getName(), equals("dog"))
-})
+})
test_that("getElement returns NULL from tree if element doesn't exists on specified path", {
collection <- "myCollection"
- characterArray <- c("animal",
+ characterArray <- c("animal",
"animal/dog",
"boat")
fishIsNULL <- is.null(fish)
expect_that(fishIsNULL, is_true())
-})
+})
test_that("getElement trims ./ from start of relativePath", {
collection <- "myCollection"
- characterArray <- c("animal",
+ characterArray <- c("animal",
"animal/dog",
"boat")
dogWithDotSlash <- collectionTree$getElement("./animal/dog")
expect_that(dogWithDotSlash$getName(), equals(dog$getName()))
-})
+})
test_that("getElement trims / from end of relativePath", {
collection <- "myCollection"
- characterArray <- c("animal",
+ characterArray <- c("animal",
"animal/dog",
"boat")
animalWithSlash <- collectionTree$getElement("animal/")
expect_that(animalWithSlash$getName(), equals(animal$getName()))
-})
+})
expect_that(barExists, is_true())
expect_that(unlist(result$bar$foo), equals(10))
-})
+})
test_that(paste("parseResponse generates and returns character vector",
"from server response if outputType is text"), {
parsedResponse <- parser$parseResponse(serverResponse, "text")
expect_that(parsedResponse, equals("random text"))
-})
+})
-webDAVResponseSample =
+webDAVResponseSample =
paste0("<?xml version=\"1.0\" encoding=\"UTF-8\"?><D:multistatus xmlns:",
"D=\"DAV:\"><D:response><D:href>/c=aaaaa-bbbbb-ccccccccccccccc</D",
":href><D:propstat><D:prop><D:resourcetype><D:collection xmlns:D=",
resultMatchExpected <- all.equal(result, expectedResult)
expect_that(resultMatchExpected, is_true())
-})
+})
test_that(paste("getFileSizesFromResponse returns file sizes",
"parsed from webDAV server response"), {
resultMatchExpected <- result == expectedResult
expect_that(resultMatchExpected, is_true())
-})
+})
context("Http Request")
-test_that("execyte raises exception if http verb is not valid", {
+test_that("execute raises exception if http verb is not valid", {
http <- HttpRequest$new()
expect_that(http$exec("FAKE VERB", "url"),
throws_error("Http verb is not valid."))
-})
+})
test_that("createQuery generates and encodes query portion of http", {
expect_that(http$createQuery(queryParams),
equals(paste0("/?filters=%5B%5B%22color%22%2C%22%3D%22%2C%22red",
"%22%5D%5D&limit=20&offset=50")))
-})
+})
test_that("createQuery generates and empty string when queryParams is an empty list", {
http <- HttpRequest$new()
expect_that(http$createQuery(list()), equals(""))
-})
+})
+
+test_that("exec calls httr functions correctly", {
+ httrNamespace <- getNamespace("httr")
+
+ # Monkeypatch httr functions and assert that they are called later
+ add_headersCalled <- FALSE
+ unlockBinding("add_headers", httrNamespace)
+ newAddHeaders <- function(h)
+ {
+ add_headersCalled <<- TRUE
+ list()
+ }
+ httrNamespace$add_headers <- newAddHeaders
+ lockBinding("add_headers", httrNamespace)
+
+ expectedConfig <- list()
+ retryCalled <- FALSE
+ unlockBinding("RETRY", httrNamespace)
+ newRETRY <- function(verb, url, body, config, times)
+ {
+ retryCalled <<- TRUE
+ expectedConfig <<- config
+ }
+ httrNamespace$RETRY <- newRETRY
+ lockBinding("RETRY", httrNamespace)
+
+ Sys.setenv("ARVADOS_API_HOST_INSECURE" = TRUE)
+ http <- HttpRequest$new()
+ http$exec("GET", "url")
+
+ expect_that(add_headersCalled, is_true())
+ expect_that(retryCalled, is_true())
+ expect_that(expectedConfig$options, equals(list(ssl_verifypeer = 0L)))
+})
+
+test_that("getConnection calls curl functions correctly", {
+ curlNamespace <- getNamespace("curl")
+
+ # Monkeypatch curl functions and assert that they are called later
+ curlCalled <- FALSE
+ unlockBinding("curl", curlNamespace)
+ newCurl <- function(url, open, handle) curlCalled <<- TRUE
+ curlNamespace$curl <- newCurl
+ lockBinding("curl", curlNamespace)
+
+ new_handleCalled <- FALSE
+ unlockBinding("new_handle", curlNamespace)
+ newHandleFun <- function()
+ {
+ new_handleCalled <<- TRUE
+ list()
+ }
+ curlNamespace$new_handle <- newHandleFun
+ lockBinding("new_handle", curlNamespace)
+
+ handle_setheadersCalled <- FALSE
+ unlockBinding("handle_setheaders", curlNamespace)
+ newHandleSetHeaders <- function(h, .list) handle_setheadersCalled <<- TRUE
+ curlNamespace$handle_setheaders <- newHandleSetHeaders
+ lockBinding("handle_setheaders", curlNamespace)
+
+ handle_setoptCalled <- FALSE
+ unlockBinding("handle_setopt", curlNamespace)
+ newHandleSetOpt <- function(h, ssl_verifypeer) handle_setoptCalled <<- TRUE
+ curlNamespace$handle_setopt <- newHandleSetOpt
+ lockBinding("handle_setopt", curlNamespace)
+
+
+ Sys.setenv("ARVADOS_API_HOST_INSECURE" = TRUE)
+ http <- HttpRequest$new()
+ http$getConnection("location", list(), "r")
+
+ expect_that(new_handleCalled, is_true())
+ expect_that(handle_setheadersCalled, is_true())
+ expect_that(handle_setoptCalled, is_true())
+ expect_that(curlCalled, is_true())
+})
expect_that(httpRequest$URLIsProperlyConfigured, is_true())
expect_that(httpRequest$requestHeaderContainsAuthorizationField, is_true())
expect_that(httpRequest$numberOfGETRequests, equals(1))
-})
+})
test_that("getWebDavHostName returns webDAV host name properly", {
REST <- RESTService$new("token", "host",
httpRequest, FakeHttpParser$new())
- expect_that("https://myWebDavServer.com", equals(REST$getWebDavHostName()))
-})
+ expect_that("https://myWebDavServer.com", equals(REST$getWebDavHostName()))
+})
test_that("create calls REST service properly", {
expect_that(fakeHttp$URLIsProperlyConfigured, is_true())
expect_that(fakeHttp$requestHeaderContainsAuthorizationField, is_true())
expect_that(fakeHttp$numberOfPUTRequests, equals(1))
-})
+})
test_that("create raises exception if server response code is not between 200 and 300", {
response$status_code <- 404
fakeHttp <- FakeHttpRequest$new(serverResponse = response)
- REST <- RESTService$new("token", "https://host/",
+ REST <- RESTService$new("token", "https://host/",
fakeHttp, HttpParser$new(),
0, "https://webDavHost/")
expect_that(REST$create("file", uuid),
throws_error("Server code: 404"))
-})
+})
test_that("delete calls REST service properly", {
fakeHttp <- FakeHttpRequest$new(expectedURL)
fakeHttpParser <- FakeHttpParser$new()
- REST <- RESTService$new("token", "https://host/",
+ REST <- RESTService$new("token", "https://host/",
fakeHttp, fakeHttpParser,
0, "https://webDavHost/")
expect_that(fakeHttp$URLIsProperlyConfigured, is_true())
expect_that(fakeHttp$requestHeaderContainsAuthorizationField, is_true())
expect_that(fakeHttp$numberOfDELETERequests, equals(1))
-})
+})
test_that("delete raises exception if server response code is not between 200 and 300", {
expect_that(REST$delete("file", uuid),
throws_error("Server code: 404"))
-})
+})
test_that("move calls REST service properly", {
fakeHttp <- FakeHttpRequest$new(expectedURL)
fakeHttpParser <- FakeHttpParser$new()
- REST <- RESTService$new("token", "https://host/",
+ REST <- RESTService$new("token", "https://host/",
fakeHttp, fakeHttpParser,
0, "https://webDavHost/")
expect_that(fakeHttp$requestHeaderContainsAuthorizationField, is_true())
expect_that(fakeHttp$requestHeaderContainsDestinationField, is_true())
expect_that(fakeHttp$numberOfMOVERequests, equals(1))
-})
+})
test_that("move raises exception if server response code is not between 200 and 300", {
response$status_code <- 404
fakeHttp <- FakeHttpRequest$new(serverResponse = response)
- REST <- RESTService$new("token", "https://host/",
+ REST <- RESTService$new("token", "https://host/",
fakeHttp, HttpParser$new(),
0, "https://webDavHost/")
expect_that(REST$move("file", "newDestination/file", uuid),
throws_error("Server code: 404"))
-})
+})
+
+test_that("copy calls REST service properly", {
+
+ uuid <- "aaaaa-j7d0g-ccccccccccccccc"
+ expectedURL <- "https://webDavHost/c=aaaaa-j7d0g-ccccccccccccccc/file"
+ fakeHttp <- FakeHttpRequest$new(expectedURL)
+ fakeHttpParser <- FakeHttpParser$new()
+
+ REST <- RESTService$new("token", "https://host/",
+ fakeHttp, fakeHttpParser,
+ 0, "https://webDavHost/")
+
+ REST$copy("file", "newDestination/file", uuid)
+
+ expect_that(fakeHttp$URLIsProperlyConfigured, is_true())
+ expect_that(fakeHttp$requestHeaderContainsAuthorizationField, is_true())
+ expect_that(fakeHttp$requestHeaderContainsDestinationField, is_true())
+ expect_that(fakeHttp$numberOfCOPYRequests, equals(1))
+})
+
+test_that("copy raises exception if server response code is not between 200 and 300", {
+
+ uuid <- "aaaaa-j7d0g-ccccccccccccccc"
+ response <- list()
+ response$status_code <- 404
+ fakeHttp <- FakeHttpRequest$new(serverResponse = response)
+
+ REST <- RESTService$new("token", "https://host/",
+ fakeHttp, HttpParser$new(),
+ 0, "https://webDavHost/")
+
+ expect_that(REST$copy("file", "newDestination/file", uuid),
+ throws_error("Server code: 404"))
+})
test_that("getCollectionContent retreives correct content from WebDAV server", {
fakeHttp <- FakeHttpRequest$new(expectedURL, returnContent)
- REST <- RESTService$new("token", "https://host/",
+ REST <- RESTService$new("token", "https://host/",
fakeHttp, FakeHttpParser$new(),
0, "https://webDavHost/")
expect_that(returnedContentMatchExpected, is_true())
expect_that(fakeHttp$requestHeaderContainsAuthorizationField, is_true())
-})
+})
test_that("getCollectionContent raises exception if server returns empty response", {
response <- ""
fakeHttp <- FakeHttpRequest$new(serverResponse = response)
- REST <- RESTService$new("token", "https://host/",
+ REST <- RESTService$new("token", "https://host/",
fakeHttp, FakeHttpParser$new(),
0, "https://webDavHost/")
expect_that(REST$getCollectionContent(uuid),
throws_error("Response is empty, request may be misconfigured"))
-})
+})
test_that("getCollectionContent parses server response", {
uuid <- "aaaaa-j7d0g-ccccccccccccccc"
fakeHttpParser <- FakeHttpParser$new()
- REST <- RESTService$new("token", "https://host/",
+ REST <- RESTService$new("token", "https://host/",
FakeHttpRequest$new(), fakeHttpParser,
0, "https://webDavHost/")
REST$getCollectionContent(uuid)
expect_that(fakeHttpParser$parserCallCount, equals(1))
-})
+})
test_that("getCollectionContent raises exception if server returns empty response", {
response <- ""
fakeHttp <- FakeHttpRequest$new(serverResponse = response)
- REST <- RESTService$new("token", "https://host/",
+ REST <- RESTService$new("token", "https://host/",
fakeHttp, FakeHttpParser$new(),
0, "https://webDavHost/")
expect_that(REST$getCollectionContent(uuid),
throws_error("Response is empty, request may be misconfigured"))
-})
+})
test_that(paste("getCollectionContent raises exception if server",
"response code is not between 200 and 300"), {
response$status_code <- 404
fakeHttp <- FakeHttpRequest$new(serverResponse = response)
- REST <- RESTService$new("token", "https://host/",
+ REST <- RESTService$new("token", "https://host/",
fakeHttp, HttpParser$new(),
0, "https://webDavHost/")
expect_that(REST$getCollectionContent(uuid),
throws_error("Server code: 404"))
-})
+})
test_that("getResourceSize calls REST service properly", {
expect_that(fakeHttp$URLIsProperlyConfigured, is_true())
expect_that(fakeHttp$requestHeaderContainsAuthorizationField, is_true())
expect_that(returnedContentMatchExpected, is_true())
-})
+})
test_that("getResourceSize raises exception if server returns empty response", {
response <- ""
fakeHttp <- FakeHttpRequest$new(serverResponse = response)
- REST <- RESTService$new("token", "https://host/",
+ REST <- RESTService$new("token", "https://host/",
fakeHttp, FakeHttpParser$new(),
0, "https://webDavHost/")
expect_that(REST$getResourceSize("file", uuid),
throws_error("Response is empty, request may be misconfigured"))
-})
+})
test_that(paste("getResourceSize raises exception if server",
"response code is not between 200 and 300"), {
response$status_code <- 404
fakeHttp <- FakeHttpRequest$new(serverResponse = response)
- REST <- RESTService$new("token", "https://host/",
+ REST <- RESTService$new("token", "https://host/",
fakeHttp, HttpParser$new(),
0, "https://webDavHost/")
expect_that(REST$getResourceSize("file", uuid),
throws_error("Server code: 404"))
-})
+})
test_that("getResourceSize parses server response", {
uuid <- "aaaaa-j7d0g-ccccccccccccccc"
fakeHttpParser <- FakeHttpParser$new()
- REST <- RESTService$new("token", "https://host/",
+ REST <- RESTService$new("token", "https://host/",
FakeHttpRequest$new(), fakeHttpParser,
0, "https://webDavHost/")
REST$getResourceSize("file", uuid)
expect_that(fakeHttpParser$parserCallCount, equals(1))
-})
+})
test_that("read calls REST service properly", {
fakeHttp <- FakeHttpRequest$new(expectedURL, serverResponse)
- REST <- RESTService$new("token", "https://host/",
+ REST <- RESTService$new("token", "https://host/",
fakeHttp, FakeHttpParser$new(),
0, "https://webDavHost/")
expect_that(fakeHttp$requestHeaderContainsAuthorizationField, is_true())
expect_that(fakeHttp$requestHeaderContainsRangeField, is_true())
expect_that(returnResult, equals("file content"))
-})
+})
test_that("read raises exception if server response code is not between 200 and 300", {
response$status_code <- 404
fakeHttp <- FakeHttpRequest$new(serverResponse = response)
- REST <- RESTService$new("token", "https://host/",
+ REST <- RESTService$new("token", "https://host/",
fakeHttp, HttpParser$new(),
0, "https://webDavHost/")
expect_that(REST$read("file", uuid),
throws_error("Server code: 404"))
-})
+})
test_that("read raises exception if contentType is not valid", {
uuid <- "aaaaa-j7d0g-ccccccccccccccc"
fakeHttp <- FakeHttpRequest$new()
- REST <- RESTService$new("token", "https://host/",
+ REST <- RESTService$new("token", "https://host/",
fakeHttp, HttpParser$new(),
0, "https://webDavHost/")
expect_that(REST$read("file", uuid, "some invalid content type"),
throws_error("Invalid contentType. Please use text or raw."))
-})
+})
test_that("read parses server response", {
uuid <- "aaaaa-j7d0g-ccccccccccccccc"
fakeHttpParser <- FakeHttpParser$new()
- REST <- RESTService$new("token", "https://host/",
+ REST <- RESTService$new("token", "https://host/",
FakeHttpRequest$new(), fakeHttpParser,
0, "https://webDavHost/")
REST$read("file", uuid, "text", 1024, 512)
expect_that(fakeHttpParser$parserCallCount, equals(1))
-})
+})
test_that("write calls REST service properly", {
- fileContent <- "new file content"
+ fileContent <- "new file content"
uuid <- "aaaaa-j7d0g-ccccccccccccccc"
expectedURL <- "https://webDavHost/c=aaaaa-j7d0g-ccccccccccccccc/file"
fakeHttp <- FakeHttpRequest$new(expectedURL)
- REST <- RESTService$new("token", "https://host/",
+ REST <- RESTService$new("token", "https://host/",
fakeHttp, FakeHttpParser$new(),
0, "https://webDavHost/")
expect_that(fakeHttp$requestBodyIsProvided, is_true())
expect_that(fakeHttp$requestHeaderContainsAuthorizationField, is_true())
expect_that(fakeHttp$requestHeaderContainsContentTypeField, is_true())
-})
+})
test_that("write raises exception if server response code is not between 200 and 300", {
uuid <- "aaaaa-j7d0g-ccccccccccccccc"
- fileContent <- "new file content"
+ fileContent <- "new file content"
response <- list()
response$status_code <- 404
fakeHttp <- FakeHttpRequest$new(serverResponse = response)
expect_that(REST$write("file", uuid, fileContent, "text/html"),
throws_error("Server code: 404"))
-})
+})
+
+test_that("getConnection calls REST service properly", {
+ uuid <- "aaaaa-j7d0g-ccccccccccccccc"
+ fakeHttp <- FakeHttpRequest$new()
+
+ REST <- RESTService$new("token", "https://host/",
+ fakeHttp, FakeHttpParser$new(),
+ 0, "https://webDavHost/")
+
+ REST$getConnection("file", uuid, "r")
+
+ expect_that(fakeHttp$numberOfgetConnectionCalls, equals(1))
+})
expect_that(animal$getRelativePath(), equals("animal"))
expect_that(fish$getRelativePath(), equals("animal/fish"))
-})
+})
test_that(paste("getFileListing by default returns sorted path of all files",
"relative to the current subcollection"), {
all(expectedResult == result)
expect_that(resultsMatch, is_true())
-})
+})
test_that(paste("getFileListing returns sorted names of all direct children",
"if fullPath is set to FALSE"), {
all(expectedResult == result)
expect_that(resultsMatch, is_true())
-})
+})
test_that("add adds content to inside collection tree", {
expect_that(animalContainsFish, is_true())
expect_that(animalContainsDog, is_true())
-})
+})
test_that("add raises exception if content name is empty string", {
throws_error("Content has invalid name.", fixed = TRUE))
})
-test_that(paste("add raises exception if ArvadosFile/Subcollection",
+test_that(paste("add raises exception if ArvadosFile/Subcollection",
"with same name already exists in the subcollection"), {
animal <- Subcollection$new("animal")
expect_that(animal$add(thirdFish),
throws_error(paste("Subcollection already contains ArvadosFile or",
"Subcollection with same name."), fixed = TRUE))
-})
+})
-test_that(paste("add raises exception if passed argument is",
+test_that(paste("add raises exception if passed argument is",
"not ArvadosFile or Subcollection"), {
animal <- Subcollection$new("animal")
expect_that(animal$add(number),
throws_error(paste("Expected AravodsFile or Subcollection object,",
"got (numeric)."), fixed = TRUE))
-})
+})
-test_that(paste("add post content to a REST service",
+test_that(paste("add post content to a REST service",
"if subcollection belongs to a collection"), {
-
+
collectionContent <- c("animal", "animal/fish")
fakeREST <- FakeRESTService$new(collectionContent)
animal$add(dog)
expect_that(fakeREST$createCallCount, equals(1))
-})
+})
test_that("remove removes content from subcollection", {
returnValueAfterRemovalIsNull <- is.null(animal$get("fish"))
expect_that(returnValueAfterRemovalIsNull, is_true())
-})
+})
-test_that(paste("remove raises exception",
+test_that(paste("remove raises exception",
"if content to remove doesn't exist in the subcollection"), {
animal <- Subcollection$new("animal")
expect_that(animal$remove("fish"),
throws_error(paste("Subcollection doesn't contains ArvadosFile",
"or Subcollection with specified name.")))
-})
+})
test_that("remove raises exception if passed argument is not character vector", {
expect_that(animal$remove(number),
throws_error(paste("Expected character,",
"got (numeric)."), fixed = TRUE))
-})
+})
-test_that(paste("remove removes content from REST service",
+test_that(paste("remove removes content from REST service",
"if subcollection belongs to a collection"), {
-
+
collectionContent <- c("animal", "animal/fish", "animal/dog")
fakeREST <- FakeRESTService$new(collectionContent)
animal$remove("fish")
expect_that(fakeREST$deleteCallCount, equals(1))
-})
+})
-test_that(paste("get returns ArvadosFile or Subcollection",
+test_that(paste("get returns ArvadosFile or Subcollection",
"if file or folder with given name exists"), {
animal <- Subcollection$new("animal")
expect_that(returnedDogIsArvadosFile, is_true())
expect_that(returnedDog$getName(), equals("dog"))
-})
+})
-test_that(paste("get returns NULL if file or folder",
+test_that(paste("get returns NULL if file or folder",
"with given name doesn't exists"), {
animal <- Subcollection$new("animal")
returnedDogIsNull <- is.null(animal$get("dog"))
expect_that(returnedDogIsNull, is_true())
-})
+})
test_that("getFirst returns first child in the subcollection", {
animal$add(fish)
expect_that(animal$getFirst()$getName(), equals("fish"))
-})
+})
test_that("getFirst returns NULL if subcollection contains no children", {
returnedElementIsNull <- is.null(animal$getFirst())
expect_that(returnedElementIsNull, is_true())
-})
+})
test_that(paste("setCollection by default sets collection",
"filed of subcollection and all its children"), {
expect_that(animal$getCollection(), equals("myCollection"))
expect_that(fish$getCollection(), equals("myCollection"))
-})
+})
test_that(paste("setCollection sets collection filed of subcollection only",
"if parameter setRecursively is set to FALSE"), {
expect_that(animal$getCollection(), equals("myCollection"))
expect_that(fishCollectionIsNull, is_true())
-})
+})
test_that(paste("move raises exception if subcollection",
"doesn't belong to any collection"), {
expect_that(animal$move("new/location"),
throws_error("Subcollection doesn't belong to any collection"))
-})
+})
test_that("move raises exception if new location contains content with the same name", {
expect_that(fish$move("fish"),
throws_error("Destination already contains content with same name."))
-})
+})
test_that(paste("move raises exception if newLocationInCollection",
"parameter is invalid"), {
fish <- collection$get("animal/fish")
expect_that(fish$move("objects/dog"),
- throws_error("Unable to get destination subcollection"))
-})
+ throws_error("Unable to get destination subcollection."))
+})
test_that("move moves subcollection inside collection tree", {
expect_that(fishIsNullOnOldLocation, is_true())
expect_that(fishExistsOnNewLocation, is_true())
-})
+})
test_that(paste("getSizeInBytes returns zero if subcollection",
"is not part of a collection"), {
animal <- Subcollection$new("animal")
expect_that(animal$getSizeInBytes(), equals(0))
-})
+})
test_that(paste("getSizeInBytes delegates size calculation",
"to REST service class"), {
expect_that(resourceSize, equals(100))
})
+
+#########################
+test_that(paste("copy raises exception if subcollection",
+ "doesn't belong to any collection"), {
+
+ animal <- Subcollection$new("animal")
+
+ expect_that(animal$copy("new/location"),
+ throws_error("Subcollection doesn't belong to any collection."))
+})
+
+test_that("copy raises exception if new location contains content with the same name", {
+
+ collectionContent <- c("animal",
+ "animal/fish",
+ "animal/dog",
+ "animal/fish/shark",
+ "fish")
+ fakeREST <- FakeRESTService$new(collectionContent)
+
+ api <- Arvados$new("myToken", "myHostName")
+ api$setRESTService(fakeREST)
+ collection <- Collection$new(api, "myUUID")
+ fish <- collection$get("animal/fish")
+
+ expect_that(fish$copy("fish"),
+ throws_error("Destination already contains content with same name."))
+
+})
+
+test_that(paste("copy raises exception if location parameter is invalid"), {
+
+ collectionContent <- c("animal",
+ "animal/fish",
+ "animal/dog",
+ "animal/fish/shark",
+ "ball")
+ fakeREST <- FakeRESTService$new(collectionContent)
+
+ api <- Arvados$new("myToken", "myHostName")
+ api$setRESTService(fakeREST)
+
+ collection <- Collection$new(api, "myUUID")
+ fish <- collection$get("animal/fish")
+
+ expect_that(fish$copy("objects/dog"),
+ throws_error("Unable to get destination subcollection."))
+})
+
+test_that("copy copies subcollection inside collection tree", {
+
+ collectionContent <- c("animal",
+ "animal/fish",
+ "animal/dog",
+ "animal/fish/shark",
+ "ball")
+ fakeREST <- FakeRESTService$new(collectionContent)
+
+ api <- Arvados$new("myToken", "myHostName")
+ api$setRESTService(fakeREST)
+ collection <- Collection$new(api, "myUUID")
+ fish <- collection$get("animal/fish")
+
+ fish$copy("fish")
+ fishExistsOnOldLocation <- !is.null(collection$get("animal/fish"))
+ fishExistsOnNewLocation <- !is.null(collection$get("fish"))
+
+ expect_that(fishExistsOnOldLocation, is_true())
+ expect_that(fishExistsOnNewLocation, is_true())
+})
+
+test_that("duplicate performs deep cloning of Subcollection", {
+ foo <- ArvadosFile$new("foo")
+ bar <- ArvadosFile$new("bar")
+ sub <- Subcollection$new("qux")
+ sub$add(foo)
+ sub$add(bar)
+
+ newSub1 <- sub$duplicate()
+ newSub2 <- sub$duplicate("quux")
+
+ expect_that(newSub1$getFileListing(), equals(sub$getFileListing()))
+ expect_that(sort(newSub2$getFileListing()), equals(c("quux/bar", "quux/foo")))
+})
result <- listAll(testFunction)
expect_that(length(result), equals(8))
-})
+})
test_that("trimFromStart trims string correctly if string starts with trimCharacters", {
result <- trimFromStart(sample, trimCharacters)
expect_that(result, equals("random"))
-})
+})
test_that("trimFromStart returns original string if string doesn't starts with trimCharacters", {
result <- trimFromStart(sample, trimCharacters)
expect_that(result, equals("./something/random"))
-})
+})
test_that("trimFromEnd trims string correctly if string ends with trimCharacters", {
result <- trimFromEnd(sample, trimCharacters)
expect_that(result, equals("./something"))
-})
+})
test_that("trimFromEnd returns original string if string doesn't end with trimCharacters", {
result <- trimFromStart(sample, trimCharacters)
expect_that(result, equals("./something/random"))
-})
+})
test_that("RListToPythonList converts nested R list to char representation of Python list", {
- sample <- list("insert", list("random", list("text")), list("here"))
+ sample <- list("insert", list("random", list("text")), list("here"))
result <- RListToPythonList(sample)
resultWithSeparator <- RListToPythonList(sample, separator = ",+")
expect_that(result, equals("[\"insert\", [\"random\", \"text\"], \"here\"]"))
expect_that(resultWithSeparator,
equals("[\"insert\",+[\"random\",+\"text\"],+\"here\"]"))
-})
+})
test_that("appendToStartIfNotExist appends characters to beginning of a string", {
result <- appendToStartIfNotExist(sample, charactersToAppend)
expect_that(result, equals("Happy New Year"))
-})
+})
test_that(paste("appendToStartIfNotExist returns original string if string",
"doesn't start with specified characters"), {
result <- appendToStartIfNotExist(sample, charactersToAppend)
expect_that(result, equals("Happy New Year"))
-})
+})
test_that(paste("splitToPathAndName splits relative path to file/folder",
"name and rest of the path"), {
expect_that(result$name, equals("file.exe"))
expect_that(result$path, equals("path/to/my"))
-})
+})
s.executables << "arv-crunch-job"
s.executables << "arv-tag"
s.required_ruby_version = '>= 2.1.0'
- s.add_runtime_dependency 'arvados', '~> 0.1', '>= 0.1.20150128223554'
+ s.add_runtime_dependency 'arvados', '~> 1.1.0', '>= 1.1.4'
# Our google-api-client dependency used to be < 0.9, but that could be
# satisfied by the buggy 0.9.pre*. https://dev.arvados.org/issues/9213
s.add_runtime_dependency 'google-api-client', '~> 0.6', '>= 0.6.3', '<0.8.9'
s.add_runtime_dependency 'json', '>= 1.7.7', '<3'
s.add_runtime_dependency 'trollop', '~> 2.0'
s.add_runtime_dependency 'andand', '~> 1.3', '>= 1.3.3'
- s.add_runtime_dependency 'oj', '~> 2.0', '>= 2.0.3'
+ s.add_runtime_dependency 'oj', '~> 3.0'
s.add_runtime_dependency 'curb', '~> 0.8'
s.homepage =
'https://arvados.org'
my $cgroup_root = "/sys/fs/cgroup";
my $docker_bin = "docker.io";
my $docker_run_args = "";
+my $srun_sync_timeout = 15*60;
GetOptions('force-unlock' => \$force_unlock,
'git-dir=s' => \$git_dir,
'job=s' => \$jobspec,
'cgroup-root=s' => \$cgroup_root,
'docker-bin=s' => \$docker_bin,
'docker-run-args=s' => \$docker_run_args,
+ 'srun-sync-timeout=i' => \$srun_sync_timeout,
);
if (defined $job_api_token) {
my ($stdout_r, $stdout_w);
pipe $stdout_r, $stdout_w or croak("pipe() failed: $!");
+ my $started_srun = scalar time;
+
my $srunpid = fork();
if ($srunpid == 0)
{
if (!$busy) {
select(undef, undef, undef, 0.1);
}
+ if (($started_srun + $srun_sync_timeout) < scalar time) {
+ # Exceeded general timeout for "srun_sync" operations, likely
+ # means something got stuck on the remote node.
+ Log(undef, "srun_sync exceeded timeout, will fail.");
+ $main::please_freeze = 1;
+ }
killem(keys %proc) if $main::please_freeze;
}
my $exited = $?;
import cwltool.process
from schema_salad.sourceline import SourceLine
import schema_salad.validate as validate
+import cwltool.argparser
import arvados
import arvados.config
from .perf import Perf
from .pathmapper import NoFollowPathMapper
from .task_queue import TaskQueue
+from .context import ArvLoadingContext, ArvRuntimeContext
from ._version import __version__
from cwltool.pack import pack
from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing
from cwltool.command_line_tool import compute_checksums
+
from arvados.api import OrderedJsonModel
logger = logging.getLogger('arvados.cwl-runner')
"""
- def __init__(self, api_client, work_api=None, keep_client=None,
- output_name=None, output_tags=None, num_retries=4,
+ def __init__(self, api_client,
+ arvargs=None,
+ keep_client=None,
+ num_retries=4,
thread_count=4):
+
+ if arvargs is None:
+ arvargs = argparse.Namespace()
+ arvargs.work_api = None
+ arvargs.output_name = None
+ arvargs.output_tags = None
+ arvargs.thread_count = 1
+
self.api = api_client
self.processes = {}
self.workflow_eval_lock = threading.Condition(threading.RLock())
self.final_output = None
self.final_status = None
- self.uploaded = {}
self.num_retries = num_retries
self.uuid = None
self.stop_polling = threading.Event()
self.poll_api = None
self.pipeline = None
self.final_output_collection = None
- self.output_name = output_name
- self.output_tags = output_tags
+ self.output_name = arvargs.output_name
+ self.output_tags = arvargs.output_tags
self.project_uuid = None
self.intermediate_output_ttl = 0
self.intermediate_output_collections = []
self.trash_intermediate = False
- self.thread_count = thread_count
+ self.thread_count = arvargs.thread_count
self.poll_interval = 12
+ self.loadingContext = None
if keep_client is not None:
self.keep_client = keep_client
try:
methods = self.api._rootDesc.get('resources')[api]['methods']
if ('httpMethod' in methods['create'] and
- (work_api == api or work_api is None)):
+ (arvargs.work_api == api or arvargs.work_api is None)):
self.work_api = api
break
except KeyError:
pass
if not self.work_api:
- if work_api is None:
+ if arvargs.work_api is None:
raise Exception("No supported APIs")
else:
raise Exception("Unsupported API '%s', expected one of %s" % (work_api, expected_api))
- def arv_make_tool(self, toolpath_object, **kwargs):
- kwargs["work_api"] = self.work_api
- kwargs["fetcher_constructor"] = self.fetcher_constructor
- kwargs["resolver"] = partial(collectionResolver, self.api, num_retries=self.num_retries)
+ if self.work_api == "jobs":
+ logger.warn("""
+*******************************
+Using the deprecated 'jobs' API.
+
+To get rid of this warning:
+
+Users: read about migrating at
+http://doc.arvados.org/user/cwl/cwl-style.html#migrate
+and use the option --api=containers
+
+Admins: configure the cluster to disable the 'jobs' API as described at:
+http://doc.arvados.org/install/install-api-server.html#disable_api_methods
+*******************************""")
+
+ self.loadingContext = ArvLoadingContext(vars(arvargs))
+ self.loadingContext.fetcher_constructor = self.fetcher_constructor
+ self.loadingContext.resolver = partial(collectionResolver, self.api, num_retries=self.num_retries)
+ self.loadingContext.construct_tool_object = self.arv_make_tool
+
+
+ def arv_make_tool(self, toolpath_object, loadingContext):
if "class" in toolpath_object and toolpath_object["class"] == "CommandLineTool":
- return ArvadosCommandTool(self, toolpath_object, **kwargs)
+ return ArvadosCommandTool(self, toolpath_object, loadingContext)
elif "class" in toolpath_object and toolpath_object["class"] == "Workflow":
- return ArvadosWorkflow(self, toolpath_object, **kwargs)
+ return ArvadosWorkflow(self, toolpath_object, loadingContext)
else:
- return cwltool.workflow.defaultMakeTool(toolpath_object, **kwargs)
+ return cwltool.workflow.default_make_tool(toolpath_object, loadingContext)
def output_callback(self, out, processStatus):
with self.workflow_eval_lock:
self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
body={"state": "Complete"}).execute(num_retries=self.num_retries)
else:
- logger.warn("Overall process status is %s", processStatus)
+ logger.error("Overall process status is %s", processStatus)
if self.pipeline:
self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
body={"state": "Failed"}).execute(num_retries=self.num_retries)
self.workflow_eval_lock.notifyAll()
- def start_run(self, runnable, kwargs):
- self.task_queue.add(partial(runnable.run, **kwargs))
+ def start_run(self, runnable, runtimeContext):
+ self.task_queue.add(partial(runnable.run, runtimeContext))
def process_submitted(self, container):
with self.workflow_eval_lock:
finally:
self.stop_polling.set()
- def get_uploaded(self):
- return self.uploaded.copy()
-
- def add_uploaded(self, src, pair):
- self.uploaded[src] = pair
-
def add_intermediate_output(self, uuid):
if uuid:
self.intermediate_output_collections.append(uuid)
with SourceLine(obj, i, UnsupportedRequirement, logger.isEnabledFor(logging.DEBUG)):
self.check_features(v)
- def make_output_collection(self, name, tagsString, outputObj):
+ def make_output_collection(self, name, storage_classes, tagsString, outputObj):
outputObj = copy.deepcopy(outputObj)
files = []
with final.open("cwl.output.json", "w") as f:
json.dump(outputObj, f, sort_keys=True, indent=4, separators=(',',': '))
- final.save_new(name=name, owner_uuid=self.project_uuid, ensure_unique_name=True)
+ final.save_new(name=name, owner_uuid=self.project_uuid, storage_classes=storage_classes, ensure_unique_name=True)
logger.info("Final output collection %s \"%s\" (%s)", final.portable_data_hash(),
final.api_response()["name"],
'progress':1.0
}).execute(num_retries=self.num_retries)
- def arv_executor(self, tool, job_order, **kwargs):
- self.debug = kwargs.get("debug")
+ def arv_executor(self, tool, job_order, runtimeContext, logger=None):
+ self.debug = runtimeContext.debug
tool.visit(self.check_features)
- self.project_uuid = kwargs.get("project_uuid")
+ self.project_uuid = runtimeContext.project_uuid
self.pipeline = None
- make_fs_access = kwargs.get("make_fs_access") or partial(CollectionFsAccess,
- collection_cache=self.collection_cache)
- self.fs_access = make_fs_access(kwargs["basedir"])
- self.secret_store = kwargs.get("secret_store")
+ self.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
+ self.secret_store = runtimeContext.secret_store
- self.trash_intermediate = kwargs["trash_intermediate"]
+ self.trash_intermediate = runtimeContext.trash_intermediate
if self.trash_intermediate and self.work_api != "containers":
raise Exception("--trash-intermediate is only supported with --api=containers.")
- self.intermediate_output_ttl = kwargs["intermediate_output_ttl"]
+ self.intermediate_output_ttl = runtimeContext.intermediate_output_ttl
if self.intermediate_output_ttl and self.work_api != "containers":
raise Exception("--intermediate-output-ttl is only supported with --api=containers.")
if self.intermediate_output_ttl < 0:
raise Exception("Invalid value %d for --intermediate-output-ttl, cannot be less than zero" % self.intermediate_output_ttl)
- if kwargs.get("submit_request_uuid") and self.work_api != "containers":
+ if runtimeContext.submit_request_uuid and self.work_api != "containers":
raise Exception("--submit-request-uuid requires containers API, but using '{}' api".format(self.work_api))
- if not kwargs.get("name"):
- kwargs["name"] = self.name = tool.tool.get("label") or tool.metadata.get("label") or os.path.basename(tool.tool["id"])
+ if not runtimeContext.name:
+ runtimeContext.name = self.name = tool.tool.get("label") or tool.metadata.get("label") or os.path.basename(tool.tool["id"])
# Upload direct dependencies of workflow steps, get back mapping of files to keep references.
# Also uploads docker images.
# Reload tool object which may have been updated by
# upload_workflow_deps
# Don't validate this time because it will just print redundant errors.
+ loadingContext = self.loadingContext.copy()
+ loadingContext.loader = tool.doc_loader
+ loadingContext.avsc_names = tool.doc_schema
+ loadingContext.metadata = tool.metadata
+ loadingContext.do_validate = False
+
tool = self.arv_make_tool(tool.doc_loader.idx[tool.tool["id"]],
- makeTool=self.arv_make_tool,
- loader=tool.doc_loader,
- avsc_names=tool.doc_schema,
- metadata=tool.metadata,
- do_validate=False)
+ loadingContext)
# Upload local file references in the job order.
- job_order = upload_job_order(self, "%s input" % kwargs["name"],
+ job_order = upload_job_order(self, "%s input" % runtimeContext.name,
tool, job_order)
- existing_uuid = kwargs.get("update_workflow")
- if existing_uuid or kwargs.get("create_workflow"):
+ existing_uuid = runtimeContext.update_workflow
+ if existing_uuid or runtimeContext.create_workflow:
# Create a pipeline template or workflow record and exit.
if self.work_api == "jobs":
tmpl = RunnerTemplate(self, tool, job_order,
- kwargs.get("enable_reuse"),
+ runtimeContext.enable_reuse,
uuid=existing_uuid,
- submit_runner_ram=kwargs.get("submit_runner_ram"),
- name=kwargs["name"],
+ submit_runner_ram=runtimeContext.submit_runner_ram,
+ name=runtimeContext.name,
merged_map=merged_map)
tmpl.save()
# cwltool.main will write our return value to stdout.
return (upload_workflow(self, tool, job_order,
self.project_uuid,
uuid=existing_uuid,
- submit_runner_ram=kwargs.get("submit_runner_ram"),
- name=kwargs["name"],
+ submit_runner_ram=runtimeContext.submit_runner_ram,
+ name=runtimeContext.name,
merged_map=merged_map),
"success")
- self.ignore_docker_for_reuse = kwargs.get("ignore_docker_for_reuse")
- self.eval_timeout = kwargs.get("eval_timeout")
+ self.ignore_docker_for_reuse = runtimeContext.ignore_docker_for_reuse
+ self.eval_timeout = runtimeContext.eval_timeout
- kwargs["make_fs_access"] = make_fs_access
- kwargs["enable_reuse"] = kwargs.get("enable_reuse")
- kwargs["use_container"] = True
- kwargs["tmpdir_prefix"] = "tmp"
- kwargs["compute_checksum"] = kwargs.get("compute_checksum")
+ runtimeContext = runtimeContext.copy()
+ runtimeContext.use_container = True
+ runtimeContext.tmpdir_prefix = "tmp"
+ runtimeContext.work_api = self.work_api
if self.work_api == "containers":
if self.ignore_docker_for_reuse:
raise Exception("--ignore-docker-for-reuse not supported with containers API.")
- kwargs["outdir"] = "/var/spool/cwl"
- kwargs["docker_outdir"] = "/var/spool/cwl"
- kwargs["tmpdir"] = "/tmp"
- kwargs["docker_tmpdir"] = "/tmp"
+ runtimeContext.outdir = "/var/spool/cwl"
+ runtimeContext.docker_outdir = "/var/spool/cwl"
+ runtimeContext.tmpdir = "/tmp"
+ runtimeContext.docker_tmpdir = "/tmp"
elif self.work_api == "jobs":
- if kwargs["priority"] != DEFAULT_PRIORITY:
+ if runtimeContext.priority != DEFAULT_PRIORITY:
raise Exception("--priority not implemented for jobs API.")
- kwargs["outdir"] = "$(task.outdir)"
- kwargs["docker_outdir"] = "$(task.outdir)"
- kwargs["tmpdir"] = "$(task.tmpdir)"
+ runtimeContext.outdir = "$(task.outdir)"
+ runtimeContext.docker_outdir = "$(task.outdir)"
+ runtimeContext.tmpdir = "$(task.tmpdir)"
- if kwargs["priority"] < 1 or kwargs["priority"] > 1000:
+ if runtimeContext.priority < 1 or runtimeContext.priority > 1000:
raise Exception("--priority must be in the range 1..1000.")
runnerjob = None
- if kwargs.get("submit"):
+ if runtimeContext.submit:
# Submit a runner job to run the workflow for us.
if self.work_api == "containers":
- if tool.tool["class"] == "CommandLineTool" and kwargs.get("wait"):
- kwargs["runnerjob"] = tool.tool["id"]
+ if tool.tool["class"] == "CommandLineTool" and runtimeContext.wait:
+ runtimeContext.runnerjob = tool.tool["id"]
runnerjob = tool.job(job_order,
self.output_callback,
- **kwargs).next()
+ runtimeContext).next()
else:
- runnerjob = RunnerContainer(self, tool, job_order, kwargs.get("enable_reuse"),
+ runnerjob = RunnerContainer(self, tool, job_order, runtimeContext.enable_reuse,
self.output_name,
self.output_tags,
- submit_runner_ram=kwargs.get("submit_runner_ram"),
- name=kwargs.get("name"),
- on_error=kwargs.get("on_error"),
- submit_runner_image=kwargs.get("submit_runner_image"),
- intermediate_output_ttl=kwargs.get("intermediate_output_ttl"),
+ submit_runner_ram=runtimeContext.submit_runner_ram,
+ name=runtimeContext.name,
+ on_error=runtimeContext.on_error,
+ submit_runner_image=runtimeContext.submit_runner_image,
+ intermediate_output_ttl=runtimeContext.intermediate_output_ttl,
merged_map=merged_map,
- priority=kwargs.get("priority"),
+ priority=runtimeContext.priority,
secret_store=self.secret_store)
elif self.work_api == "jobs":
- runnerjob = RunnerJob(self, tool, job_order, kwargs.get("enable_reuse"),
+ runnerjob = RunnerJob(self, tool, job_order, runtimeContext.enable_reuse,
self.output_name,
self.output_tags,
- submit_runner_ram=kwargs.get("submit_runner_ram"),
- name=kwargs.get("name"),
- on_error=kwargs.get("on_error"),
- submit_runner_image=kwargs.get("submit_runner_image"),
+ submit_runner_ram=runtimeContext.submit_runner_ram,
+ name=runtimeContext.name,
+ on_error=runtimeContext.on_error,
+ submit_runner_image=runtimeContext.submit_runner_image,
merged_map=merged_map)
- elif "cwl_runner_job" not in kwargs and self.work_api == "jobs":
+ elif runtimeContext.cwl_runner_job is None and self.work_api == "jobs":
# Create pipeline for local run
self.pipeline = self.api.pipeline_instances().create(
body={
"owner_uuid": self.project_uuid,
- "name": kwargs["name"] if kwargs.get("name") else shortname(tool.tool["id"]),
+ "name": runtimeContext.name if runtimeContext.name else shortname(tool.tool["id"]),
"components": {},
"state": "RunningOnClient"}).execute(num_retries=self.num_retries)
logger.info("Pipeline instance %s", self.pipeline["uuid"])
- if runnerjob and not kwargs.get("wait"):
- submitargs = kwargs.copy()
- submitargs['submit'] = False
- runnerjob.run(**submitargs)
+ if runnerjob and not runtimeContext.wait:
+ submitargs = runtimeContext.copy()
+ submitargs.submit = False
+ runnerjob.run(submitargs)
return (runnerjob.uuid, "success")
self.poll_api = arvados.api('v1')
if runnerjob:
jobiter = iter((runnerjob,))
else:
- if "cwl_runner_job" in kwargs:
- self.uuid = kwargs.get("cwl_runner_job").get('uuid')
+ if runtimeContext.cwl_runner_job is not None:
+ self.uuid = runtimeContext.cwl_runner_job.get('uuid')
jobiter = tool.job(job_order,
self.output_callback,
- **kwargs)
+ runtimeContext)
try:
self.workflow_eval_lock.acquire()
if runnable:
with Perf(metrics, "run"):
- self.start_run(runnable, kwargs)
+ self.start_run(runnable, runtimeContext)
else:
if (self.task_queue.in_flight + len(self.processes)) > 0:
self.workflow_eval_lock.wait(3)
if self.final_output is None:
raise WorkflowException("Workflow did not return a result.")
- if kwargs.get("submit") and isinstance(runnerjob, Runner):
+ if runtimeContext.submit and isinstance(runnerjob, Runner):
logger.info("Final output collection %s", runnerjob.final_output)
else:
if self.output_name is None:
self.output_name = "Output of %s" % (shortname(tool.tool["id"]))
if self.output_tags is None:
self.output_tags = ""
- self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, self.output_tags, self.final_output)
+
+ storage_classes = runtimeContext.storage_classes.strip().split(",")
+ self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes, self.output_tags, self.final_output)
self.set_crunch_output()
- if kwargs.get("compute_checksum"):
+ if runtimeContext.compute_checksum:
adjustDirObjs(self.final_output, partial(get_listing, self.fs_access))
adjustFileObjs(self.final_output, partial(compute_checksums, self.fs_access))
parser.add_argument("--submit-runner-ram", type=int,
help="RAM (in MiB) required for the workflow runner job (default 1024)",
- default=1024)
+ default=None)
parser.add_argument("--submit-runner-image", type=str,
help="Docker image for workflow runner job, default arvados/jobs:%s" % __version__,
parser.add_argument("--enable-dev", action="store_true",
help="Enable loading and running development versions "
"of CWL spec.", default=False)
+ parser.add_argument('--storage-classes', default="default", type=str,
+ help="Specify comma separated list of storage classes to be used when saving workflow output to Keep.")
parser.add_argument("--intermediate-output-ttl", type=int, metavar="N",
help="If N > 0, intermediate output collections will be trashed N seconds after creation. Default is 0 (don't trash).",
job_order_object = None
arvargs = parser.parse_args(args)
+ if len(arvargs.storage_classes.strip().split(',')) > 1:
+ logger.error("Multiple storage classes are not supported currently.")
+ return 1
+
+ arvargs.use_container = True
+ arvargs.relax_path_checks = True
+ arvargs.print_supported_versions = False
+
if install_sig_handlers:
arv_cmd.install_signal_handlers()
if api_client is None:
api_client = arvados.safeapi.ThreadSafeApiCache(api_params={"model": OrderedJsonModel()}, keep_params={"num_retries": 4})
keep_client = api_client.keep
+ # Make an API object now so errors are reported early.
+ api_client.users().current().execute()
if keep_client is None:
keep_client = arvados.keep.KeepClient(api_client=api_client, num_retries=4)
- runner = ArvCwlRunner(api_client, work_api=arvargs.work_api, keep_client=keep_client,
- num_retries=4, output_name=arvargs.output_name,
- output_tags=arvargs.output_tags,
- thread_count=arvargs.thread_count)
+ runner = ArvCwlRunner(api_client, arvargs, keep_client=keep_client, num_retries=4)
except Exception as e:
logger.error(e)
return 1
else:
arvados.log_handler.setFormatter(logging.Formatter('%(name)s %(levelname)s: %(message)s'))
- arvargs.conformance_test = None
- arvargs.use_container = True
- arvargs.relax_path_checks = True
- arvargs.print_supported_versions = False
+ for key, val in cwltool.argparser.get_default_args().items():
+ if not hasattr(arvargs, key):
+ setattr(arvargs, key, val)
- make_fs_access = partial(CollectionFsAccess,
- collection_cache=runner.collection_cache)
+ runtimeContext = ArvRuntimeContext(vars(arvargs))
+ runtimeContext.make_fs_access = partial(CollectionFsAccess,
+ collection_cache=runner.collection_cache)
return cwltool.main.main(args=arvargs,
stdout=stdout,
stderr=stderr,
executor=runner.arv_executor,
- makeTool=runner.arv_make_tool,
versionfunc=versionstring,
job_order_object=job_order_object,
- make_fs_access=make_fs_access,
- fetcher_constructor=partial(CollectionFetcher,
- api_client=api_client,
- fs_access=make_fs_access(""),
- num_retries=runner.num_retries),
- resolver=partial(collectionResolver, api_client, num_retries=runner.num_retries),
logger_handler=arvados.log_handler,
- custom_schema_callback=add_arv_hints)
+ custom_schema_callback=add_arv_hints,
+ loadingContext=runner.loadingContext,
+ runtimeContext=runtimeContext)
"_type": "@id"
refScope: 0
+- name: cwltool:TimeLimit
+ type: record
+ inVocab: false
+ extends: cwl:ProcessRequirement
+ doc: |
+ Set an upper limit on the execution time of a CommandLineTool or
+ ExpressionTool. A tool execution which exceeds the time limit may
+ be preemptively terminated and considered failed. May also be
+ used by batch systems to make scheduling decisions.
+ fields:
+ - name: class
+ type: string
+ doc: "Always 'TimeLimit'"
+ jsonldPredicate:
+ "_id": "@type"
+ "_type": "@vocab"
+ - name: timelimit
+ type: [long, string]
+ doc: |
+ The time limit, in seconds. A time limit of zero means no
+ time limit. Negative time limits are an error.
+
- name: RunInSingleContainer
type: record
extends: cwl:ProcessRequirement
_type: "@vocab"
- name: enableReuse
type: boolean
+
+- name: WorkflowRunnerResources
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify memory or cores resource request for the CWL runner process itself.
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:WorkflowRunnerResources'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ ramMin:
+ type: int?
+ doc: Minimum RAM, in mebibytes (2**20)
+ jsonldPredicate: "https://w3id.org/cwl/cwl#ResourceRequirement/ramMin"
+ coresMin:
+ type: int?
+ doc: Minimum cores allocated to cwl-runner
+ jsonldPredicate: "https://w3id.org/cwl/cwl#ResourceRequirement/coresMin"
\ No newline at end of file
import ciso8601
import uuid
+from arvados_cwl.util import get_current_container, get_intermediate_collection_info
import ruamel.yaml as yaml
from cwltool.errors import WorkflowException
-from cwltool.process import get_feature, UnsupportedRequirement, shortname
+from cwltool.process import UnsupportedRequirement, shortname
from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
from cwltool.utils import aslist
+from cwltool.job import JobBase
import arvados.collection
logger = logging.getLogger('arvados.cwl-runner')
metrics = logging.getLogger('arvados.cwl-runner.metrics')
-class ArvadosContainer(object):
+class ArvadosContainer(JobBase):
"""Submit and manage a Crunch container request for executing a CWL CommandLineTool."""
- def __init__(self, runner):
+ def __init__(self, runner,
+ builder, # type: Builder
+ joborder, # type: Dict[Text, Union[Dict[Text, Any], List, Text]]
+ make_path_mapper, # type: Callable[..., PathMapper]
+ requirements, # type: List[Dict[Text, Text]]
+ hints, # type: List[Dict[Text, Text]]
+ name # type: Text
+ ):
+ super(ArvadosContainer, self).__init__(builder, joborder, make_path_mapper, requirements, hints, name)
self.arvrunner = runner
self.running = False
self.uuid = None
def update_pipeline_component(self, r):
pass
- def run(self, dry_run=False, pull_image=True, **kwargs):
+ def run(self, runtimeContext):
# ArvadosCommandTool subclasses from cwltool.CommandLineTool,
# which calls makeJobRunner() to get a new ArvadosContainer
# object. The fields that define execution such as
"name": self.name,
"output_path": self.outdir,
"cwd": self.outdir,
- "priority": kwargs.get("priority"),
+ "priority": runtimeContext.priority,
"state": "Committed",
"properties": {},
}
keepemptydirs(vwd)
- with Perf(metrics, "generatefiles.save_new %s" % self.name):
- vwd.save_new()
+ if not runtimeContext.current_container:
+ runtimeContext.current_container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
+ info = get_intermediate_collection_info(self.name, runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
+ vwd.save_new(name=info["name"],
+ owner_uuid=self.arvrunner.project_uuid,
+ ensure_unique_name=True,
+ trash_at=info["trash_at"],
+ properties=info["properties"])
prev = None
for f, p in sorteditems:
mounts["stdout"] = {"kind": "file",
"path": "%s/%s" % (self.outdir, self.stdout)}
- (docker_req, docker_is_req) = get_feature(self, "DockerRequirement")
+ (docker_req, docker_is_req) = self.get_requirement("DockerRequirement")
if not docker_req:
docker_req = {"dockerImageId": "arvados/jobs"}
container_request["container_image"] = arv_docker_get_image(self.arvrunner.api,
docker_req,
- pull_image,
+ runtimeContext.pull_image,
self.arvrunner.project_uuid)
- api_req, _ = get_feature(self, "http://arvados.org/cwl#APIRequirement")
+ api_req, _ = self.get_requirement("http://arvados.org/cwl#APIRequirement")
if api_req:
runtime_constraints["API"] = True
- runtime_req, _ = get_feature(self, "http://arvados.org/cwl#RuntimeConstraints")
+ runtime_req, _ = self.get_requirement("http://arvados.org/cwl#RuntimeConstraints")
if runtime_req:
if "keep_cache" in runtime_req:
runtime_constraints["keep_cache_ram"] = runtime_req["keep_cache"] * 2**20
"writable": True
}
- partition_req, _ = get_feature(self, "http://arvados.org/cwl#PartitionRequirement")
+ partition_req, _ = self.get_requirement("http://arvados.org/cwl#PartitionRequirement")
if partition_req:
scheduling_parameters["partitions"] = aslist(partition_req["partition"])
- intermediate_output_req, _ = get_feature(self, "http://arvados.org/cwl#IntermediateOutput")
+ intermediate_output_req, _ = self.get_requirement("http://arvados.org/cwl#IntermediateOutput")
if intermediate_output_req:
self.output_ttl = intermediate_output_req["outputTTL"]
else:
if self.output_ttl < 0:
raise WorkflowException("Invalid value %d for output_ttl, cannot be less than zero" % container_request["output_ttl"])
+ if self.timelimit is not None:
+ scheduling_parameters["max_run_time"] = self.timelimit
+
+ container_request["output_name"] = "Output for step %s" % (self.name)
container_request["output_ttl"] = self.output_ttl
container_request["mounts"] = mounts
container_request["secret_mounts"] = secret_mounts
container_request["runtime_constraints"] = runtime_constraints
container_request["scheduling_parameters"] = scheduling_parameters
- enable_reuse = kwargs.get("enable_reuse", True)
+ enable_reuse = runtimeContext.enable_reuse
if enable_reuse:
- reuse_req, _ = get_feature(self, "http://arvados.org/cwl#ReuseRequirement")
+ reuse_req, _ = self.get_requirement("http://arvados.org/cwl#ReuseRequirement")
if reuse_req:
enable_reuse = reuse_req["enableReuse"]
container_request["use_existing"] = enable_reuse
- if kwargs.get("runnerjob", "").startswith("arvwf:"):
- wfuuid = kwargs["runnerjob"][6:kwargs["runnerjob"].index("#")]
+ if runtimeContext.runnerjob.startswith("arvwf:"):
+ wfuuid = runtimeContext.runnerjob[6:runtimeContext.runnerjob.index("#")]
wfrecord = self.arvrunner.api.workflows().get(uuid=wfuuid).execute(num_retries=self.arvrunner.num_retries)
if container_request["name"] == "main":
container_request["name"] = wfrecord["name"]
self.output_callback = self.arvrunner.get_wrapped_callback(self.output_callback)
try:
- if kwargs.get("submit_request_uuid"):
+ if runtimeContext.submit_request_uuid:
response = self.arvrunner.api.container_requests().update(
- uuid=kwargs["submit_request_uuid"],
+ uuid=runtimeContext.submit_request_uuid,
body=container_request
).execute(num_retries=self.arvrunner.num_retries)
else:
api_client=self.arvrunner.api,
keep_client=self.arvrunner.keep_client,
num_retries=self.arvrunner.num_retries)
- done.logtail(logc, logger, "%s error log:" % self.arvrunner.label(self))
+ done.logtail(logc, logger.error, "%s (%s) error log:" % (self.arvrunner.label(self), record["uuid"]), maxlen=40)
if record["output_uuid"]:
if self.arvrunner.trash_intermediate or self.arvrunner.intermediate_output_ttl:
class RunnerContainer(Runner):
"""Submit and manage a container that runs arvados-cwl-runner."""
- def arvados_job_spec(self, dry_run=False, pull_image=True, **kwargs):
+ def arvados_job_spec(self, runtimeContext):
"""Create an Arvados container request for this workflow.
The returned dict can be used to create a container passed as
},
"secret_mounts": secret_mounts,
"runtime_constraints": {
- "vcpus": 1,
+ "vcpus": self.submit_runner_cores,
"ram": 1024*1024 * self.submit_runner_ram,
"API": True
},
if self.output_tags:
command.append("--output-tags=" + self.output_tags)
- if kwargs.get("debug"):
+ if runtimeContext.debug:
command.append("--debug")
+ if runtimeContext.storage_classes != "default":
+ command.append("--storage-classes=" + runtimeContext.storage_classes)
+
if self.on_error:
command.append("--on-error=" + self.on_error)
return container_req
- def run(self, **kwargs):
- kwargs["keepprefix"] = "keep:"
- job_spec = self.arvados_job_spec(**kwargs)
+ def run(self, runtimeContext):
+ runtimeContext.keepprefix = "keep:"
+ job_spec = self.arvados_job_spec(runtimeContext)
if self.arvrunner.project_uuid:
job_spec["owner_uuid"] = self.arvrunner.project_uuid
- if kwargs.get("submit_request_uuid"):
+ if runtimeContext.submit_request_uuid:
response = self.arvrunner.api.container_requests().update(
- uuid=kwargs["submit_request_uuid"],
+ uuid=runtimeContext.submit_request_uuid,
body=job_spec
).execute(num_retries=self.arvrunner.num_retries)
else:
import json
import time
-from cwltool.process import get_feature, shortname, UnsupportedRequirement
+from cwltool.process import shortname, UnsupportedRequirement
from cwltool.errors import WorkflowException
from cwltool.command_line_tool import revmap_file, CommandLineTool
from cwltool.load_tool import fetch_document
from cwltool.builder import Builder
from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
+from cwltool.job import JobBase
from schema_salad.sourceline import SourceLine
+from arvados_cwl.util import get_current_container, get_intermediate_collection_info
import ruamel.yaml as yaml
import arvados.collection
crunchrunner_git_commit = 'a3f2cb186e437bfce0031b024b2157b73ed2717d'
-class ArvadosJob(object):
+class ArvadosJob(JobBase):
"""Submit and manage a Crunch job for executing a CWL CommandLineTool."""
- def __init__(self, runner):
+ def __init__(self, runner,
+ builder, # type: Builder
+ joborder, # type: Dict[Text, Union[Dict[Text, Any], List, Text]]
+ make_path_mapper, # type: Callable[..., PathMapper]
+ requirements, # type: List[Dict[Text, Text]]
+ hints, # type: List[Dict[Text, Text]]
+ name # type: Text
+ ):
+ super(ArvadosJob, self).__init__(builder, joborder, make_path_mapper, requirements, hints, name)
self.arvrunner = runner
self.running = False
self.uuid = None
- def run(self, dry_run=False, pull_image=True, **kwargs):
+ def run(self, runtimeContext):
script_parameters = {
"command": self.command_line
}
if vwd:
with Perf(metrics, "generatefiles.save_new %s" % self.name):
- vwd.save_new()
+ if not runtimeContext.current_container:
+ runtimeContext.current_container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
+ info = get_intermediate_collection_info(self.name, runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
+ vwd.save_new(name=info["name"],
+ owner_uuid=self.arvrunner.project_uuid,
+ ensure_unique_name=True,
+ trash_at=info["trash_at"],
+ properties=info["properties"])
for f, p in generatemapper.items():
if p.type == "File":
script_parameters["task.permanentFailCodes"] = self.permanentFailCodes
with Perf(metrics, "arv_docker_get_image %s" % self.name):
- (docker_req, docker_is_req) = get_feature(self, "DockerRequirement")
- if docker_req and kwargs.get("use_container") is not False:
+ (docker_req, docker_is_req) = self.get_requirement("DockerRequirement")
+ if docker_req and runtimeContext.use_container is not False:
if docker_req.get("dockerOutputDirectory"):
raise SourceLine(docker_req, "dockerOutputDirectory", UnsupportedRequirement).makeError(
"Option 'dockerOutputDirectory' of DockerRequirement not supported.")
- runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api, docker_req, pull_image, self.arvrunner.project_uuid)
+ runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api,
+ docker_req,
+ runtimeContext.pull_image,
+ self.arvrunner.project_uuid)
else:
runtime_constraints["docker_image"] = "arvados/jobs"
runtime_constraints["min_ram_mb_per_node"] = resources.get("ram")
runtime_constraints["min_scratch_mb_per_node"] = resources.get("tmpdirSize", 0) + resources.get("outdirSize", 0)
- runtime_req, _ = get_feature(self, "http://arvados.org/cwl#RuntimeConstraints")
+ runtime_req, _ = self.get_requirement("http://arvados.org/cwl#RuntimeConstraints")
if runtime_req:
if "keep_cache" in runtime_req:
runtime_constraints["keep_cache_mb_per_task"] = runtime_req["keep_cache"]
if not self.arvrunner.ignore_docker_for_reuse:
filters.append(["docker_image_locator", "in docker", runtime_constraints["docker_image"]])
- enable_reuse = kwargs.get("enable_reuse", True)
+ enable_reuse = runtimeContext.enable_reuse
if enable_reuse:
- reuse_req, _ = get_feature(self, "http://arvados.org/cwl#ReuseRequirement")
+ reuse_req, _ = self.get_requirement("http://arvados.org/cwl#ReuseRequirement")
if reuse_req:
enable_reuse = reuse_req["enableReuse"]
dirs[g.group(1)] = g.group(2)
if processStatus == "permanentFail":
- done.logtail(logc, logger, "%s error log:" % self.arvrunner.label(self))
+ done.logtail(logc, logger.error, "%s (%s) error log:" % (self.arvrunner.label(self), record["uuid"]), maxlen=40)
with Perf(metrics, "output collection %s" % self.name):
outputs = done.done(self, record, dirs["tmpdir"],
class RunnerJob(Runner):
"""Submit and manage a Crunch job that runs crunch_scripts/cwl-runner."""
- def arvados_job_spec(self, dry_run=False, pull_image=True, **kwargs):
+ def arvados_job_spec(self, debug=False):
"""Create an Arvados job specification for this workflow.
The returned dict can be used to create a job (i.e., passed as
if self.on_error:
self.job_order["arv:on_error"] = self.on_error
- if kwargs.get("debug"):
+ if debug:
self.job_order["arv:debug"] = True
return {
}
}
- def run(self, **kwargs):
- job_spec = self.arvados_job_spec(**kwargs)
+ def run(self, runtimeContext):
+ job_spec = self.arvados_job_spec(runtimeContext.debug)
job_spec.setdefault("owner_uuid", self.arvrunner.project_uuid)
body=instance_spec).execute(num_retries=self.arvrunner.num_retries)
logger.info("Created pipeline %s", self.arvrunner.pipeline["uuid"])
- if kwargs.get("wait") is False:
+ if runtimeContext.wait is False:
self.uuid = self.arvrunner.pipeline["uuid"]
return
from .arvjob import ArvadosJob
from .arvcontainer import ArvadosContainer
from .pathmapper import ArvPathMapper
+from functools import partial
class ArvadosCommandTool(CommandLineTool):
"""Wrap cwltool CommandLineTool to override selected methods."""
- def __init__(self, arvrunner, toolpath_object, **kwargs):
- super(ArvadosCommandTool, self).__init__(toolpath_object, **kwargs)
+ def __init__(self, arvrunner, toolpath_object, loadingContext):
+ super(ArvadosCommandTool, self).__init__(toolpath_object, loadingContext)
self.arvrunner = arvrunner
- self.work_api = kwargs["work_api"]
-
- def makeJobRunner(self, **kwargs):
- if self.work_api == "containers":
- return ArvadosContainer(self.arvrunner)
- elif self.work_api == "jobs":
- return ArvadosJob(self.arvrunner)
-
- def makePathMapper(self, reffiles, stagedir, **kwargs):
- # type: (List[Any], unicode, **Any) -> PathMapper
- if self.work_api == "containers":
- return ArvPathMapper(self.arvrunner, reffiles+kwargs.get("extra_reffiles", []), kwargs["basedir"],
+
+ def make_job_runner(self, runtimeContext):
+ if runtimeContext.work_api == "containers":
+ return partial(ArvadosContainer, self.arvrunner)
+ elif runtimeContext.work_api == "jobs":
+ return partial(ArvadosJob, self.arvrunner)
+ else:
+ raise Exception("Unsupported work_api %s", runtimeContext.work_api)
+
+ def make_path_mapper(self, reffiles, stagedir, runtimeContext, separateDirs):
+ if runtimeContext.work_api == "containers":
+ return ArvPathMapper(self.arvrunner, reffiles+runtimeContext.extra_reffiles, runtimeContext.basedir,
"/keep/%s",
- "/keep/%s/%s",
- **kwargs)
- elif self.work_api == "jobs":
- return ArvPathMapper(self.arvrunner, reffiles, kwargs["basedir"],
+ "/keep/%s/%s")
+ elif runtimeContext.work_api == "jobs":
+ return ArvPathMapper(self.arvrunner, reffiles, runtimeContext.basedir,
"$(task.keep)/%s",
- "$(task.keep)/%s/%s",
- **kwargs)
+ "$(task.keep)/%s/%s")
- def job(self, joborder, output_callback, **kwargs):
+ def job(self, joborder, output_callback, runtimeContext):
# Workaround for #13365
- builderargs = kwargs.copy()
- builderargs["toplevel"] = True
- builderargs["tmp_outdir_prefix"] = ""
- builder = self._init_job(joborder, **builderargs)
+ builderargs = runtimeContext.copy()
+ builderargs.toplevel = True
+ builderargs.tmp_outdir_prefix = ""
+ builder = self._init_job(joborder, builderargs)
joborder = builder.job
- if self.work_api == "containers":
+ runtimeContext = runtimeContext.copy()
+
+ if runtimeContext.work_api == "containers":
dockerReq, is_req = self.get_requirement("DockerRequirement")
if dockerReq and dockerReq.get("dockerOutputDirectory"):
- kwargs["outdir"] = dockerReq.get("dockerOutputDirectory")
- kwargs["docker_outdir"] = dockerReq.get("dockerOutputDirectory")
+ runtimeContext.outdir = dockerReq.get("dockerOutputDirectory")
+ runtimeContext.docker_outdir = dockerReq.get("dockerOutputDirectory")
else:
- kwargs["outdir"] = "/var/spool/cwl"
- kwargs["docker_outdir"] = "/var/spool/cwl"
- elif self.work_api == "jobs":
- kwargs["outdir"] = "$(task.outdir)"
- kwargs["docker_outdir"] = "$(task.outdir)"
- kwargs["tmpdir"] = "$(task.tmpdir)"
- kwargs["docker_tmpdir"] = "$(task.tmpdir)"
- return super(ArvadosCommandTool, self).job(joborder, output_callback, **kwargs)
+ runtimeContext.outdir = "/var/spool/cwl"
+ runtimeContext.docker_outdir = "/var/spool/cwl"
+ elif runtimeContext.work_api == "jobs":
+ runtimeContext.outdir = "$(task.outdir)"
+ runtimeContext.docker_outdir = "$(task.outdir)"
+ runtimeContext.tmpdir = "$(task.tmpdir)"
+ runtimeContext.docker_tmpdir = "$(task.tmpdir)"
+ return super(ArvadosCommandTool, self).job(joborder, output_callback, runtimeContext)
from cwltool.workflow import Workflow, WorkflowException
from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
from cwltool.builder import Builder
+from cwltool.context import LoadingContext
import ruamel.yaml as yaml
upload_dependencies(arvRunner, name, tool.doc_loader,
packed, tool.tool["id"], False)
- # TODO nowhere for submit_runner_ram to go.
+ if submit_runner_ram:
+ hints = main.get("hints", [])
+ found = False
+ for h in hints:
+ if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources":
+ h["ramMin"] = submit_runner_ram
+ found = True
+ break
+ if not found:
+ hints.append({"class": "http://arvados.org/cwl#WorkflowRunnerResources",
+ "ramMin": submit_runner_ram})
+ main["hints"] = hints
body = {
"workflow": {
"name": name,
"description": tool.tool.get("doc", ""),
- "definition":yaml.round_trip_dump(packed)
+ "definition":json.dumps(packed, sort_keys=True, indent=4, separators=(',',': '))
}}
if project_uuid:
body["workflow"]["owner_uuid"] = project_uuid
class ArvadosWorkflow(Workflow):
"""Wrap cwltool Workflow to override selected methods."""
- def __init__(self, arvrunner, toolpath_object, **kwargs):
- super(ArvadosWorkflow, self).__init__(toolpath_object, **kwargs)
+ def __init__(self, arvrunner, toolpath_object, loadingContext):
+ super(ArvadosWorkflow, self).__init__(toolpath_object, loadingContext)
self.arvrunner = arvrunner
- self.work_api = kwargs["work_api"]
self.wf_pdh = None
self.dynamic_resource_req = []
self.static_resource_req = []
self.wf_reffiles = []
+ self.loadingContext = loadingContext
- def job(self, joborder, output_callback, **kwargs):
- kwargs["work_api"] = self.work_api
+ def job(self, joborder, output_callback, runtimeContext):
req, _ = self.get_requirement("http://arvados.org/cwl#RunInSingleContainer")
if req:
with SourceLine(self.tool, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
packed = pack(document_loader, workflowobj, uri, self.metadata)
- builder = Builder()
- builder.job = joborder
- builder.requirements = workflowobj["requirements"]
- builder.hints = workflowobj["hints"]
- builder.resources = {}
+ builder = Builder(joborder,
+ requirements=workflowobj["requirements"],
+ hints=workflowobj["hints"],
+ resources={})
def visit(item):
for t in ("hints", "requirements"):
self.static_resource_req = [get_overall_res_req(self.static_resource_req)]
upload_dependencies(self.arvrunner,
- kwargs.get("name", ""),
+ runtimeContext.name,
document_loader,
packed,
uri,
if self.dynamic_resource_req:
- builder = Builder()
- builder.job = joborder
- builder.requirements = self.requirements
- builder.hints = self.hints
- builder.resources = {}
+ builder = Builder(joborder,
+ requirements=self.requirements,
+ hints=self.hints,
+ resources={})
# Evaluate dynamic resource requirements using current builder
rs = copy.copy(self.static_resource_req)
reffiles = []
visit_class(joborder_keepmount, ("File", "Directory"), reffiles.append)
- mapper = ArvPathMapper(self.arvrunner, reffiles+self.wf_reffiles, kwargs["basedir"],
- "/keep/%s",
- "/keep/%s/%s",
- **kwargs)
+ mapper = ArvPathMapper(self.arvrunner, reffiles+self.wf_reffiles, runtimeContext.basedir,
+ "/keep/%s",
+ "/keep/%s/%s")
# For containers API, we need to make sure any extra
# referenced files (ie referenced by the workflow but
# not in the inputs) are included in the mounts.
- kwargs["extra_reffiles"] = copy.deepcopy(self.wf_reffiles)
+ if self.wf_reffiles:
+ runtimeContext = runtimeContext.copy()
+ runtimeContext.extra_reffiles = copy.deepcopy(self.wf_reffiles)
def keepmount(obj):
remove_redundant_fields(obj)
"outputs": self.tool["outputs"],
"stdout": "cwl.output.json",
"requirements": self.requirements+job_res_reqs+[
+ {"class": "InlineJavascriptRequirement"},
{
"class": "InitialWorkDirRequirement",
"listing": [{
"entryname": "workflow.cwl",
- "entry": {
- "class": "File",
- "location": "keep:%s/workflow.cwl" % self.wf_pdh
- }
+ "entry": '$({"class": "File", "location": "keep:%s/workflow.cwl"})' % self.wf_pdh
}, {
"entryname": "cwl.input.yml",
"entry": json.dumps(joborder_keepmount, indent=2, sort_keys=True, separators=(',',': ')).replace("\\", "\\\\").replace('$(', '\$(').replace('${', '\${')
"arguments": ["--no-container", "--move-outputs", "--preserve-entire-environment", "workflow.cwl#main", "cwl.input.yml"],
"id": "#"
})
- kwargs["loader"] = self.doc_loader
- kwargs["avsc_names"] = self.doc_schema
- kwargs["metadata"] = self.metadata
- return ArvadosCommandTool(self.arvrunner, wf_runner, **kwargs).job(joborder_resolved, output_callback, **kwargs)
+ return ArvadosCommandTool(self.arvrunner, wf_runner, self.loadingContext).job(joborder_resolved, output_callback, runtimeContext)
else:
- return super(ArvadosWorkflow, self).job(joborder, output_callback, **kwargs)
+ return super(ArvadosWorkflow, self).job(joborder, output_callback, runtimeContext)
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from cwltool.context import LoadingContext, RuntimeContext
+
+class ArvLoadingContext(LoadingContext):
+ def __init__(self, kwargs=None):
+ super(ArvLoadingContext, self).__init__(kwargs)
+
+class ArvRuntimeContext(RuntimeContext):
+ def __init__(self, kwargs=None):
+ self.work_api = None
+ self.extra_reffiles = []
+ self.priority = 500
+ self.enable_reuse = True
+ self.runnerjob = ""
+ self.submit_request_uuid = None
+ self.project_uuid = None
+ self.trash_intermediate = False
+ self.intermediate_output_ttl = 0
+ self.update_workflow = ""
+ self.create_workflow = False
+ self.submit_runner_ram = 0
+ self.ignore_docker_for_reuse = False
+ self.submit = True
+ self.submit_runner_image = None
+ self.wait = True
+ self.cwl_runner_job = None
+ self.storage_classes = "default"
+ self.current_container = None
+
+ super(ArvRuntimeContext, self).__init__(kwargs)
from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, normalizeFilesDirs
from cwltool.load_tool import load_tool
from cwltool.errors import WorkflowException
+from arvados_cwl.context import ArvRuntimeContext
from .fsaccess import CollectionFetcher, CollectionFsAccess
debug = job_order_object["arv:debug"]
del job_order_object["arv:debug"]
+ arvargs = argparse.Namespace()
+ arvargs.work_api = "jobs"
+ arvargs.output_name = output_name
+ arvargs.output_tags = output_tags
+ arvargs.thread_count = 1
+
runner = arvados_cwl.ArvCwlRunner(api_client=arvados.safeapi.ThreadSafeApiCache(
api_params={"model": OrderedJsonModel()}, keep_params={"num_retries": 4}),
- output_name=output_name, output_tags=output_tags)
+ arvargs=arvargs)
make_fs_access = functools.partial(CollectionFsAccess,
collection_cache=runner.collection_cache)
- t = load_tool(toolpath, runner.arv_make_tool,
- fetcher_constructor=functools.partial(CollectionFetcher,
- api_client=runner.api,
- fs_access=make_fs_access(""),
- num_retries=runner.num_retries))
+ t = load_tool(toolpath, runner.loadingContext)
if debug:
logger.setLevel(logging.DEBUG)
logging.getLogger('arvados').setLevel(logging.DEBUG)
logging.getLogger("cwltool").setLevel(logging.DEBUG)
- args = argparse.Namespace()
+ args = ArvRuntimeContext(vars(arvargs))
args.project_uuid = arvados.current_job()["owner_uuid"]
args.enable_reuse = enable_reuse
args.on_error = on_error
args.disable_js_validation = False
args.tmp_outdir_prefix = "tmp"
- runner.arv_executor(t, job_order_object, **vars(args))
+ runner.arv_executor(t, job_order_object, args, logger=logger)
except Exception as e:
if isinstance(e, WorkflowException):
logging.info("Workflow error %s", e)
crunchstat_re = re.compile(r"^\d{4}-\d\d-\d\d_\d\d:\d\d:\d\d [a-z0-9]{5}-8i9sb-[a-z0-9]{15} \d+ \d+ stderr crunchstat:")
timestamp_re = re.compile(r"^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d+Z) (.*)")
-def logtail(logcollection, logger, header, maxlen=25):
+def logtail(logcollection, logfunc, header, maxlen=25):
if len(logcollection) == 0:
- logger.info(header)
- logger.info(" ** log is empty **")
+ logfunc(header)
+ logfunc(" ** log is empty **")
return
containersapi = ("crunch-run.txt" in logcollection)
loglines = mergelogs.values()[0]
logtxt = "\n ".join(l.strip() for l in loglines)
- logger.info(header)
- logger.info("\n %s", logtxt)
+ logfunc(header)
+ logfunc("\n %s", logtxt)
else:
return super(CollectionFsAccess, self).exists(fn)
+ def size(self, fn): # type: (unicode) -> bool
+ collection, rest = self.get_collection(fn)
+ if collection is not None:
+ if rest:
+ arvfile = collection.find(rest)
+ if isinstance(arvfile, arvados.arvfile.ArvadosFile):
+ return arvfile.size()
+ raise IOError(errno.EINVAL, "Not a path to a file %s" % (fn))
+ else:
+ return super(CollectionFsAccess, self).size(fn)
+
def isfile(self, fn): # type: (unicode) -> bool
collection, rest = self.get_collection(fn)
if collection is not None:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
import requests
import email.utils
import time
import arvados.collection
import urlparse
import logging
+import calendar
logger = logging.getLogger('arvados.cwl-runner')
def my_formatdate(dt):
- return email.utils.formatdate(timeval=time.mktime(dt.timetuple()),
+ return email.utils.formatdate(timeval=calendar.timegm(dt.timetuple()),
localtime=False, usegmt=True)
def my_parsedate(text):
- parsed = email.utils.parsedate(text)
+ parsed = email.utils.parsedate_tz(text)
if parsed:
- return datetime.datetime(*parsed[:6])
+ if parsed[9]:
+ # Adjust to UTC
+ return datetime.datetime(*parsed[:6]) + datetime.timedelta(seconds=parsed[9])
+ else:
+ # TZ is zero or missing, assume UTC.
+ return datetime.datetime(*parsed[:6])
else:
return datetime.datetime(1970, 1, 1)
import os
import urllib
+from arvados_cwl.util import get_current_container, get_intermediate_collection_info
import arvados.commands.run
import arvados.collection
from schema_salad.sourceline import SourceLine
+from arvados.errors import ApiError
from cwltool.pathmapper import PathMapper, MapperEnt, abspath, adjustFileObjs, adjustDirObjs
from cwltool.workflow import WorkflowException
pdh_dirpath = re.compile(r'^keep:[0-9a-f]{32}\+\d+(/.*)?$')
def __init__(self, arvrunner, referenced_files, input_basedir,
- collection_pattern, file_pattern, name=None, single_collection=False, **kwargs):
+ collection_pattern, file_pattern, name=None, single_collection=False):
self.arvrunner = arvrunner
self.input_basedir = input_basedir
self.collection_pattern = collection_pattern
keep_client=self.arvrunner.keep_client,
num_retries=self.arvrunner.num_retries)
- already_uploaded = self.arvrunner.get_uploaded()
- copied_files = set()
- for k in referenced_files:
- loc = k["location"]
- if loc in already_uploaded:
- v = already_uploaded[loc]
- self._pathmap[loc] = MapperEnt(v.resolved, self.collection_pattern % urllib.unquote(v.resolved[5:]), v.type, True)
- if self.single_collection:
- basename = k["basename"]
- if basename not in collection:
- self.addentry({"location": loc, "class": v.type, "basename": basename}, collection, ".", [])
- copied_files.add((loc, basename, v.type))
-
for srcobj in referenced_files:
self.visit(srcobj, uploadfiles)
fnPattern="keep:%s/%s",
name=self.name,
project=self.arvrunner.project_uuid,
- collection=collection)
+ collection=collection,
+ packed=False)
for src, ab, st in uploadfiles:
self._pathmap[src] = MapperEnt(urllib.quote(st.fn, "/:+@"), self.collection_pattern % st.fn[5:],
"Directory" if os.path.isdir(ab) else "File", True)
- self.arvrunner.add_uploaded(src, self._pathmap[src])
-
- for loc, basename, cls in copied_files:
- fn = "keep:%s/%s" % (collection.portable_data_hash(), basename)
- self._pathmap[loc] = MapperEnt(urllib.quote(fn, "/:+@"), self.collection_pattern % fn[5:], cls, True)
for srcobj in referenced_files:
remap = []
for l in srcobj.get("listing", []):
self.addentry(l, c, ".", remap)
- check = self.arvrunner.api.collections().list(filters=[["portable_data_hash", "=", c.portable_data_hash()]], limit=1).execute(num_retries=self.arvrunner.num_retries)
- if not check["items"]:
- c.save_new(owner_uuid=self.arvrunner.project_uuid)
+ container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
+ info = get_intermediate_collection_info(None, container, self.arvrunner.intermediate_output_ttl)
+
+ c.save_new(name=info["name"],
+ owner_uuid=self.arvrunner.project_uuid,
+ ensure_unique_name=True,
+ trash_at=info["trash_at"],
+ properties=info["properties"])
ab = self.collection_pattern % c.portable_data_hash()
self._pathmap[srcobj["location"]] = MapperEnt("keep:"+c.portable_data_hash(), ab, "Directory", True)
num_retries=self.arvrunner.num_retries )
self.addentry(srcobj, c, ".", remap)
- check = self.arvrunner.api.collections().list(filters=[["portable_data_hash", "=", c.portable_data_hash()]], limit=1).execute(num_retries=self.arvrunner.num_retries)
- if not check["items"]:
- c.save_new(owner_uuid=self.arvrunner.project_uuid)
+ container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
+ info = get_intermediate_collection_info(None, container, self.arvrunner.intermediate_output_ttl)
+
+ c.save_new(name=info["name"],
+ owner_uuid=self.arvrunner.project_uuid,
+ ensure_unique_name=True,
+ trash_at=info["trash_at"],
+ properties=info["properties"])
ab = self.file_pattern % (c.portable_data_hash(), srcobj["basename"])
self._pathmap[srcobj["location"]] = MapperEnt("keep:%s/%s" % (c.portable_data_hash(), srcobj["basename"]),
else:
return None
+
class StagingPathMapper(PathMapper):
_follow_dirs = True
from functools import partial
import logging
import json
-import subprocess
+import subprocess32 as subprocess
from collections import namedtuple
from StringIO import StringIO
from cwltool.command_line_tool import CommandLineTool
import cwltool.workflow
-from cwltool.process import get_feature, scandeps, UnsupportedRequirement, normalizeFilesDirs, shortname
+from cwltool.process import scandeps, UnsupportedRequirement, normalizeFilesDirs, shortname
from cwltool.load_tool import fetch_document
from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
from cwltool.utils import aslist
# that external references in $include and $mixin are captured.
scanobj = loadref("", workflowobj["id"])
- sc = scandeps(uri, scanobj,
+ sc_result = scandeps(uri, scanobj,
loadref_fields,
set(("$include", "$schemas", "location")),
loadref, urljoin=document_loader.fetcher.urljoin)
+ sc = []
+ def only_real(obj):
+ if obj.get("location", "").startswith("file:"):
+ sc.append(obj)
+
+ visit_class(sc_result, ("File", "Directory"), only_real)
+
normalizeFilesDirs(sc)
if include_primary and "id" in workflowobj:
"""Uploads Docker images used in CommandLineTool objects."""
if isinstance(tool, CommandLineTool):
- (docker_req, docker_is_req) = get_feature(tool, "DockerRequirement")
+ (docker_req, docker_is_req) = tool.get_requirement("DockerRequirement")
if docker_req:
if docker_req.get("dockerOutputDirectory") and arvrunner.work_api != "containers":
# TODO: can be supported by containers API, but not jobs API.
def __init__(self, runner, tool, job_order, enable_reuse,
output_name, output_tags, submit_runner_ram=0,
name=None, on_error=None, submit_runner_image=None,
- intermediate_output_ttl=0, merged_map=None, priority=None,
- secret_store=None):
+ intermediate_output_ttl=0, merged_map=None,
+ priority=None, secret_store=None):
self.arvrunner = runner
self.tool = tool
self.job_order = job_order
if enable_reuse:
# If reuse is permitted by command line arguments but
# disabled by the workflow itself, disable it.
- reuse_req, _ = get_feature(self.tool, "http://arvados.org/cwl#ReuseRequirement")
+ reuse_req, _ = self.tool.get_requirement("http://arvados.org/cwl#ReuseRequirement")
if reuse_req:
enable_reuse = reuse_req["enableReuse"]
self.enable_reuse = enable_reuse
self.priority = priority
self.secret_store = secret_store
+ self.submit_runner_cores = 1
+ self.submit_runner_ram = 1024 # defaut 1 GiB
+
+ runner_resource_req, _ = self.tool.get_requirement("http://arvados.org/cwl#WorkflowRunnerResources")
+ if runner_resource_req:
+ if runner_resource_req.get("coresMin"):
+ self.submit_runner_cores = runner_resource_req["coresMin"]
+ if runner_resource_req.get("ramMin"):
+ self.submit_runner_ram = runner_resource_req["ramMin"]
+
if submit_runner_ram:
+ # Command line / initializer overrides default and/or spec from workflow
self.submit_runner_ram = submit_runner_ram
- else:
- self.submit_runner_ram = 3000
if self.submit_runner_ram <= 0:
- raise Exception("Value of --submit-runner-ram must be greater than zero")
+ raise Exception("Value of submit-runner-ram must be greater than zero")
+
+ if self.submit_runner_cores <= 0:
+ raise Exception("Value of submit-runner-cores must be greater than zero")
self.merged_map = merged_map or {}
api_client=self.arvrunner.api,
keep_client=self.arvrunner.keep_client,
num_retries=self.arvrunner.num_retries)
- done.logtail(logc, logger, "%s error log:" % self.arvrunner.label(self), maxlen=40)
+ done.logtail(logc, logger.error, "%s (%s) error log:" % (self.arvrunner.label(self), record["uuid"]), maxlen=40)
self.final_output = record["output"]
outc = arvados.collection.CollectionReader(self.final_output,
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import datetime
+from arvados.errors import ApiError
+
+def get_intermediate_collection_info(workflow_step_name, current_container, intermediate_output_ttl):
+ if workflow_step_name:
+ name = "Intermediate collection for step %s" % (workflow_step_name)
+ else:
+ name = "Intermediate collection"
+ trash_time = None
+ if intermediate_output_ttl > 0:
+ trash_time = datetime.datetime.utcnow() + datetime.timedelta(seconds=intermediate_output_ttl)
+ container_uuid = None
+ if current_container:
+ container_uuid = current_container['uuid']
+ props = {"type": "intermediate", "container": container_uuid}
+
+ return {"name" : name, "trash_at" : trash_time, "properties" : props}
+
+def get_current_container(api, num_retries=0, logger=None):
+ current_container = None
+ try:
+ current_container = api.containers().current().execute(num_retries=num_retries)
+ except ApiError as e:
+ # Status code 404 just means we're not running in a container.
+ if e.resp.status != 404 and logger:
+ logger.info("Getting current container: %s", e)
+ return current_container
import os
import re
+SETUP_DIR = os.path.dirname(__file__) or '.'
+
def git_latest_tag():
gitinfo = subprocess.check_output(
['git', 'describe', '--abbrev=0']).strip()
return str(gitinfo.decode('utf-8'))
+def choose_version_from():
+ sdk_ts = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', os.path.join(SETUP_DIR, "../python")]).strip()
+ cwl_ts = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', SETUP_DIR]).strip()
+ if int(sdk_ts) > int(cwl_ts):
+ getver = os.path.join(SETUP_DIR, "../python")
+ else:
+ getver = SETUP_DIR
+ return getver
+
def git_timestamp_tag():
gitinfo = subprocess.check_output(
['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', '.']).strip()
+ '--format=format:%ct', choose_version_from()]).strip()
return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))
def save_version(setup_dir, module, v):
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20180524215209',
- 'schema-salad==2.7.20180501211602',
+ 'cwltool==1.0.20180806194258',
+ 'schema-salad==2.7.20180719125426',
'typing >= 3.5.3',
'ruamel.yaml >=0.13.11, <0.15',
- 'arvados-python-client>=1.1.4.20180507184611',
+ 'arvados-python-client>=1.1.4.20180607143841',
'setuptools',
- 'ciso8601 >=1.0.6'
+ 'ciso8601 >=1.0.6, <2.0.0',
+ 'subprocess32>=3.5.1',
],
data_files=[
('share/doc/arvados-cwl-runner', ['LICENSE-2.0.txt', 'README.rst']),
],
test_suite='tests',
- tests_require=['mock>=1.0'],
+ tests_require=[
+ 'mock>=1.0',
+ 'subprocess32>=3.5.1',
+ ],
zip_safe=True
)
--- /dev/null
+fastq1:
+ class: File
+ location: keep:20850f01122e860fb878758ac1320877+71/sample1_S01_R1_001.fastq.gz
\ No newline at end of file
--- /dev/null
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ fastq1: File
+outputs:
+ out: stdout
+baseCommand: echo
+arguments:
+ - $(inputs.fastq1.size)
+stdout: size.txt
\ No newline at end of file
if ! arv-get 4d8a70b1e63b2aad6984e40e338e2373+69 > /dev/null ; then
arv-put --portable-data-hash secondaryFiles/hello.txt*
fi
+if ! arv-get 20850f01122e860fb878758ac1320877+71 > /dev/null ; then
+ arv-put --portable-data-hash samples/sample1_S01_R1_001.fastq.gz
+fi
+
exec cwltest --test arvados-tests.yml --tool arvados-cwl-runner $@ -- --disable-reuse --compute-checksum
out: null
tool: wf-defaults/wf7.cwl
doc: workflow level default in RunInSingleContainer
+
+- job: 13931-size-job.yml
+ output:
+ "out": {
+ "checksum": "sha1$5bf6e5357bd42a6b1d2a3a040e16a91490064d26",
+ "location": "size.txt",
+ "class": "File",
+ "size": 3
+ }
+ tool: 13931-size.cwl
+ doc: Test that size is set for files in Keep
#
# SPDX-License-Identifier: Apache-2.0
-cwlVersion: v1.0
-$graph:
-- class: Workflow
- inputs: []
- outputs: []
- steps:
- - in: []
- out: []
- run: '#step1.cwl'
- id: '#main/step1'
- - in: []
- out: []
- run: '#step2.cwl'
- id: '#main/step2'
- id: '#main'
-- class: CommandLineTool
- inputs:
- - type: File
- default:
- class: File
- location: keep:b9fca8bf06b170b8507b80b2564ee72b+57/a.txt
- id: '#step1.cwl/a'
- - type: File
- default:
- class: File
- location: keep:b9fca8bf06b170b8507b80b2564ee72b+57/b.txt
- id: '#step1.cwl/b'
- outputs: []
- arguments: [echo, $(inputs.a), $(inputs.b)]
- id: '#step1.cwl'
-- class: CommandLineTool
- inputs:
- - type: File
- default:
- class: File
- location: keep:8e2d09a066d96cdffdd2be41579e4e2e+57/b.txt
- id: '#step2.cwl/b'
- - type: File
- default:
- class: File
- location: keep:8e2d09a066d96cdffdd2be41579e4e2e+57/c.txt
- id: '#step2.cwl/c'
- outputs: []
- arguments: [echo, $(inputs.c), $(inputs.b)]
- id: '#step2.cwl'
+{
+ "$graph": [
+ {
+ "class": "Workflow",
+ "id": "#main",
+ "inputs": [],
+ "outputs": [],
+ "steps": [
+ {
+ "id": "#main/step1",
+ "in": [],
+ "out": [],
+ "run": "#step1.cwl"
+ },
+ {
+ "id": "#main/step2",
+ "in": [],
+ "out": [],
+ "run": "#step2.cwl"
+ }
+ ]
+ },
+ {
+ "arguments": [
+ "echo",
+ "$(inputs.a)",
+ "$(inputs.b)"
+ ],
+ "class": "CommandLineTool",
+ "id": "#step1.cwl",
+ "inputs": [
+ {
+ "default": {
+ "class": "File",
+ "location": "keep:b9fca8bf06b170b8507b80b2564ee72b+57/a.txt"
+ },
+ "id": "#step1.cwl/a",
+ "type": "File"
+ },
+ {
+ "default": {
+ "class": "File",
+ "location": "keep:b9fca8bf06b170b8507b80b2564ee72b+57/b.txt"
+ },
+ "id": "#step1.cwl/b",
+ "type": "File"
+ }
+ ],
+ "outputs": []
+ },
+ {
+ "arguments": [
+ "echo",
+ "$(inputs.c)",
+ "$(inputs.b)"
+ ],
+ "class": "CommandLineTool",
+ "id": "#step2.cwl",
+ "inputs": [
+ {
+ "default": {
+ "class": "File",
+ "location": "keep:8e2d09a066d96cdffdd2be41579e4e2e+57/b.txt"
+ },
+ "id": "#step2.cwl/b",
+ "type": "File"
+ },
+ {
+ "default": {
+ "class": "File",
+ "location": "keep:8e2d09a066d96cdffdd2be41579e4e2e+57/c.txt"
+ },
+ "id": "#step2.cwl/c",
+ "type": "File"
+ }
+ ],
+ "outputs": []
+ }
+ ],
+ "cwlVersion": "v1.0"
+}
\ No newline at end of file
--- /dev/null
+class: CommandLineTool
+cwlVersion: v1.0
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - $(inputs.inp1)
+ - $(inputs.inp2)
+ - $(inputs.inp3)
+inputs:
+ inp1: File
+ inp2: [File, Directory]
+ inp3: Directory
+outputs: []
+arguments: [echo, $(inputs.inp1), $(inputs.inp2), $(inputs.inp3)]
--- /dev/null
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ SubworkflowFeatureRequirement: {}
+inputs:
+ inp1:
+ type: File
+ default:
+ class: File
+ location: hello1.txt
+ inp2:
+ type: [File, Directory]
+ default:
+ class: File
+ basename: "hello2.txt"
+ contents: "Hello world"
+ inp3:
+ type: [File, Directory]
+ default:
+ class: Directory
+ basename: inp3
+ listing:
+ - class: File
+ basename: "hello3.txt"
+ contents: "hello world"
+outputs: []
+steps:
+ step1:
+ requirements:
+ arv:RunInSingleContainer: {}
+ in:
+ inp1: inp1
+ inp2: inp2
+ inp3: inp3
+ out: []
+ run: subwf.cwl
--- /dev/null
+cwlVersion: v1.0
+class: Workflow
+inputs:
+ inp1: File
+ inp2: File
+ inp3: Directory
+outputs: []
+steps:
+ step1:
+ in:
+ inp1: inp1
+ inp2: inp2
+ inp3: inp3
+ out: []
+ run: echo.cwl
# SPDX-License-Identifier: Apache-2.0
import arvados_cwl
+import arvados_cwl.context
from arvados_cwl.arvdocker import arv_docker_clear_cache
import logging
import mock
logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
-
class TestContainer(unittest.TestCase):
+ def helper(self, runner, enable_reuse=True):
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+
+ make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+ collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
+ loadingContext = arvados_cwl.context.ArvLoadingContext(
+ {"avsc_names": avsc_names,
+ "basedir": "",
+ "make_fs_access": make_fs_access,
+ "loader": Loader({}),
+ "metadata": {"cwlVersion": "v1.0"}})
+ runtimeContext = arvados_cwl.context.ArvRuntimeContext(
+ {"work_api": "containers",
+ "basedir": "",
+ "name": "test_run_"+str(enable_reuse),
+ "make_fs_access": make_fs_access,
+ "tmpdir": "/tmp",
+ "enable_reuse": enable_reuse,
+ "priority": 500})
+
+ return loadingContext, runtimeContext
+
# The test passes no builder.resources
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
runner.api.collections().get().execute.return_value = {
"portable_data_hash": "99999999999999999999999999999993+99"}
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
-
tool = cmap({
"inputs": [],
"outputs": [],
"id": "#",
"class": "CommandLineTool"
})
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
- collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
- arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
- basedir="", make_fs_access=make_fs_access, loader=Loader({}),
- metadata={"cwlVersion": "v1.0"})
+
+ loadingContext, runtimeContext = self.helper(runner, enable_reuse)
+
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
arvtool.formatgraph = None
- for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
- make_fs_access=make_fs_access, tmpdir="/tmp"):
- j.run(enable_reuse=enable_reuse, priority=500)
+
+ for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
runner.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher({
'environment': {
"capacity": 1073741824 }
},
'state': 'Committed',
+ 'output_name': 'Output for step test_run_'+str(enable_reuse),
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
runner.intermediate_output_ttl = 3600
runner.secret_store = cwltool.secrets.SecretStore()
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
-
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
"portable_data_hash": "99999999999999999999999999999993+99"}
"id": "#",
"class": "CommandLineTool"
})
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
- collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
- arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
- avsc_names=avsc_names, make_fs_access=make_fs_access,
- loader=Loader({}), metadata={"cwlVersion": "v1.0"})
+
+ loadingContext, runtimeContext = self.helper(runner)
+ runtimeContext.name = "test_resource_requirements"
+
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
arvtool.formatgraph = None
- for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
- make_fs_access=make_fs_access, tmpdir="/tmp"):
- j.run(enable_reuse=True, priority=500)
+ for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
call_args, call_kwargs = runner.api.container_requests().create.call_args
"capacity": 5242880000 }
},
'state': 'Committed',
+ 'output_name': 'Output for step test_resource_requirements',
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 7200,
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
-
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
"portable_data_hash": "99999999999999999999999999999993+99"}
"id": "#",
"class": "CommandLineTool"
})
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
- collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
- arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
- avsc_names=avsc_names, make_fs_access=make_fs_access,
- loader=Loader({}), metadata={"cwlVersion": "v1.0"})
+
+ loadingContext, runtimeContext = self.helper(runner)
+ runtimeContext.name = "test_initial_work_dir"
+
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
arvtool.formatgraph = None
- for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_initial_work_dir",
- make_fs_access=make_fs_access, tmpdir="/tmp"):
- j.run(priority=500)
+ for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
call_args, call_kwargs = runner.api.container_requests().create.call_args
}
},
'state': 'Committed',
+ 'output_name': 'Output for step test_initial_work_dir',
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
"id": "#",
"class": "CommandLineTool"
})
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
- collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
- arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
- basedir="", make_fs_access=make_fs_access, loader=Loader({}),
- metadata={"cwlVersion": "v1.0"})
+
+ loadingContext, runtimeContext = self.helper(runner)
+ runtimeContext.name = "test_run_redirect"
+
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
arvtool.formatgraph = None
- for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_redirect",
- make_fs_access=make_fs_access, tmpdir="/tmp"):
- j.run(priority=500)
+ for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
runner.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher({
'environment': {
},
},
'state': 'Committed',
+ "output_name": "Output for step test_run_redirect",
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
col().open.return_value = []
- arvjob = arvados_cwl.ArvadosContainer(runner)
- arvjob.name = "testjob"
- arvjob.builder = mock.MagicMock()
+ arvjob = arvados_cwl.ArvadosContainer(runner,
+ mock.MagicMock(),
+ {},
+ None,
+ [],
+ [],
+ "testjob")
arvjob.output_callback = mock.MagicMock()
arvjob.collect_outputs = mock.MagicMock()
arvjob.successCodes = [0]
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
- "portable_data_hash": "99999999999999999999999999999993+99"}
+ "portable_data_hash": "99999999999999999999999999999994+99",
+ "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
"id": "#",
"class": "CommandLineTool"
})
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
- collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
- arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
- basedir="", make_fs_access=make_fs_access, loader=Loader({}),
- metadata={"cwlVersion": "v1.0"})
+
+ loadingContext, runtimeContext = self.helper(runner)
+ runtimeContext.name = "test_run_mounts"
+
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
arvtool.formatgraph = None
job_order = {
"p1": {
]
}
}
- for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_run_mounts",
- make_fs_access=make_fs_access, tmpdir="/tmp"):
- j.run(priority=500)
+ for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
runner.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher({
'environment': {
"capacity": 1073741824 }
},
'state': 'Committed',
+ 'output_name': 'Output for step test_run_mounts',
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
]
}
]})
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
- collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
- arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
- basedir="", make_fs_access=make_fs_access, loader=Loader({}),
- metadata={"cwlVersion": "v1.0"})
+
+ loadingContext, runtimeContext = self.helper(runner)
+ runtimeContext.name = "test_secrets"
+
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
arvtool.formatgraph = None
job_order = {"pw": "blorp"}
runner.secret_store.store(["pw"], job_order)
- for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_secrets",
- make_fs_access=make_fs_access, tmpdir="/tmp"):
- j.run(enable_reuse=True, priority=500)
+ for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
runner.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher({
'environment': {
"capacity": 1073741824 }
},
'state': 'Committed',
+ 'output_name': 'Output for step test_secrets',
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
}
}
}))
+
+ # The test passes no builder.resources
+ # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
+ @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
+ def test_timelimit(self, keepdocker):
+ arv_docker_clear_cache()
+
+ runner = mock.MagicMock()
+ runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+ runner.ignore_docker_for_reuse = False
+ runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
+
+ keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
+ runner.api.collections().get().execute.return_value = {
+ "portable_data_hash": "99999999999999999999999999999993+99"}
+
+ tool = cmap({
+ "inputs": [],
+ "outputs": [],
+ "baseCommand": "ls",
+ "arguments": [{"valueFrom": "$(runtime.outdir)"}],
+ "id": "#",
+ "class": "CommandLineTool",
+ "hints": [
+ {
+ "class": "http://commonwl.org/cwltool#TimeLimit",
+ "timelimit": 42
+ }
+ ]
+ })
+
+ loadingContext, runtimeContext = self.helper(runner)
+ runtimeContext.name = "test_timelimit"
+
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
+ arvtool.formatgraph = None
+
+ for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
+
+ _, kwargs = runner.api.container_requests().create.call_args
+ self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
class TestJob(unittest.TestCase):
+ def helper(self, runner, enable_reuse=True):
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+
+ make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+ collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
+ loadingContext = arvados_cwl.context.ArvLoadingContext(
+ {"avsc_names": avsc_names,
+ "basedir": "",
+ "make_fs_access": make_fs_access,
+ "loader": Loader({}),
+ "metadata": {"cwlVersion": "v1.0"},
+ "makeTool": runner.arv_make_tool})
+ runtimeContext = arvados_cwl.context.ArvRuntimeContext(
+ {"work_api": "jobs",
+ "basedir": "",
+ "name": "test_run_job_"+str(enable_reuse),
+ "make_fs_access": make_fs_access,
+ "enable_reuse": enable_reuse,
+ "priority": 500})
+
+ return loadingContext, runtimeContext
+
# The test passes no builder.resources
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@mock.patch('arvados.commands.keepdocker.list_images_in_arv')
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.num_retries = 0
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
"id": "#",
"class": "CommandLineTool"
})
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
- collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
- arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names,
- basedir="", make_fs_access=make_fs_access, loader=Loader({}),
- metadata={"cwlVersion": "v1.0"})
+
+ loadingContext, runtimeContext = self.helper(runner, enable_reuse)
+
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
arvtool.formatgraph = None
- for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access):
- j.run(enable_reuse=enable_reuse)
+ for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
runner.api.jobs().create.assert_called_with(
body=JsonDiffMatcher({
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
runner.api.links().create.side_effect = ApiError(
mock.MagicMock(return_value={'status': 403}),
'Permission denied')
- j.run(enable_reuse=enable_reuse)
+ j.run(runtimeContext)
else:
assert not runner.api.links().create.called
list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
-
-
tool = {
"inputs": [],
"outputs": [],
"id": "#",
"class": "CommandLineTool"
}
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
- collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
- arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names,
- make_fs_access=make_fs_access, loader=Loader({}),
- metadata={"cwlVersion": "v1.0"})
+
+ loadingContext, runtimeContext = self.helper(runner)
+
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
arvtool.formatgraph = None
- for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access):
- j.run(enable_reuse=True)
+ for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
runner.api.jobs().create.assert_called_with(
body=JsonDiffMatcher({
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
{"items": []},
{"items": [{"manifest_text": "ABC"}]})
- arvjob = arvados_cwl.ArvadosJob(runner)
- arvjob.name = "testjob"
- arvjob.builder = mock.MagicMock()
+ arvjob = arvados_cwl.ArvadosJob(runner,
+ mock.MagicMock(),
+ {},
+ None,
+ [],
+ [],
+ "testjob")
arvjob.output_callback = mock.MagicMock()
arvjob.collect_outputs = mock.MagicMock()
arvjob.collect_outputs.return_value = {"out": "stuff"}
{"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},
)
- arvjob = arvados_cwl.ArvadosJob(runner)
- arvjob.name = "testjob"
- arvjob.builder = mock.MagicMock()
+ arvjob = arvados_cwl.ArvadosJob(runner,
+ mock.MagicMock(),
+ {},
+ None,
+ [],
+ [],
+ "testjob")
arvjob.output_callback = mock.MagicMock()
arvjob.collect_outputs = mock.MagicMock()
arvjob.collect_outputs.return_value = {"out": "stuff"}
class TestWorkflow(unittest.TestCase):
+ def helper(self, runner, enable_reuse=True):
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+
+ make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+ collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
+
+ document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
+ document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
+ document_loader.fetch_text = document_loader.fetcher.fetch_text
+ document_loader.check_exists = document_loader.fetcher.check_exists
+
+ loadingContext = arvados_cwl.context.ArvLoadingContext(
+ {"avsc_names": avsc_names,
+ "basedir": "",
+ "make_fs_access": make_fs_access,
+ "loader": document_loader,
+ "metadata": {"cwlVersion": "v1.0"},
+ "construct_tool_object": runner.arv_make_tool})
+ runtimeContext = arvados_cwl.context.ArvRuntimeContext(
+ {"work_api": "jobs",
+ "basedir": "",
+ "name": "test_run_wf_"+str(enable_reuse),
+ "make_fs_access": make_fs_access,
+ "enable_reuse": enable_reuse,
+ "priority": 500})
+
+ return loadingContext, runtimeContext
+
# The test passes no builder.resources
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@mock.patch("arvados.collection.CollectionReader")
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.num_retries = 0
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
- collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
- document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=api, fs_access=make_fs_access(""))
- document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
- document_loader.fetch_text = document_loader.fetcher.fetch_text
- document_loader.check_exists = document_loader.fetcher.check_exists
+ loadingContext, runtimeContext = self.helper(runner)
- tool, metadata = document_loader.resolve_ref("tests/wf/scatter2.cwl")
+ tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
metadata["cwlVersion"] = tool["cwlVersion"]
mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
+ mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
- arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names,
- basedir="", make_fs_access=make_fs_access, loader=document_loader,
- makeTool=runner.arv_make_tool, metadata=metadata)
+ arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
arvtool.formatgraph = None
- it = arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access, tmp_outdir_prefix="")
- it.next().run()
- it.next().run()
+ it = arvtool.job({}, mock.MagicMock(), runtimeContext)
+
+ it.next().run(runtimeContext)
+ it.next().run(runtimeContext)
with open("tests/wf/scatter2_subwf.cwl") as f:
subwf = StripYAMLComments(f.read())
"fileblub": {
"basename": "token.txt",
"class": "File",
- "location": "/keep/99999999999999999999999999999999+118/token.txt"
+ "location": "/keep/99999999999999999999999999999999+118/token.txt",
+ "size": 0
},
"sleeptime": 5
}''')])
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.num_retries = 0
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
- collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
- document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=api, fs_access=make_fs_access(""))
- document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
- document_loader.fetch_text = document_loader.fetcher.fetch_text
- document_loader.check_exists = document_loader.fetcher.check_exists
+ loadingContext, runtimeContext = self.helper(runner)
- tool, metadata = document_loader.resolve_ref("tests/wf/echo-wf.cwl")
+ tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
metadata["cwlVersion"] = tool["cwlVersion"]
mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
- arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names,
- basedir="", make_fs_access=make_fs_access, loader=document_loader,
- makeTool=runner.arv_make_tool, metadata=metadata)
+ arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
arvtool.formatgraph = None
- it = arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access, tmp_outdir_prefix="")
- it.next().run()
- it.next().run()
+ it = arvtool.job({}, mock.MagicMock(), runtimeContext)
+ it.next().run(runtimeContext)
+ it.next().run(runtimeContext)
with open("tests/wf/echo-subwf.cwl") as f:
subwf = StripYAMLComments(f.read())
final.open.return_value = openmock
openmock.__enter__.return_value = cwlout
- _, runner.final_output_collection = runner.make_output_collection("Test output", "tag0,tag1,tag2", {
+ _, runner.final_output_collection = runner.make_output_collection("Test output", ["foo"], "tag0,tag1,tag2", {
"foo": {
"class": "File",
"location": "keep:99999999999999999999999999999991+99/foo.txt",
final.copy.assert_has_calls([mock.call('bar.txt', 'baz.txt', overwrite=False, source_collection=readermock)])
final.copy.assert_has_calls([mock.call('foo.txt', 'foo.txt', overwrite=False, source_collection=readermock)])
- final.save_new.assert_has_calls([mock.call(ensure_unique_name=True, name='Test output', owner_uuid='zzzzz-j7d0g-zzzzzzzzzzzzzzz')])
+ final.save_new.assert_has_calls([mock.call(ensure_unique_name=True, name='Test output', owner_uuid='zzzzz-j7d0g-zzzzzzzzzzzzzzz', storage_classes=['foo'])])
self.assertEqual("""{
"bar": {
"basename": "baz.txt",
from arvados_cwl.pathmapper import ArvPathMapper
-def upload_mock(files, api, dry_run=False, num_retries=0, project=None, fnPattern="$(file %s/%s)", name=None, collection=None):
+def upload_mock(files, api, dry_run=False, num_retries=0, project=None, fnPattern="$(file %s/%s)", name=None, collection=None, packed=None):
pdh = "99999999999999999999999999999991+99"
for c in files:
c.keepref = "%s/%s" % (pdh, os.path.basename(c.fn))
self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
p._pathmap)
- @mock.patch("arvados.commands.run.uploadfiles")
- def test_prev_uploaded(self, upl):
- """Test pathmapper handling previously uploaded files."""
-
- arvrunner = arvados_cwl.ArvCwlRunner(self.api)
- arvrunner.add_uploaded('file:tests/hw.py', MapperEnt(resolved='keep:99999999999999999999999999999992+99/hw.py', target='', type='File', staged=True))
-
- upl.side_effect = upload_mock
-
- p = ArvPathMapper(arvrunner, [{
- "class": "File",
- "location": "file:tests/hw.py"
- }], "", "/test/%s", "/test/%s/%s")
-
- self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999992+99/hw.py', target='/test/99999999999999999999999999999992+99/hw.py', type='File', staged=True)},
- p._pathmap)
-
@mock.patch("arvados.commands.run.uploadfiles")
@mock.patch("arvados.commands.run.statfile")
def test_statfile(self, statfile, upl):
"listing": [{
"basename": "renamed.txt",
"class": "File",
- "location": "keep:99999999999999999999999999999998+99/file1.txt"
+ "location": "keep:99999999999999999999999999999998+99/file1.txt",
+ "size": 0
}],
'class': 'Directory'
},
{
'basename': 'renamed.txt',
'class': 'File', 'location':
- 'keep:99999999999999999999999999999998+99/file1.txt'
+ 'keep:99999999999999999999999999999998+99/file1.txt',
+ 'size': 0
}
]}},
'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
{'basename': 'renamed.txt',
'class': 'File',
- 'location': 'keep:99999999999999999999999999999998+99/file1.txt'
+ 'location': 'keep:99999999999999999999999999999998+99/file1.txt',
+ 'size': 0
}
]}
},
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_pipeline_uuid + '\n')
+ @stubs
+ def test_error_when_multiple_storage_classes_specified(self, stubs):
+ storage_classes = "foo,bar"
+ exited = arvados_cwl.main(
+ ["--debug", "--storage-classes", storage_classes,
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ sys.stdin, sys.stderr, api_client=stubs.api)
+ self.assertEqual(exited, 1)
+
@mock.patch("time.sleep")
@stubs
def test_submit_on_error(self, stubs, tm):
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
+ @stubs
+ def test_submit_storage_classes(self, stubs):
+ capture_stdout = cStringIO.StringIO()
+ try:
+ exited = arvados_cwl.main(
+ ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+ expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate',
+ '--eval-timeout=20', '--thread-count=4',
+ '--enable-reuse', "--debug",
+ "--storage-classes=foo", '--on-error=continue',
+ '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container))
+ self.assertEqual(capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+
+ @mock.patch("arvados_cwl.task_queue.TaskQueue")
+ @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
+ @mock.patch("arvados_cwl.ArvCwlRunner.make_output_collection", return_value = (None, None))
+ @stubs
+ def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
+ def set_final_output(job_order, output_callback, runtimeContext):
+ output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
+ return []
+ job.side_effect = set_final_output
+
+ try:
+ exited = arvados_cwl.main(
+ ["--debug", "--local", "--storage-classes=foo",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
+
+ make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
+
+ @mock.patch("arvados_cwl.task_queue.TaskQueue")
+ @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
+ @mock.patch("arvados_cwl.ArvCwlRunner.make_output_collection", return_value = (None, None))
+ @stubs
+ def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
+ def set_final_output(job_order, output_callback, runtimeContext):
+ output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
+ return []
+ job.side_effect = set_final_output
+
+ try:
+ exited = arvados_cwl.main(
+ ["--debug", "--local",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
+
+ make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
@stubs
def test_submit_container_output_ttl(self, stubs):
@stubs
def test_submit_file_keepref(self, stubs, tm, collectionReader):
capture_stdout = cStringIO.StringIO()
+ collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug",
"tests/wf/submit_keepref_wf.cwl"],
stubs.expect_container_request_uuid + '\n')
+ @stubs
+ def test_submit_wf_runner_resources(self, stubs):
+ capture_stdout = cStringIO.StringIO()
+ try:
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug",
+ "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
+ capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+ expect_container["runtime_constraints"] = {
+ "API": True,
+ "vcpus": 2,
+ "ram": 2000 * 2**20
+ }
+ expect_container["name"] = "submit_wf_runner_resources.cwl"
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
+ {
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources",
+ "coresMin": 2,
+ "ramMin": 2000
+ }
+ ]
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
+ "arv": "http://arvados.org/cwl#",
+ }
+
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container))
+ self.assertEqual(capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+
+
@mock.patch("arvados.commands.keepdocker.find_one_image_hash")
@mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
@mock.patch("arvados.api")
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import unittest
+import mock
+import datetime
+import httplib2
+
+from arvados_cwl.util import *
+from arvados.errors import ApiError
+
+class MockDateTime(datetime.datetime):
+ @classmethod
+ def utcnow(cls):
+ return datetime.datetime(2018, 1, 1, 0, 0, 0, 0)
+
+datetime.datetime = MockDateTime
+
+class TestUtil(unittest.TestCase):
+ def test_get_intermediate_collection_info(self):
+ name = "one"
+ current_container = {"uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
+ intermediate_output_ttl = 120
+
+ info = get_intermediate_collection_info(name, current_container, intermediate_output_ttl)
+
+ self.assertEqual(info["name"], "Intermediate collection for step one")
+ self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 2, 0, 0))
+ self.assertEqual(info["properties"], {"type" : "intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})
+
+ def test_get_current_container_success(self):
+ api = mock.MagicMock()
+ api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
+
+ current_container = get_current_container(api)
+
+ self.assertEqual(current_container, {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})
+
+ def test_get_current_container_error(self):
+ api = mock.MagicMock()
+ api.containers().current().execute.side_effect = ApiError(httplib2.Response({"status": 300}), "")
+ logger = mock.MagicMock()
+
+ self.assertRaises(ApiError, get_current_container(api, num_retries=0, logger=logger))
#
# SPDX-License-Identifier: Apache-2.0
-cwlVersion: v1.0
-$graph:
-- class: CommandLineTool
- requirements:
- - class: DockerRequirement
- dockerPull: debian:8
- inputs:
- - id: '#submit_tool.cwl/x'
- type: File
- default:
- class: File
- location: keep:5d373e7629203ce39e7c22af98a0f881+52/blub.txt
- inputBinding:
- position: 1
- outputs: []
- baseCommand: cat
- id: '#submit_tool.cwl'
-- class: Workflow
- inputs:
- - id: '#main/x'
- type: File
- default: {class: File, location: keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt,
- size: 16, basename: blorp.txt, nameroot: blorp, nameext: .txt}
- - id: '#main/y'
- type: Directory
- default: {class: Directory, location: keep:99999999999999999999999999999998+99,
- basename: 99999999999999999999999999999998+99}
- - id: '#main/z'
- type: Directory
- default: {class: Directory, basename: anonymous, listing: [{basename: renamed.txt,
- class: File, location: keep:99999999999999999999999999999998+99/file1.txt,
- nameroot: renamed, nameext: .txt}]}
- outputs: []
- steps:
- - id: '#main/step1'
- in:
- - {id: '#main/step1/x', source: '#main/x'}
- out: []
- run: '#submit_tool.cwl'
- id: '#main'
+{
+ "$graph": [
+ {
+ "baseCommand": "cat",
+ "class": "CommandLineTool",
+ "id": "#submit_tool.cwl",
+ "inputs": [
+ {
+ "default": {
+ "class": "File",
+ "location": "keep:5d373e7629203ce39e7c22af98a0f881+52/blub.txt"
+ },
+ "id": "#submit_tool.cwl/x",
+ "inputBinding": {
+ "position": 1
+ },
+ "type": "File"
+ }
+ ],
+ "outputs": [],
+ "requirements": [
+ {
+ "class": "DockerRequirement",
+ "dockerPull": "debian:8"
+ }
+ ]
+ },
+ {
+ "class": "Workflow",
+ "id": "#main",
+ "inputs": [
+ {
+ "default": {
+ "basename": "blorp.txt",
+ "class": "File",
+ "location": "keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt",
+ "nameext": ".txt",
+ "nameroot": "blorp",
+ "size": 16
+ },
+ "id": "#main/x",
+ "type": "File"
+ },
+ {
+ "default": {
+ "basename": "99999999999999999999999999999998+99",
+ "class": "Directory",
+ "location": "keep:99999999999999999999999999999998+99"
+ },
+ "id": "#main/y",
+ "type": "Directory"
+ },
+ {
+ "default": {
+ "basename": "anonymous",
+ "class": "Directory",
+ "listing": [
+ {
+ "basename": "renamed.txt",
+ "class": "File",
+ "location": "keep:99999999999999999999999999999998+99/file1.txt",
+ "nameext": ".txt",
+ "nameroot": "renamed",
+ "size": 0
+ }
+ ]
+ },
+ "id": "#main/z",
+ "type": "Directory"
+ }
+ ],
+ "outputs": [],
+ "steps": [
+ {
+ "id": "#main/step1",
+ "in": [
+ {
+ "id": "#main/step1/x",
+ "source": "#main/x"
+ }
+ ],
+ "out": [],
+ "run": "#submit_tool.cwl"
+ }
+ ]
+ }
+ ],
+ "cwlVersion": "v1.0"
+}
\ No newline at end of file
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Test case for arvados-cwl-runner
+#
+# Used to test whether scanning a workflow file for dependencies
+# (e.g. submit_tool.cwl) and uploading to Keep works as intended.
+
+class: Workflow
+cwlVersion: v1.0
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+hints:
+ arv:WorkflowRunnerResources:
+ ramMin: 2000
+ coresMin: 2
+inputs:
+ - id: x
+ type: File
+ - id: y
+ type: Directory
+ - id: z
+ type: Directory
+outputs: []
+steps:
+ - id: step1
+ in:
+ - { id: x, source: "#x" }
+ out: []
+ run: ../tool/submit_tool.cwl
RUN apt-get update -q && apt-get install -qy git python-pip python-virtualenv python-dev libcurl4-gnutls-dev libgnutls28-dev nodejs python-pyasn1-modules
-RUN pip install -U setuptools
+RUN pip install -U setuptools six
ARG sdk
ARG runner
type APIClientAuthorizationList struct {
Items []APIClientAuthorization `json:"items"`
}
+
+func (aca APIClientAuthorization) TokenV2() string {
+ return "v2/" + aca.UUID + "/" + aca.APIToken
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "encoding/json"
+ "fmt"
+ "math"
+ "strings"
+)
+
+type ByteSize int64
+
+var prefixValue = map[string]int64{
+ "": 1,
+ "K": 1000,
+ "Ki": 1 << 10,
+ "M": 1000000,
+ "Mi": 1 << 20,
+ "G": 1000000000,
+ "Gi": 1 << 30,
+ "T": 1000000000000,
+ "Ti": 1 << 40,
+ "P": 1000000000000000,
+ "Pi": 1 << 50,
+ "E": 1000000000000000000,
+ "Ei": 1 << 60,
+}
+
+func (n *ByteSize) UnmarshalJSON(data []byte) error {
+ if len(data) == 0 || data[0] != '"' {
+ var i int64
+ err := json.Unmarshal(data, &i)
+ if err != nil {
+ return err
+ }
+ *n = ByteSize(i)
+ return nil
+ }
+ var s string
+ err := json.Unmarshal(data, &s)
+ if err != nil {
+ return err
+ }
+ split := strings.LastIndexAny(s, "0123456789.+-eE") + 1
+ if split == 0 {
+ return fmt.Errorf("invalid byte size %q", s)
+ }
+ if s[split-1] == 'E' {
+ // We accepted an E as if it started the exponent part
+ // of a json number, but if the next char isn't +, -,
+ // or digit, then the E must have meant Exa. Instead
+ // of "4.5E"+"iB" we want "4.5"+"EiB".
+ split--
+ }
+ var val json.Number
+ dec := json.NewDecoder(strings.NewReader(s[:split]))
+ dec.UseNumber()
+ err = dec.Decode(&val)
+ if err != nil {
+ return err
+ }
+ if split == len(s) {
+ return nil
+ }
+ prefix := strings.Trim(s[split:], " ")
+ if strings.HasSuffix(prefix, "B") {
+ prefix = prefix[:len(prefix)-1]
+ }
+ pval, ok := prefixValue[prefix]
+ if !ok {
+ return fmt.Errorf("invalid unit %q", strings.Trim(s[split:], " "))
+ }
+ if intval, err := val.Int64(); err == nil {
+ if pval > 1 && (intval*pval)/pval != intval {
+ return fmt.Errorf("size %q overflows int64", s)
+ }
+ *n = ByteSize(intval * pval)
+ return nil
+ } else if floatval, err := val.Float64(); err == nil {
+ if floatval*float64(pval) > math.MaxInt64 {
+ return fmt.Errorf("size %q overflows int64", s)
+ }
+ *n = ByteSize(int64(floatval * float64(pval)))
+ return nil
+ } else {
+ return fmt.Errorf("bug: json.Number for %q is not int64 or float64: %s", s, err)
+ }
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "github.com/ghodss/yaml"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&ByteSizeSuite{})
+
+type ByteSizeSuite struct{}
+
+func (s *ByteSizeSuite) TestUnmarshal(c *check.C) {
+ for _, testcase := range []struct {
+ in string
+ out int64
+ }{
+ {"0", 0},
+ {"5", 5},
+ {"5B", 5},
+ {"5 B", 5},
+ {" 4 KiB ", 4096},
+ {"0K", 0},
+ {"0Ki", 0},
+ {"0 KiB", 0},
+ {"4K", 4000},
+ {"4KB", 4000},
+ {"4Ki", 4096},
+ {"4KiB", 4096},
+ {"4MB", 4000000},
+ {"4MiB", 4194304},
+ {"4GB", 4000000000},
+ {"4 GiB", 4294967296},
+ {"4TB", 4000000000000},
+ {"4TiB", 4398046511104},
+ {"4PB", 4000000000000000},
+ {"4PiB", 4503599627370496},
+ {"4EB", 4000000000000000000},
+ {"4EiB", 4611686018427387904},
+ {"4.5EiB", 5188146770730811392},
+ {"1.5 GB", 1500000000},
+ {"1.5 GiB", 1610612736},
+ {"1.234 GiB", 1324997410}, // rounds down from 1324997410.816
+ {"1e2 KB", 100000},
+ {"20E-1 KiB", 2048},
+ {"1E0EB", 1000000000000000000},
+ {"1E-1EB", 100000000000000000},
+ {"1E-1EiB", 115292150460684704},
+ {"4.5E15 K", 4500000000000000000},
+ } {
+ var n ByteSize
+ err := yaml.Unmarshal([]byte(testcase.in+"\n"), &n)
+ c.Logf("%v => %v: %v", testcase.in, testcase.out, n)
+ c.Check(err, check.IsNil)
+ c.Check(int64(n), check.Equals, testcase.out)
+ }
+ for _, testcase := range []string{
+ "B", "K", "KB", "KiB", "4BK", "4iB", "4A", "b", "4b", "4mB", "4m", "4mib", "4KIB", "4K iB", "4Ki B", "BB", "4BB",
+ "400000 EB", // overflows int64
+ "4.11e4 EB", // ok as float64, but overflows int64
+ } {
+ var n ByteSize
+ err := yaml.Unmarshal([]byte(testcase+"\n"), &n)
+ c.Logf("%v => error: %v", n, err)
+ c.Check(err, check.NotNil)
+ }
+}
package arvados
import (
+ "encoding/json"
+ "errors"
"fmt"
"os"
}
type Cluster struct {
- ClusterID string `json:"-"`
- ManagementToken string
- SystemNodes map[string]SystemNode
- InstanceTypes []InstanceType
+ ClusterID string `json:"-"`
+ ManagementToken string
+ NodeProfiles map[string]NodeProfile
+ InstanceTypes InstanceTypeMap
+ HTTPRequestTimeout Duration
+ RemoteClusters map[string]RemoteCluster
+ PostgreSQL PostgreSQL
+}
+
+type PostgreSQL struct {
+ Connection PostgreSQLConnection
+ ConnectionPool int
+}
+
+type PostgreSQLConnection map[string]string
+
+type RemoteCluster struct {
+ // API endpoint host or host:port; default is {id}.arvadosapi.com
+ Host string
+ // Perform a proxy request when a local client requests an
+ // object belonging to this remote.
+ Proxy bool
+ // Scheme, default "https". Can be set to "http" for testing.
+ Scheme string
+ // Disable TLS verify. Can be set to true for testing.
+ Insecure bool
}
type InstanceType struct {
Name string
ProviderType string
VCPUs int
- RAM int64
- Scratch int64
+ RAM ByteSize
+ Scratch ByteSize
Price float64
+ Preemptible bool
}
-// GetThisSystemNode returns a SystemNode for the node we're running
-// on right now.
-func (cc *Cluster) GetThisSystemNode() (*SystemNode, error) {
- hostname, err := os.Hostname()
+type InstanceTypeMap map[string]InstanceType
+
+var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
+
+// UnmarshalJSON handles old config files that provide an array of
+// instance types instead of a hash.
+func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
+ if len(data) > 0 && data[0] == '[' {
+ var arr []InstanceType
+ err := json.Unmarshal(data, &arr)
+ if err != nil {
+ return err
+ }
+ if len(arr) == 0 {
+ *it = nil
+ return nil
+ }
+ *it = make(map[string]InstanceType, len(arr))
+ for _, t := range arr {
+ if _, ok := (*it)[t.Name]; ok {
+ return errDuplicateInstanceTypeName
+ }
+ (*it)[t.Name] = t
+ }
+ return nil
+ }
+ var hash map[string]InstanceType
+ err := json.Unmarshal(data, &hash)
if err != nil {
- return nil, err
+ return err
}
- return cc.GetSystemNode(hostname)
+ // Fill in Name field using hash key.
+ *it = InstanceTypeMap(hash)
+ for name, t := range *it {
+ t.Name = name
+ (*it)[name] = t
+ }
+ return nil
}
-// GetSystemNode returns a SystemNode for the given hostname. An error
-// is returned if the appropriate configuration can't be determined
-// (e.g., this does not appear to be a system node).
-func (cc *Cluster) GetSystemNode(node string) (*SystemNode, error) {
- if cfg, ok := cc.SystemNodes[node]; ok {
+// GetNodeProfile returns a NodeProfile for the given hostname. An
+// error is returned if the appropriate configuration can't be
+// determined (e.g., this does not appear to be a system node). If
+// node is empty, use the OS-reported hostname.
+func (cc *Cluster) GetNodeProfile(node string) (*NodeProfile, error) {
+ if node == "" {
+ hostname, err := os.Hostname()
+ if err != nil {
+ return nil, err
+ }
+ node = hostname
+ }
+ if cfg, ok := cc.NodeProfiles[node]; ok {
return &cfg, nil
}
// If node is not listed, but "*" gives a default system node
// config, use the default config.
- if cfg, ok := cc.SystemNodes["*"]; ok {
+ if cfg, ok := cc.NodeProfiles["*"]; ok {
return &cfg, nil
}
return nil, fmt.Errorf("config does not provision host %q as a system node", node)
}
-type SystemNode struct {
+type NodeProfile struct {
+ Controller SystemServiceInstance `json:"arvados-controller"`
Health SystemServiceInstance `json:"arvados-health"`
Keepproxy SystemServiceInstance `json:"keepproxy"`
Keepstore SystemServiceInstance `json:"keepstore"`
Workbench SystemServiceInstance `json:"arvados-workbench"`
}
+type ServiceName string
+
+const (
+ ServiceNameRailsAPI ServiceName = "arvados-api-server"
+ ServiceNameController ServiceName = "arvados-controller"
+ ServiceNameNodemanager ServiceName = "arvados-node-manager"
+ ServiceNameWorkbench ServiceName = "arvados-workbench"
+ ServiceNameWebsocket ServiceName = "arvados-ws"
+ ServiceNameKeepweb ServiceName = "keep-web"
+ ServiceNameKeepproxy ServiceName = "keepproxy"
+ ServiceNameKeepstore ServiceName = "keepstore"
+)
+
// ServicePorts returns the configured listening address (or "" if
// disabled) for each service on the node.
-func (sn *SystemNode) ServicePorts() map[string]string {
- return map[string]string{
- "arvados-api-server": sn.RailsAPI.Listen,
- "arvados-node-manager": sn.Nodemanager.Listen,
- "arvados-workbench": sn.Workbench.Listen,
- "arvados-ws": sn.Websocket.Listen,
- "keep-web": sn.Keepweb.Listen,
- "keepproxy": sn.Keepproxy.Listen,
- "keepstore": sn.Keepstore.Listen,
+func (np *NodeProfile) ServicePorts() map[ServiceName]string {
+ return map[ServiceName]string{
+ ServiceNameRailsAPI: np.RailsAPI.Listen,
+ ServiceNameController: np.Controller.Listen,
+ ServiceNameNodemanager: np.Nodemanager.Listen,
+ ServiceNameWorkbench: np.Workbench.Listen,
+ ServiceNameWebsocket: np.Websocket.Listen,
+ ServiceNameKeepweb: np.Keepweb.Listen,
+ ServiceNameKeepproxy: np.Keepproxy.Listen,
+ ServiceNameKeepstore: np.Keepstore.Listen,
}
}
type SystemServiceInstance struct {
- Listen string
+ Listen string
+ TLS bool
+ Insecure bool
}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "github.com/ghodss/yaml"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&ConfigSuite{})
+
+type ConfigSuite struct{}
+
+func (s *ConfigSuite) TestInstanceTypesAsArray(c *check.C) {
+ var cluster Cluster
+ yaml.Unmarshal([]byte("InstanceTypes:\n- Name: foo\n"), &cluster)
+ c.Check(len(cluster.InstanceTypes), check.Equals, 1)
+ c.Check(cluster.InstanceTypes["foo"].Name, check.Equals, "foo")
+}
+
+func (s *ConfigSuite) TestInstanceTypesAsHash(c *check.C) {
+ var cluster Cluster
+ yaml.Unmarshal([]byte("InstanceTypes:\n foo:\n ProviderType: bar\n"), &cluster)
+ c.Check(len(cluster.InstanceTypes), check.Equals, 1)
+ c.Check(cluster.InstanceTypes["foo"].Name, check.Equals, "foo")
+ c.Check(cluster.InstanceTypes["foo"].ProviderType, check.Equals, "bar")
+}
+
+func (s *ConfigSuite) TestInstanceTypeSize(c *check.C) {
+ var it InstanceType
+ err := yaml.Unmarshal([]byte("Name: foo\nScratch: 4GB\nRAM: 4GiB\n"), &it)
+ c.Check(err, check.IsNil)
+ c.Check(int64(it.Scratch), check.Equals, int64(4000000000))
+ c.Check(int64(it.RAM), check.Equals, int64(4294967296))
+}
// SchedulingParameters specify a container's scheduling parameters
// such as Partitions
type SchedulingParameters struct {
- Partitions []string `json:"partitions"`
+ Partitions []string `json:"partitions"`
+ Preemptible bool `json:"preemptible"`
+ MaxRunTime int `json:"max_run_time"`
}
// ContainerList is an arvados#containerList resource.
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import "strings"
+
+func (c PostgreSQLConnection) String() string {
+ s := ""
+ for k, v := range c {
+ s += strings.ToLower(k)
+ s += "='"
+ s += strings.Replace(
+ strings.Replace(v, `\`, `\\`, -1),
+ `'`, `\'`, -1)
+ s += "' "
+ }
+ return s
+}
FooCollectionSharingTokenUUID = "zzzzz-gj3su-gf02tdm4g1z3e3u"
FooCollectionSharingToken = "iknqgmunrhgsyfok8uzjlwun9iscwm3xacmzmg65fa1j1lpdss"
+
+ WorkflowWithDefinitionYAMLUUID = "zzzzz-7fd4e-validworkfloyml"
)
// PathologicalManifest : A valid manifest designed to test
// token.
var DecodeTokenCookie func(string) ([]byte, error) = base64.URLEncoding.DecodeString
-// LoadTokensFromHttpRequest loads all tokens it can find in the
+// LoadTokensFromHTTPRequest loads all tokens it can find in the
// headers and query string of an http query.
func (a *Credentials) LoadTokensFromHTTPRequest(r *http.Request) {
// Load plain token from "Authorization: OAuth2 ..." header
a.Tokens = append(a.Tokens, string(token))
}
-// TODO: LoadTokensFromHttpRequestBody(). We can't assume in
-// LoadTokensFromHttpRequest() that [or how] we should read and parse
-// the request body. This has to be requested explicitly by the
-// application.
+// LoadTokensFromHTTPRequestBody() loads credentials from the request
+// body.
+//
+// This is separate from LoadTokensFromHTTPRequest() because it's not
+// always desirable to read the request body. This has to be requested
+// explicitly by the application.
+func (a *Credentials) LoadTokensFromHTTPRequestBody(r *http.Request) error {
+ if r.Header.Get("Content-Type") != "application/x-www-form-urlencoded" {
+ return nil
+ }
+ if err := r.ParseForm(); err != nil {
+ return err
+ }
+ if t := r.PostFormValue("api_token"); t != "" {
+ a.Tokens = append(a.Tokens, t)
+ }
+ return nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package auth
+
+import (
+ "crypto/hmac"
+ "crypto/sha1"
+ "errors"
+ "fmt"
+ "io"
+ "regexp"
+ "strings"
+)
+
+var (
+ reObsoleteToken = regexp.MustCompile(`^[0-9a-z]{41,}$`)
+ ErrObsoleteToken = errors.New("obsolete token format")
+ ErrTokenFormat = errors.New("badly formatted token")
+ ErrSalted = errors.New("token already salted")
+)
+
+func SaltToken(token, remote string) (string, error) {
+ parts := strings.Split(token, "/")
+ if len(parts) < 3 || parts[0] != "v2" {
+ if reObsoleteToken.MatchString(token) {
+ return "", ErrObsoleteToken
+ } else {
+ return "", ErrTokenFormat
+ }
+ }
+ uuid := parts[1]
+ secret := parts[2]
+ if len(secret) != 40 {
+ // not already salted
+ hmac := hmac.New(sha1.New, []byte(secret))
+ io.WriteString(hmac, remote)
+ secret = fmt.Sprintf("%x", hmac.Sum(nil))
+ return "v2/" + uuid + "/" + secret, nil
+ } else if strings.HasPrefix(uuid, remote) {
+ // already salted for the desired remote
+ return token, nil
+ } else {
+ // salted for a different remote, can't be used
+ return "", ErrSalted
+ }
+}
params := arvadosclient.Dict{
"filters": filters,
"order": []string{"priority desc"}}
-
- var list arvados.ContainerList
- for offset, more := 0, true; more; offset += len(list.Items) {
+ offset := 0
+ for {
params["offset"] = offset
+
+ // This list variable must be a new one declared
+ // inside the loop: otherwise, items in the API
+ // response would get deep-merged into the items
+ // loaded in previous iterations.
+ var list arvados.ContainerList
+
err := d.Arv.List("containers", params, &list)
if err != nil {
log.Printf("Error getting list of containers: %q", err)
return false
}
- more = len(list.Items) > 0 && list.ItemsAvailable > len(list.Items)+offset
d.checkListForUpdates(list.Items, todo)
+ offset += len(list.Items)
+ if len(list.Items) == 0 || list.ItemsAvailable <= offset {
+ return true
+ }
}
- return true
}
func (d *Dispatcher) checkListForUpdates(containers []arvados.Container, todo map[string]*runTracker) {
// exposes problems that can't be expressed in Checks, like
// "service S is needed, but isn't configured to run
// anywhere."
- Services map[string]ServiceHealth `json:"services"`
+ Services map[arvados.ServiceName]ServiceHealth `json:"services"`
}
type CheckResult struct {
resp := ClusterHealthResponse{
Health: "OK",
Checks: make(map[string]CheckResult),
- Services: make(map[string]ServiceHealth),
+ Services: make(map[arvados.ServiceName]ServiceHealth),
}
mtx := sync.Mutex{}
wg := sync.WaitGroup{}
- for node, nodeConfig := range cluster.SystemNodes {
- for svc, addr := range nodeConfig.ServicePorts() {
+ for profileName, profile := range cluster.NodeProfiles {
+ for svc, addr := range profile.ServicePorts() {
// Ensure svc is listed in resp.Services.
mtx.Lock()
if _, ok := resp.Services[svc]; !ok {
}
wg.Add(1)
- go func(node, svc, addr string) {
+ go func(profileName string, svc arvados.ServiceName, addr string) {
defer wg.Done()
var result CheckResult
- url, err := agg.pingURL(node, addr)
+ url, err := agg.pingURL(profileName, addr)
if err != nil {
result = CheckResult{
Health: "ERROR",
mtx.Lock()
defer mtx.Unlock()
- resp.Checks[svc+"+"+url] = result
+ resp.Checks[fmt.Sprintf("%s+%s", svc, url)] = result
if result.Health == "OK" {
h := resp.Services[svc]
h.N++
} else {
resp.Health = "ERROR"
}
- }(node, svc, addr)
+ }(profileName, svc, addr)
}
}
wg.Wait()
Clusters: map[string]arvados.Cluster{
"zzzzz": {
ManagementToken: arvadostest.ManagementToken,
- SystemNodes: map[string]arvados.SystemNode{},
+ NodeProfiles: map[string]arvados.NodeProfile{},
},
},
}}
func (s *AggregatorSuite) TestUnhealthy(c *check.C) {
srv, listen := s.stubServer(&unhealthyHandler{})
defer srv.Close()
- s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
+ s.handler.Config.Clusters["zzzzz"].NodeProfiles["localhost"] = arvados.NodeProfile{
Keepstore: arvados.SystemServiceInstance{Listen: listen},
}
s.handler.ServeHTTP(s.resp, s.req)
func (s *AggregatorSuite) TestHealthy(c *check.C) {
srv, listen := s.stubServer(&healthyHandler{})
defer srv.Close()
- s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
+ s.handler.Config.Clusters["zzzzz"].NodeProfiles["localhost"] = arvados.NodeProfile{
+ Controller: arvados.SystemServiceInstance{Listen: listen},
Keepproxy: arvados.SystemServiceInstance{Listen: listen},
Keepstore: arvados.SystemServiceInstance{Listen: listen},
Keepweb: arvados.SystemServiceInstance{Listen: listen},
defer srvH.Close()
srvU, listenU := s.stubServer(&unhealthyHandler{})
defer srvU.Close()
- s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
+ s.handler.Config.Clusters["zzzzz"].NodeProfiles["localhost"] = arvados.NodeProfile{
+ Controller: arvados.SystemServiceInstance{Listen: listenH},
Keepproxy: arvados.SystemServiceInstance{Listen: listenH},
Keepstore: arvados.SystemServiceInstance{Listen: listenH},
Keepweb: arvados.SystemServiceInstance{Listen: listenH},
Websocket: arvados.SystemServiceInstance{Listen: listenH},
Workbench: arvados.SystemServiceInstance{Listen: listenH},
}
- s.handler.Config.Clusters["zzzzz"].SystemNodes["127.0.0.1"] = arvados.SystemNode{
+ s.handler.Config.Clusters["zzzzz"].NodeProfiles["127.0.0.1"] = arvados.NodeProfile{
Keepstore: arvados.SystemServiceInstance{Listen: listenU},
}
s.handler.ServeHTTP(s.resp, s.req)
s.handler.timeout = arvados.Duration(100 * time.Millisecond)
srv, listen := s.stubServer(&slowHandler{})
defer srv.Close()
- s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
+ s.handler.Config.Clusters["zzzzz"].NodeProfiles["localhost"] = arvados.NodeProfile{
Keepstore: arvados.SystemServiceInstance{Listen: listen},
}
s.handler.ServeHTTP(s.resp, s.req)
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package httpserver
+
+import (
+ "encoding/json"
+ "net/http"
+)
+
+type ErrorResponse struct {
+ Errors []string `json:"errors"`
+}
+
+func Error(w http.ResponseWriter, error string, code int) {
+ w.Header().Set("Content-Type", "application/json")
+ w.Header().Set("X-Content-Type-Options", "nosniff")
+ w.WriteHeader(code)
+ json.NewEncoder(w).Encode(ErrorResponse{Errors: []string{error}})
+}
name string
}
-var requestTimeContextKey = contextKey{"requestTime"}
-
-var Logger logrus.FieldLogger = logrus.StandardLogger()
+var (
+ requestTimeContextKey = contextKey{"requestTime"}
+ loggerContextKey = contextKey{"logger"}
+)
// LogRequests wraps an http.Handler, logging each request and
-// response via logrus.
-func LogRequests(h http.Handler) http.Handler {
+// response via logger.
+func LogRequests(logger logrus.FieldLogger, h http.Handler) http.Handler {
+ if logger == nil {
+ logger = logrus.StandardLogger()
+ }
return http.HandlerFunc(func(wrapped http.ResponseWriter, req *http.Request) {
w := &responseTimer{ResponseWriter: WrapResponseWriter(wrapped)}
- req = req.WithContext(context.WithValue(req.Context(), &requestTimeContextKey, time.Now()))
- lgr := Logger.WithFields(logrus.Fields{
+ lgr := logger.WithFields(logrus.Fields{
"RequestID": req.Header.Get("X-Request-Id"),
"remoteAddr": req.RemoteAddr,
"reqForwardedFor": req.Header.Get("X-Forwarded-For"),
"reqQuery": req.URL.RawQuery,
"reqBytes": req.ContentLength,
})
+ ctx := req.Context()
+ ctx = context.WithValue(ctx, &requestTimeContextKey, time.Now())
+ ctx = context.WithValue(ctx, &loggerContextKey, lgr)
+ req = req.WithContext(ctx)
+
logRequest(w, req, lgr)
defer logResponse(w, req, lgr)
h.ServeHTTP(w, req)
})
}
+func Logger(req *http.Request) logrus.FieldLogger {
+ if lgr, ok := req.Context().Value(&loggerContextKey).(logrus.FieldLogger); ok {
+ return lgr
+ } else {
+ return logrus.StandardLogger()
+ }
+}
+
func logRequest(w *responseTimer, req *http.Request, lgr *logrus.Entry) {
lgr.Info("request")
}
"encoding/json"
"net/http"
"net/http/httptest"
- "os"
"testing"
"time"
- log "github.com/Sirupsen/logrus"
+ "github.com/Sirupsen/logrus"
check "gopkg.in/check.v1"
)
type Suite struct{}
func (s *Suite) TestLogRequests(c *check.C) {
- defer log.SetOutput(os.Stdout)
captured := &bytes.Buffer{}
- log.SetOutput(captured)
- log.SetFormatter(&log.JSONFormatter{
+ log := logrus.New()
+ log.Out = captured
+ log.Formatter = &logrus.JSONFormatter{
TimestampFormat: time.RFC3339Nano,
- })
+ }
+
h := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
w.Write([]byte("hello world"))
})
req.Header.Set("X-Forwarded-For", "1.2.3.4:12345")
c.Assert(err, check.IsNil)
resp := httptest.NewRecorder()
- AddRequestIDs(LogRequests(h)).ServeHTTP(resp, req)
+ AddRequestIDs(LogRequests(log, h)).ServeHTTP(resp, req)
dec := json.NewDecoder(captured)
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package httpserver
+
+import (
+ "net/http"
+ "strconv"
+ "strings"
+ "time"
+
+ "git.curoverse.com/arvados.git/sdk/go/stats"
+ "github.com/Sirupsen/logrus"
+ "github.com/gogo/protobuf/jsonpb"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+type Handler interface {
+ http.Handler
+
+ // Returns an http.Handler that serves the Handler's metrics
+ // data at /metrics and /metrics.json, and passes other
+ // requests through to next.
+ ServeAPI(next http.Handler) http.Handler
+}
+
+type metrics struct {
+ next http.Handler
+ logger *logrus.Logger
+ registry *prometheus.Registry
+ reqDuration *prometheus.SummaryVec
+ timeToStatus *prometheus.SummaryVec
+ exportProm http.Handler
+}
+
+func (*metrics) Levels() []logrus.Level {
+ return logrus.AllLevels
+}
+
+// Fire implements logrus.Hook in order to collect data points from
+// request logs.
+func (m *metrics) Fire(ent *logrus.Entry) error {
+ if tts, ok := ent.Data["timeToStatus"].(stats.Duration); !ok {
+ } else if method, ok := ent.Data["reqMethod"].(string); !ok {
+ } else if code, ok := ent.Data["respStatusCode"].(int); !ok {
+ } else {
+ m.timeToStatus.WithLabelValues(strconv.Itoa(code), strings.ToLower(method)).Observe(time.Duration(tts).Seconds())
+ }
+ return nil
+}
+
+func (m *metrics) exportJSON(w http.ResponseWriter, req *http.Request) {
+ jm := jsonpb.Marshaler{Indent: " "}
+ mfs, _ := m.registry.Gather()
+ w.Write([]byte{'['})
+ for i, mf := range mfs {
+ if i > 0 {
+ w.Write([]byte{','})
+ }
+ jm.Marshal(w, mf)
+ }
+ w.Write([]byte{']'})
+}
+
+// ServeHTTP implements http.Handler.
+func (m *metrics) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+ m.next.ServeHTTP(w, req)
+}
+
+// ServeAPI returns a new http.Handler that serves current data at
+// metrics API endpoints (currently "GET /metrics(.json)?") and passes
+// other requests through to next.
+//
+// Typical example:
+//
+// m := Instrument(...)
+// srv := http.Server{Handler: m.ServeAPI(m)}
+func (m *metrics) ServeAPI(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ switch {
+ case req.Method != "GET" && req.Method != "HEAD":
+ next.ServeHTTP(w, req)
+ case req.URL.Path == "/metrics.json":
+ m.exportJSON(w, req)
+ case req.URL.Path == "/metrics":
+ m.exportProm.ServeHTTP(w, req)
+ default:
+ next.ServeHTTP(w, req)
+ }
+ })
+}
+
+// Instrument returns a new Handler that passes requests through to
+// the next handler in the stack, and tracks metrics of those
+// requests.
+//
+// For the metrics to be accurate, the caller must ensure every
+// request passed to the Handler also passes through
+// LogRequests(logger, ...), and vice versa.
+//
+// If registry is nil, a new registry is created.
+//
+// If logger is nil, logrus.StandardLogger() is used.
+func Instrument(registry *prometheus.Registry, logger *logrus.Logger, next http.Handler) Handler {
+ if logger == nil {
+ logger = logrus.StandardLogger()
+ }
+ if registry == nil {
+ registry = prometheus.NewRegistry()
+ }
+ reqDuration := prometheus.NewSummaryVec(prometheus.SummaryOpts{
+ Name: "request_duration_seconds",
+ Help: "Summary of request duration.",
+ }, []string{"code", "method"})
+ timeToStatus := prometheus.NewSummaryVec(prometheus.SummaryOpts{
+ Name: "time_to_status_seconds",
+ Help: "Summary of request TTFB.",
+ }, []string{"code", "method"})
+ registry.MustRegister(timeToStatus)
+ registry.MustRegister(reqDuration)
+ m := &metrics{
+ next: promhttp.InstrumentHandlerDuration(reqDuration, next),
+ logger: logger,
+ registry: registry,
+ reqDuration: reqDuration,
+ timeToStatus: timeToStatus,
+ exportProm: promhttp.HandlerFor(registry, promhttp.HandlerOpts{
+ ErrorLog: logger,
+ }),
+ }
+ m.logger.AddHook(m)
+ return m
+}
Retries int
BlockCache *BlockCache
RequestID string
+ StorageClasses []string
// set to 1 if all writable services are of disk type, otherwise 0
replicasPerService int
}
type StubPutHandler struct {
- c *C
- expectPath string
- expectApiToken string
- expectBody string
- handled chan string
+ c *C
+ expectPath string
+ expectApiToken string
+ expectBody string
+ expectStorageClass string
+ handled chan string
}
func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
+ sph.c.Check(req.Header.Get("X-Keep-Storage-Classes"), Equals, sph.expectStorageClass)
body, err := ioutil.ReadAll(req.Body)
sph.c.Check(err, Equals, nil)
sph.c.Check(body, DeepEquals, []byte(sph.expectBody))
"acbd18db4cc2f85cedef654fccc4a4d8",
"abc123",
"foo",
+ "hot",
make(chan string)}
UploadToStubHelper(c, st,
func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
-
+ kc.StorageClasses = []string{"hot"}
go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), kc.getRequestID())
writer.Write([]byte("foo"))
"acbd18db4cc2f85cedef654fccc4a4d8",
"abc123",
"foo",
+ "",
make(chan string)}
UploadToStubHelper(c, st,
hash,
"abc123",
"foo",
+ "",
make(chan string, 5)}
arv, _ := arvadosclient.MakeArvadosClient()
hash,
"abc123",
"foo",
+ "",
make(chan string, 5)}
arv, _ := arvadosclient.MakeArvadosClient()
hash,
"abc123",
"foo",
+ "",
make(chan string, 4)}
fh := FailHandler{
hash,
"abc123",
"foo",
+ "",
make(chan string, 1)}
fh := FailHandler{
hash,
"abc123",
"foo",
+ "",
make(chan string, 5)}
arv, _ := arvadosclient.MakeArvadosClient()
hash,
"abc123",
"foo",
+ "",
make(chan string, 5)}
arv, _ := arvadosclient.MakeArvadosClient()
Md5String("foo"),
"abc123",
"foo",
+ "",
make(chan string, 5)}}
arv, _ := arvadosclient.MakeArvadosClient()
req.Header.Add("Authorization", "OAuth2 "+this.Arvados.ApiToken)
req.Header.Add("Content-Type", "application/octet-stream")
req.Header.Add(X_Keep_Desired_Replicas, fmt.Sprint(this.Want_replicas))
+ if len(this.StorageClasses) > 0 {
+ req.Header.Add("X-Keep-Storage-Classes", strings.Join(this.StorageClasses, ", "))
+ }
var resp *http.Response
if resp, err = this.httpClient().Do(req); err != nil {
import pprint
import re
import string
-import subprocess
import sys
-import threading
import time
import types
import zlib
delay, exc_info=True)
for conn in self.connections.values():
conn.close()
+ except httplib2.SSLHandshakeError as e:
+ # Intercept and re-raise with a better error message.
+ raise httplib2.SSLHandshakeError("Could not connect to %s\n%s\nPossible causes: remote SSL/TLS certificate expired, or was issued by an untrusted certificate authority." % (uri, e))
+
time.sleep(delay)
delay = delay * self._retry_delay_backoff
if apiconfig is None:
apiconfig = config.settings()
+ errors = []
for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
if x not in apiconfig:
- raise ValueError("%s is not set. Aborting." % x)
+ errors.append(x)
+ if errors:
+ raise ValueError(" and ".join(errors)+" not set.\nPlease set in %s or export environment variable." % config.default_config_file)
host = apiconfig.get('ARVADOS_API_HOST')
token = apiconfig.get('ARVADOS_API_TOKEN')
insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE', apiconfig)
import re
import errno
import hashlib
+import datetime
+import ciso8601
import time
import threading
def root_collection(self):
return self
+ def get_properties(self):
+ if self._api_response and self._api_response["properties"]:
+ return self._api_response["properties"]
+ else:
+ return {}
+
+ def get_trash_at(self):
+ if self._api_response and self._api_response["trash_at"]:
+ return ciso8601.parse_datetime(self._api_response["trash_at"])
+ else:
+ return None
+
def stream_name(self):
return "."
@must_be_writable
@synchronized
@retry_method
- def save(self, storage_classes=None, merge=True, num_retries=None):
+ def save(self,
+ properties=None,
+ storage_classes=None,
+ trash_at=None,
+ merge=True,
+ num_retries=None):
"""Save collection to an existing collection record.
Commit pending buffer blocks to Keep, merge with remote record (if
- merge=True, the default), and update the collection record. Returns
+ merge=True, the default), and update the collection record. Returns
the current manifest text.
Will raise AssertionError if not associated with a collection record on
the API server. If you want to save a manifest to Keep only, see
`save_new()`.
+ :properties:
+ Additional properties of collection. This value will replace any existing
+ properties of collection.
+
:storage_classes:
Specify desirable storage classes to be used when writing data to Keep.
+ :trash_at:
+ A collection is *expiring* when it has a *trash_at* time in the future.
+ An expiring collection can be accessed as normal,
+ but is scheduled to be trashed automatically at the *trash_at* time.
+
:merge:
Update and merge remote changes before saving. Otherwise, any
remote changes will be ignored and overwritten.
Retry count on API calls (if None, use the collection default)
"""
+ if properties and type(properties) is not dict:
+ raise errors.ArgumentError("properties must be dictionary type.")
+
if storage_classes and type(storage_classes) is not list:
raise errors.ArgumentError("storage_classes must be list type.")
+ if trash_at and type(trash_at) is not datetime.datetime:
+ raise errors.ArgumentError("trash_at must be datetime type.")
+
+ body={}
+ if properties:
+ body["properties"] = properties
+ if storage_classes:
+ body["storage_classes_desired"] = storage_classes
+ if trash_at:
+ t = trash_at.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+ body["trash_at"] = t
+
if not self.committed():
if not self._has_collection_uuid():
raise AssertionError("Collection manifest_locator is not a collection uuid. Use save_new() for new collections.")
self.update()
text = self.manifest_text(strip=False)
- body={'manifest_text': text}
- if storage_classes:
- body["storage_classes_desired"] = storage_classes
+ body['manifest_text'] = text
self._remember_api_response(self._my_api().collections().update(
uuid=self._manifest_locator,
body=body
- ).execute(
- num_retries=num_retries))
+ ).execute(num_retries=num_retries))
self._manifest_text = self._api_response["manifest_text"]
self._portable_data_hash = self._api_response["portable_data_hash"]
self.set_committed(True)
- elif storage_classes:
+ elif body:
self._remember_api_response(self._my_api().collections().update(
uuid=self._manifest_locator,
- body={"storage_classes_desired": storage_classes}
- ).execute(
- num_retries=num_retries))
+ body=body
+ ).execute(num_retries=num_retries))
return self._manifest_text
def save_new(self, name=None,
create_collection_record=True,
owner_uuid=None,
+ properties=None,
storage_classes=None,
+ trash_at=None,
ensure_unique_name=False,
num_retries=None):
"""Save collection to a new collection record.
Commit pending buffer blocks to Keep and, when create_collection_record
is True (default), create a new collection record. After creating a
new collection record, this Collection object will be associated with
- the new record used by `save()`. Returns the current manifest text.
+ the new record used by `save()`. Returns the current manifest text.
:name:
The collection name.
the user, or project uuid that will own this collection.
If None, defaults to the current user.
+ :properties:
+ Additional properties of collection. This value will replace any existing
+ properties of collection.
+
:storage_classes:
Specify desirable storage classes to be used when writing data to Keep.
+ :trash_at:
+ A collection is *expiring* when it has a *trash_at* time in the future.
+ An expiring collection can be accessed as normal,
+ but is scheduled to be trashed automatically at the *trash_at* time.
+
:ensure_unique_name:
If True, ask the API server to rename the collection
if it conflicts with a collection with the same name and owner. If
Retry count on API calls (if None, use the collection default)
"""
+ if properties and type(properties) is not dict:
+ raise errors.ArgumentError("properties must be dictionary type.")
+
+ if storage_classes and type(storage_classes) is not list:
+ raise errors.ArgumentError("storage_classes must be list type.")
+
+ if trash_at and type(trash_at) is not datetime.datetime:
+ raise errors.ArgumentError("trash_at must be datetime type.")
+
self._my_block_manager().commit_all()
text = self.manifest_text(strip=False)
"replication_desired": self.replication_desired}
if owner_uuid:
body["owner_uuid"] = owner_uuid
+ if properties:
+ body["properties"] = properties
if storage_classes:
- if type(storage_classes) is not list:
- raise errors.ArgumentError("storage_classes must be list type.")
body["storage_classes_desired"] = storage_classes
+ if trash_at:
+ t = trash_at.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+ body["trash_at"] = t
self._remember_api_response(self._my_api().collections().create(ensure_unique_name=ensure_unique_name, body=body).execute(num_retries=num_retries))
text = self._api_response["manifest_text"]
import json
import os
import re
-import subprocess
+import subprocess32 as subprocess
import sys
import tarfile
import tempfile
return prefix+fn
-def write_file(collection, pathprefix, fn):
+def write_file(collection, pathprefix, fn, flush=False):
with open(os.path.join(pathprefix, fn)) as src:
dst = collection.open(fn, "w")
r = src.read(1024*128)
while r:
dst.write(r)
r = src.read(1024*128)
- dst.close(flush=False)
+ dst.close(flush=flush)
def uploadfiles(files, api, dry_run=False, num_retries=0,
project=None,
fnPattern="$(file %s/%s)",
name=None,
- collection=None):
+ collection=None,
+ packed=True):
# Find the smallest path prefix that includes all the files that need to be uploaded.
# This starts at the root and iteratively removes common parent directory prefixes
# until all file paths no longer have a common parent.
continue
prev = localpath
if os.path.isfile(localpath):
- write_file(collection, pathprefix, f.fn)
+ write_file(collection, pathprefix, f.fn, not packed)
elif os.path.isdir(localpath):
for root, dirs, iterfiles in os.walk(localpath):
root = root[len(pathprefix):]
for src in iterfiles:
- write_file(collection, pathprefix, os.path.join(root, src))
+ write_file(collection, pathprefix, os.path.join(root, src), not packed)
filters=[["portable_data_hash", "=", collection.portable_data_hash()]]
if name:
def __init__(self, root, user_agent_pool=queue.LifoQueue(),
upload_counter=None,
download_counter=None,
- headers={}):
+ headers={},
+ insecure=False):
self.root = root
self._user_agent_pool = user_agent_pool
self._result = {'error': None}
self.put_headers = headers
self.upload_counter = upload_counter
self.download_counter = download_counter
+ self.insecure = insecure
def usable(self):
"""Is it worth attempting a request?"""
'{}: {}'.format(k,v) for k,v in self.get_headers.items()])
curl.setopt(pycurl.WRITEFUNCTION, response_body.write)
curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
+ if self.insecure:
+ curl.setopt(pycurl.SSL_VERIFYPEER, 0)
if method == "HEAD":
curl.setopt(pycurl.NOBODY, True)
self._setcurltimeouts(curl, timeout)
'{}: {}'.format(k,v) for k,v in self.put_headers.items()])
curl.setopt(pycurl.WRITEFUNCTION, response_body.write)
curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
+ if self.insecure:
+ curl.setopt(pycurl.SSL_VERIFYPEER, 0)
self._setcurltimeouts(curl, timeout)
try:
curl.perform()
if local_store is None:
local_store = os.environ.get('KEEP_LOCAL_STORE')
+ if api_client is None:
+ self.insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE')
+ else:
+ self.insecure = api_client.insecure
+
self.block_cache = block_cache if block_cache else KeepBlockCache()
self.timeout = timeout
self.proxy_timeout = proxy_timeout
root, self._user_agent_pool,
upload_counter=self.upload_counter,
download_counter=self.download_counter,
- headers=headers)
+ headers=headers,
+ insecure=self.insecure)
return local_roots
@staticmethod
root: self.KeepService(root, self._user_agent_pool,
upload_counter=self.upload_counter,
download_counter=self.download_counter,
- headers=headers)
+ headers=headers,
+ insecure=self.insecure)
for root in hint_roots
}
self.apiconfig = copy.copy(apiconfig)
self.api_params = api_params
self.local = threading.local()
+
+ # Initialize an API object for this thread before creating
+ # KeepClient, this will report if ARVADOS_API_HOST or
+ # ARVADOS_API_TOKEN are missing.
+ self.localapi()
+
self.keep = keep.KeepClient(api_client=self, **keep_params)
def localapi(self):
('share/doc/arvados-python-client', ['LICENSE-2.0.txt', 'README.rst']),
],
install_requires=[
- 'ciso8601 >=1.0.6',
+ 'ciso8601 >=1.0.6, <2.0.0',
'future',
'google-api-python-client >=1.6.2, <1.7',
'httplib2 >=0.9.2',
'ruamel.yaml >=0.13.11, <0.15',
'setuptools',
'ws4py <0.4',
+ 'subprocess32>=3.5.1',
],
test_suite='tests',
tests_require=['pbr<1.7.0', 'mock>=1.0', 'PyYAML'],
# SPDX-License-Identifier: Apache-2.0
daemon off;
-error_log stderr info; # Yes, must be specified here _and_ cmdline
+error_log "{{ERRORLOG}}" info; # Yes, must be specified here _and_ cmdline
events {
}
http {
- access_log {{ACCESSLOG}} combined;
+ log_format customlog
+ '[$time_local] $server_name $status $body_bytes_sent $request_time $request_method "$scheme://$http_host$request_uri" $remote_addr:$remote_port '
+ '"$http_referer" "$http_user_agent"';
+ access_log "{{ACCESSLOG}}" customlog;
+ client_body_temp_path "{{TMPDIR}}";
upstream arv-git-http {
server localhost:{{GITPORT}};
}
server {
listen *:{{GITSSLPORT}} ssl default_server;
- server_name _;
- ssl_certificate {{SSLCERT}};
- ssl_certificate_key {{SSLKEY}};
+ server_name arv-git-http;
+ ssl_certificate "{{SSLCERT}}";
+ ssl_certificate_key "{{SSLKEY}}";
location / {
proxy_pass http://arv-git-http;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
}
}
upstream keepproxy {
}
server {
listen *:{{KEEPPROXYSSLPORT}} ssl default_server;
- server_name _;
- ssl_certificate {{SSLCERT}};
- ssl_certificate_key {{SSLKEY}};
+ server_name keepproxy;
+ ssl_certificate "{{SSLCERT}}";
+ ssl_certificate_key "{{SSLKEY}}";
location / {
proxy_pass http://keepproxy;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
}
}
upstream keep-web {
}
server {
listen *:{{KEEPWEBSSLPORT}} ssl default_server;
- server_name ~^(?<request_host>.*)$;
- ssl_certificate {{SSLCERT}};
- ssl_certificate_key {{SSLKEY}};
+ server_name keep-web;
+ ssl_certificate "{{SSLCERT}}";
+ ssl_certificate_key "{{SSLKEY}}";
location / {
proxy_pass http://keep-web;
- proxy_set_header Host $request_host:{{KEEPWEBPORT}};
+ proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
}
}
server {
listen *:{{KEEPWEBDLSSLPORT}} ssl default_server;
- server_name ~.*;
- ssl_certificate {{SSLCERT}};
- ssl_certificate_key {{SSLKEY}};
+ server_name keep-web-dl ~.*;
+ ssl_certificate "{{SSLCERT}}";
+ ssl_certificate_key "{{SSLKEY}}";
location / {
proxy_pass http://keep-web;
- proxy_set_header Host download:{{KEEPWEBPORT}};
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_redirect //download:{{KEEPWEBPORT}}/ https://$host:{{KEEPWEBDLSSLPORT}}/;
+ proxy_set_header X-Forwarded-Proto https;
+
+ # Unlike other proxy sections, here we need to override the
+ # requested Host header and use proxy_redirect because of the
+ # way the test suite orchestrates services. Keep-web's "download
+ # only" behavior relies on the Host header matching a configured
+ # value, but when run_test_servers.py writes keep-web's command
+ # line, the keep-web-dl TLS port (which clients will connect to
+ # and include in their Host header) has not yet been assigned.
+ #
+ # In production, "proxy_set_header Host $http_host;
+ # proxy_redirect off;" works: keep-web's redirect URLs will
+ # match the request URL received by Nginx.
+ #
+ # Here, keep-web will issue redirects to https://download/ and
+ # Nginx will rewrite them.
+ #
+ proxy_set_header Host download;
+ proxy_redirect https://download/ https://$host:{{KEEPWEBDLSSLPORT}}/;
}
}
upstream ws {
}
server {
listen *:{{WSSPORT}} ssl default_server;
- server_name ~^(?<request_host>.*)$;
- ssl_certificate {{SSLCERT}};
- ssl_certificate_key {{SSLKEY}};
+ server_name websocket;
+ ssl_certificate "{{SSLCERT}}";
+ ssl_certificate_key "{{SSLKEY}}";
location / {
proxy_pass http://ws;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
- proxy_set_header Host $request_host:{{WSPORT}};
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
+ }
+ }
+ upstream controller {
+ server localhost:{{CONTROLLERPORT}};
+ }
+ server {
+ listen *:{{CONTROLLERSSLPORT}} ssl default_server;
+ server_name controller;
+ ssl_certificate "{{SSLCERT}}";
+ ssl_certificate_key "{{SSLKEY}}";
+ location / {
+ proxy_pass http://controller;
+ proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
}
}
}
sock.close()
return port
-def _wait_until_port_listens(port, timeout=10):
+def _wait_until_port_listens(port, timeout=10, warn=True):
"""Wait for a process to start listening on the given port.
If nothing listens on the port within the specified timeout (given
except subprocess.CalledProcessError:
time.sleep(0.1)
continue
- return
- print(
- "WARNING: Nothing is listening on port {} (waited {} seconds).".
- format(port, timeout),
- file=sys.stderr)
+ return True
+ if warn:
+ print(
+ "WARNING: Nothing is listening on port {} (waited {} seconds).".
+ format(port, timeout),
+ file=sys.stderr)
+ return False
+
+def _logfilename(label):
+ """Set up a labelled log file, and return a path to write logs to.
-def _fifo2stderr(label):
- """Create a fifo, and copy it to stderr, prepending label to each line.
+ Normally, the returned path is {tmpdir}/{label}.log.
- Return value is the path to the new FIFO.
+ In debug mode, logs are also written to stderr, with [label]
+ prepended to each line. The returned path is a FIFO.
+label+ should contain only alphanumerics: it is also used as part
of the FIFO filename.
+
"""
+ logfilename = os.path.join(TEST_TMPDIR, label+'.log')
+ if not os.environ.get('ARVADOS_DEBUG', ''):
+ return logfilename
fifo = os.path.join(TEST_TMPDIR, label+'.fifo')
try:
os.remove(fifo)
if error.errno != errno.ENOENT:
raise
os.mkfifo(fifo, 0o700)
+ stdbuf = ['stdbuf', '-i0', '-oL', '-eL']
+ # open(fifo, 'r') would block waiting for someone to open the fifo
+ # for writing, so we need a separate cat process to open it for
+ # us.
+ cat = subprocess.Popen(
+ stdbuf+['cat', fifo],
+ stdin=open('/dev/null'),
+ stdout=subprocess.PIPE)
+ tee = subprocess.Popen(
+ stdbuf+['tee', '-a', logfilename],
+ stdin=cat.stdout,
+ stdout=subprocess.PIPE)
subprocess.Popen(
- ['stdbuf', '-i0', '-oL', '-eL', 'sed', '-e', 's/^/['+label+'] /', fifo],
+ stdbuf+['sed', '-e', 's/^/['+label+'] /'],
+ stdin=tee.stdout,
stdout=sys.stderr)
return fifo
'POST',
headers={'Authorization': 'OAuth2 {}'.format(token)})
os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
- os.environ['ARVADOS_API_HOST'] = existing_api_host
os.environ['ARVADOS_API_TOKEN'] = token
+ if _wait_until_port_listens(_getport('controller-ssl'), timeout=0.5, warn=False):
+ os.environ['ARVADOS_API_HOST'] = '0.0.0.0:'+str(_getport('controller-ssl'))
+ else:
+ os.environ['ARVADOS_API_HOST'] = existing_api_host
def stop(force=False):
"""Stop the API server, if one is running.
kill_server_pid(_pidfile('api'))
my_api_host = None
+def run_controller():
+ if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ return
+ stop_controller()
+ rails_api_port = int(string.split(os.environ.get('ARVADOS_TEST_API_HOST', my_api_host), ':')[-1])
+ port = find_available_port()
+ conf = os.path.join(TEST_TMPDIR, 'arvados.yml')
+ with open(conf, 'w') as f:
+ f.write("""
+Clusters:
+ zzzzz:
+ PostgreSQL:
+ ConnectionPool: 32
+ Connection:
+ host: {}
+ dbname: {}
+ user: {}
+ password: {}
+ NodeProfiles:
+ "*":
+ "arvados-controller":
+ Listen: ":{}"
+ "arvados-api-server":
+ Listen: ":{}"
+ TLS: true
+ Insecure: true
+ """.format(
+ _dbconfig('host'),
+ _dbconfig('database'),
+ _dbconfig('username'),
+ _dbconfig('password'),
+ port,
+ rails_api_port,
+ ))
+ logf = open(_logfilename('controller'), 'a')
+ controller = subprocess.Popen(
+ ["arvados-server", "controller", "-config", conf],
+ stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+ with open(_pidfile('controller'), 'w') as f:
+ f.write(str(controller.pid))
+ _wait_until_port_listens(port)
+ _setport('controller', port)
+ return port
+
+def stop_controller():
+ if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ return
+ kill_server_pid(_pidfile('controller'))
+
def run_ws():
if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
return
_dbconfig('database'),
_dbconfig('username'),
_dbconfig('password')))
- logf = open(_fifo2stderr('ws'), 'w')
+ logf = open(_logfilename('ws'), 'a')
ws = subprocess.Popen(
["ws", "-config", conf],
stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
for arg, val in keep_args.items():
keep_cmd.append("{}={}".format(arg, val))
- logf = open(_fifo2stderr('keep{}'.format(n)), 'w')
+ logf = open(_logfilename('keep{}'.format(n)), 'a')
kp0 = subprocess.Popen(
keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
port = find_available_port()
env = os.environ.copy()
env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
- logf = open(_fifo2stderr('keepproxy'), 'w')
+ logf = open(_logfilename('keepproxy'), 'a')
kp = subprocess.Popen(
['keepproxy',
'-pid='+_pidfile('keepproxy'),
gitport = find_available_port()
env = os.environ.copy()
env.pop('ARVADOS_API_TOKEN', None)
- logf = open(_fifo2stderr('arv-git-httpd'), 'w')
+ logf = open(_logfilename('arv-git-httpd'), 'a')
agh = subprocess.Popen(
['arv-git-httpd',
'-repo-root='+gitdir+'/test',
keepwebport = find_available_port()
env = os.environ.copy()
env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
- logf = open(_fifo2stderr('keep-web'), 'w')
+ logf = open(_logfilename('keep-web'), 'a')
keepweb = subprocess.Popen(
['keep-web',
'-allow-anonymous',
- '-attachment-only-host=download:'+str(keepwebport),
+ '-attachment-only-host=download',
'-listen=:'+str(keepwebport)],
env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
with open(_pidfile('keep-web'), 'w') as f:
return
stop_nginx()
nginxconf = {}
+ nginxconf['CONTROLLERPORT'] = _getport('controller')
+ nginxconf['CONTROLLERSSLPORT'] = find_available_port()
nginxconf['KEEPWEBPORT'] = _getport('keep-web')
nginxconf['KEEPWEBDLSSLPORT'] = find_available_port()
nginxconf['KEEPWEBSSLPORT'] = find_available_port()
nginxconf['WSSPORT'] = _getport('wss')
nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
- nginxconf['ACCESSLOG'] = _fifo2stderr('nginx_access_log')
+ nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
+ nginxconf['ERRORLOG'] = _logfilename('nginx_error')
+ nginxconf['TMPDIR'] = TEST_TMPDIR
conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
'-g', 'pid '+_pidfile('nginx')+';',
'-c', conffile],
env=env, stdin=open('/dev/null'), stdout=sys.stderr)
+ _setport('controller-ssl', nginxconf['CONTROLLERSSLPORT'])
_setport('keep-web-dl-ssl', nginxconf['KEEPWEBDLSSLPORT'])
_setport('keep-web-ssl', nginxconf['KEEPWEBSSLPORT'])
_setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
actions = [
'start', 'stop',
'start_ws', 'stop_ws',
+ 'start_controller', 'stop_controller',
'start_keep', 'stop_keep',
'start_keep_proxy', 'stop_keep_proxy',
'start_keep-web', 'stop_keep-web',
run_ws()
elif args.action == 'stop_ws':
stop_ws()
+ elif args.action == 'start_controller':
+ run_controller()
+ elif args.action == 'stop_controller':
+ stop_controller()
elif args.action == 'start_keep':
run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers)
elif args.action == 'stop_keep':
stop_keep_web()
elif args.action == 'start_nginx':
run_nginx()
+ print("export ARVADOS_API_HOST=0.0.0.0:{}".format(_getport('controller-ssl')))
elif args.action == 'stop_nginx':
stop_nginx()
else:
import re
import sys
import tempfile
+import datetime
+import ciso8601
import time
import unittest
self.assertEqual(fn0, c.items()[0][0])
self.assertEqual(fn1, c.items()[1][0])
+ def test_get_properties(self):
+ c = Collection()
+ self.assertEqual(c.get_properties(), {})
+ c.save_new(properties={"foo":"bar"})
+ self.assertEqual(c.get_properties(), {"foo":"bar"})
+
+ def test_get_trash_at(self):
+ c = Collection()
+ self.assertEqual(c.get_trash_at(), None)
+ c.save_new(trash_at=datetime.datetime(2111, 1, 1, 11, 11, 11, 111111))
+ self.assertEqual(c.get_trash_at(), ciso8601.parse_datetime('2111-01-01T11:11:11.111111000Z'))
+
class CollectionOpenModes(run_test_server.TestCaseWithServers):
def test_create_and_save(self):
c = self.create_count_txt()
- c.save(storage_classes=['archive'])
+ c.save(properties={'type' : 'Intermediate'},
+ storage_classes=['archive'],
+ trash_at=datetime.datetime(2111, 1, 1, 11, 11, 11, 111111))
self.assertRegex(
c.manifest_text(),
r"^\. 781e5e245d69b566979b86e28d23f2c7\+10\+A[a-f0-9]{40}@[a-f0-9]{8} 0:10:count\.txt$",)
self.assertEqual(c.api_response()["storage_classes_desired"], ['archive'])
+ self.assertEqual(c.api_response()["properties"], {'type' : 'Intermediate'})
+ self.assertEqual(c.api_response()["trash_at"], '2111-01-01T11:11:11.111111000Z')
def test_create_and_save_new(self):
c = self.create_count_txt()
- c.save_new(storage_classes=['archive'])
+ c.save_new(properties={'type' : 'Intermediate'},
+ storage_classes=['archive'],
+ trash_at=datetime.datetime(2111, 1, 1, 11, 11, 11, 111111))
self.assertRegex(
c.manifest_text(),
r"^\. 781e5e245d69b566979b86e28d23f2c7\+10\+A[a-f0-9]{40}@[a-f0-9]{8} 0:10:count\.txt$",)
self.assertEqual(c.api_response()["storage_classes_desired"], ['archive'])
+ self.assertEqual(c.api_response()["properties"], {'type' : 'Intermediate'})
+ self.assertEqual(c.api_response()["trash_at"], '2111-01-01T11:11:11.111111000Z')
- def test_update_storage_classes_desired_if_collection_is_commited(self):
+ def test_create_and_save_after_commiting(self):
c = self.create_count_txt()
- c.save(storage_classes=['hot'])
- c.save(storage_classes=['cold'])
+ c.save(properties={'type' : 'Intermediate'},
+ storage_classes=['hot'],
+ trash_at=datetime.datetime(2111, 1, 1, 11, 11, 11, 111111))
+ c.save(properties={'type' : 'Output'},
+ storage_classes=['cold'],
+ trash_at=datetime.datetime(2222, 2, 2, 22, 22, 22, 222222))
self.assertEqual(c.api_response()["storage_classes_desired"], ['cold'])
+ self.assertEqual(c.api_response()["properties"], {'type' : 'Output'})
+ self.assertEqual(c.api_response()["trash_at"], '2222-02-02T22:22:22.222222000Z')
def test_create_diff_apply(self):
c1 = self.create_count_txt()
self.assertEqual('100::1', service.hostname)
self.assertEqual(10, service.port)
+ def test_insecure_disables_tls_verify(self):
+ api_client = self.mock_keep_services(count=1)
+ force_timeout = socket.timeout("timed out")
+
+ api_client.insecure = True
+ with tutil.mock_keep_responses(b'foo', 200) as mock:
+ keep_client = arvados.KeepClient(api_client=api_client)
+ keep_client.get('acbd18db4cc2f85cedef654fccc4a4d8+3')
+ self.assertEqual(
+ mock.responses[0].getopt(pycurl.SSL_VERIFYPEER),
+ 0)
+
+ api_client.insecure = False
+ with tutil.mock_keep_responses(b'foo', 200) as mock:
+ keep_client = arvados.KeepClient(api_client=api_client)
+ keep_client.get('acbd18db4cc2f85cedef654fccc4a4d8+3')
+ # getopt()==None here means we didn't change the
+ # default. If we were using real pycurl instead of a mock,
+ # it would return the default value 1.
+ self.assertEqual(
+ mock.responses[0].getopt(pycurl.SSL_VERIFYPEER),
+ None)
+
# test_*_timeout verify that KeepClient instructs pycurl to use
# the appropriate connection and read timeouts. They don't care
# whether pycurl actually exhibits the expected timeout behavior
def __getattr__(self, r):
if r == "api_token":
return "abc"
+ elif r == "insecure":
+ return False
else:
raise arvados.errors.KeepReadError()
keep_client = arvados.KeepClient(api_client=ApiMock(),
# SPDX-License-Identifier: Apache-2.0
require 'google/api_client'
+# Monkeypatch google-api-client gem to avoid sending newline characters
+# on headers to make ruby-2.3.7+ happy.
+# See: https://dev.arvados.org/issues/13920
+Google::APIClient::ENV::OS_VERSION.strip!
+
require 'json'
require 'tempfile'
gem 'mocha', require: false
end
+# We need this dependency because of crunchv1
+gem 'arvados-cli'
+
# We'll need to update related code prior to Rails 5.
# See: https://github.com/rails/activerecord-deprecated_finders
gem 'activerecord-deprecated_finders', require: 'active_record/deprecated_finders'
gem 'multi_json'
gem 'oj'
-gem 'oj_mimic_json'
# for building assets
gem 'sass-rails', '~> 4.0'
gem 'themes_for_rails', git: 'https://github.com/curoverse/themes_for_rails'
gem 'arvados', '>= 0.1.20150615153458'
-gem 'arvados-cli', '>= 0.1.20161017193526'
gem 'httpclient'
gem 'sshkey'
i18n (~> 0)
json (>= 1.7.7, < 3)
jwt (>= 0.1.5, < 2)
- arvados-cli (1.1.4.20180412190507)
+ arvados-cli (1.1.4.20180723133344)
activesupport (>= 3.2.13, < 5)
andand (~> 1.3, >= 1.3.3)
arvados (~> 0.1, >= 0.1.20150128223554)
curb (~> 0.8)
google-api-client (~> 0.6, >= 0.6.3, < 0.8.9)
json (>= 1.7.7, < 3)
- oj (~> 2.0, >= 2.0.3)
+ oj (~> 3.0)
trollop (~> 2.0)
autoparse (0.3.3)
addressable (>= 2.3.1)
coffee-script-source (1.12.2)
concurrent-ruby (1.0.5)
crass (1.0.4)
- curb (0.9.4)
+ curb (0.9.6)
database_cleaner (1.7.0)
erubis (2.7.0)
eventmachine (1.2.6)
multi_json (~> 1.3)
multi_xml (~> 0.5)
rack (>= 1.2, < 3)
- oj (2.18.5)
- oj_mimic_json (1.0.1)
+ oj (3.6.4)
omniauth (1.4.3)
hashie (>= 1.2, < 4)
rack (>= 1.6.2, < 3)
simplecov-html (0.7.1)
simplecov-rcov (0.2.3)
simplecov (>= 0.4.1)
- sprockets (2.12.4)
+ sprockets (2.12.5)
hike (~> 1.2)
multi_json (~> 1.0)
rack (~> 1.0)
acts_as_api
andand
arvados (>= 0.1.20150615153458)
- arvados-cli (>= 0.1.20161017193526)
+ arvados-cli
coffee-rails (~> 4.0)
database_cleaner
factory_girl_rails
mocha
multi_json
oj
- oj_mimic_json
omniauth (~> 1.4.0)
omniauth-oauth2 (~> 1.1)
passenger
uglifier (~> 2.0)
BUNDLED WITH
- 1.16.1
+ 1.16.3
end
def update
- # container updates can trigger container request lookups, which
- # can deadlock if we don't lock the container_requests table
- # first.
- @object.transaction do
- ActiveRecord::Base.connection.execute('LOCK container_requests, containers IN EXCLUSIVE MODE')
+ @object.with_lock do
@object.reload
super
end
all_objects = []
@items_available = 0
+ # Reload the orders param, this time without prefixing unqualified
+ # columns ("name" => "groups.name"). Here, unqualified orders
+ # apply to each table being searched, not "groups".
+ load_limit_offset_order_params(fill_table_names: false)
+
# Trick apply_where_limit_order_params into applying suitable
# per-table values. *_all are the real ones we'll apply to the
# aggregate set.
# table_name for the current klass, apply that order.
# Otherwise, order by recency.
request_order =
- request_orders.andand.find { |r| r =~ /^#{klass.table_name}\./i } ||
+ request_orders.andand.find { |r| r =~ /^#{klass.table_name}\./i || r !~ /\./ } ||
klass.default_orders.join(", ")
@select = nil
version: "v1",
revision: "20131114",
source_version: AppVersion.hash,
+ sourceVersion: AppVersion.hash, # source_version should be deprecated in the future
+ packageVersion: AppVersion.package_version,
generatedAt: db_current_time.iso8601,
title: "Arvados API",
description: "The API to interact with Arvados.",
end
def empty
- render text: "-"
+ render text: ""
end
end
end
end
- if Rails.configuration.new_users_are_active
+ if Rails.configuration.new_users_are_active ||
+ Rails.configuration.auto_activate_users_from.include?(remote_user['uuid'][0..4])
# Update is_active to whatever it is at the remote end
user.is_active = remote_user['is_active']
elsif !remote_user['is_active']
end
def manifest_files
+ return '' if !self.manifest_text
+
names = ''
- if self.manifest_text
- self.manifest_text.scan(/ \d+:\d+:(\S+)/) do |name|
- names << name.first.gsub('\040',' ') + "\n"
- break if names.length > 2**12
- end
+ self.manifest_text.scan(/ \d+:\d+:(\S+)/) do |name|
+ names << name.first.gsub('\040',' ') + "\n"
end
-
- if self.manifest_text and names.length < 2**12
- self.manifest_text.scan(/^\.\/(\S+)/m) do |stream_name|
- names << stream_name.first.gsub('\040',' ') + "\n"
- break if names.length > 2**12
- end
+ self.manifest_text.scan(/^\.\/(\S+)/m) do |stream_name|
+ names << stream_name.first.gsub('\040',' ') + "\n"
end
-
- names[0,2**12]
+ names
end
def default_empty_manifest
require 'log_reuse_info'
require 'whitelist_update'
require 'safe_json'
+require 'update_priority'
class Container < ArvadosModel
include ArvadosModelUpdates
before_save :scrub_secret_mounts
after_save :handle_completed
after_save :propagate_priority
+ after_commit { UpdatePriority.run_update_thread }
has_many :container_requests, :foreign_key => :container_uuid, :class_name => 'ContainerRequest', :primary_key => :uuid
belongs_to :auth, :class_name => 'ApiClientAuthorization', :foreign_key => :auth_uuid, :primary_key => :uuid
# Update the priority of child container requests to match new
# priority of the parent container (ignoring requests with no
# container assigned, because their priority doesn't matter).
- ActiveRecord::Base.connection.execute('LOCK container_requests, containers IN EXCLUSIVE MODE')
ContainerRequest.
where(requesting_container_uuid: self.uuid,
state: ContainerRequest::Committed).
# (because state might have changed while acquiring the lock).
check_lock_fail
transaction do
- # Locking involves assigning auth_uuid, which involves looking
- # up container requests, so we must lock both tables in the
- # proper order to avoid deadlock.
- ActiveRecord::Base.connection.execute('LOCK container_requests, containers IN EXCLUSIVE MODE')
reload
check_lock_fail
update_attributes!(state: Locked)
if self.state_changed? and self.final?
act_as_system_user do
- ActiveRecord::Base.connection.execute('LOCK container_requests, containers IN EXCLUSIVE MODE')
if self.state == Cancelled
retryable_requests = ContainerRequest.where("container_uuid = ? and priority > 0 and state = 'Committed' and container_count < container_count_max", uuid)
else
before_validation :fill_field_defaults, :if => :new_record?
before_validation :validate_runtime_constraints
- before_validation :validate_scheduling_parameters
+ before_validation :set_default_preemptible_scheduling_parameter
before_validation :set_container
validates :command, :container_image, :output_path, :cwd, :presence => true
validates :output_ttl, numericality: { only_integer: true, greater_than_or_equal_to: 0 }
validates :priority, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: 1000 }
+ validate :validate_scheduling_parameters
validate :validate_state_change
validate :check_update_whitelist
validate :secret_mounts_key_conflict
Committed => [Final]
}
- AttrsPermittedAlways = [:owner_uuid, :state, :name, :description]
+ AttrsPermittedAlways = [:owner_uuid, :state, :name, :description, :properties]
AttrsPermittedBeforeCommit = [:command, :container_count_max,
:container_image, :cwd, :environment, :filters, :mounts,
- :output_path, :priority, :properties,
+ :output_path, :priority,
:runtime_constraints, :state, :container_uuid, :use_existing,
:scheduling_parameters, :secret_mounts, :output_name, :output_ttl]
end
end
+ def set_default_preemptible_scheduling_parameter
+ c = get_requesting_container()
+ if self.state == Committed
+ # If preemptible instances (eg: AWS Spot Instances) are allowed,
+ # ask them on child containers by default.
+ if Rails.configuration.preemptible_instances and !c.nil? and
+ self.scheduling_parameters['preemptible'].nil?
+ self.scheduling_parameters['preemptible'] = true
+ end
+ end
+ end
+
def validate_runtime_constraints
case self.state
when Committed
scheduling_parameters['partitions'].size)
errors.add :scheduling_parameters, "partitions must be an array of strings"
end
+ if !Rails.configuration.preemptible_instances and scheduling_parameters['preemptible']
+ errors.add :scheduling_parameters, "preemptible instances are not allowed"
+ end
+ if scheduling_parameters.include? 'max_run_time' and
+ (!scheduling_parameters['max_run_time'].is_a?(Integer) ||
+ scheduling_parameters['max_run_time'] < 0)
+ errors.add :scheduling_parameters, "max_run_time must be positive integer"
+ end
end
end
def update_priority
return unless state_changed? || priority_changed? || container_uuid_changed?
act_as_system_user do
- ActiveRecord::Base.connection.execute('LOCK container_requests, containers IN EXCLUSIVE MODE')
Container.
where('uuid in (?)', [self.container_uuid_was, self.container_uuid].compact).
map(&:update_priority!)
end
def set_requesting_container_uuid
- return if !current_api_client_authorization
- ActiveRecord::Base.connection.execute('LOCK container_requests, containers IN EXCLUSIVE MODE')
- if (c = Container.where('auth_uuid=?', current_api_client_authorization.uuid).select([:uuid, :priority]).first)
+ c = get_requesting_container()
+ if !c.nil?
self.requesting_container_uuid = c.uuid
self.priority = c.priority>0 ? 1 : 0
end
end
+
+ def get_requesting_container
+ return self.requesting_container_uuid if !self.requesting_container_uuid.nil?
+ return if !current_api_client_authorization
+ if (c = Container.where('auth_uuid=?', current_api_client_authorization.uuid).select([:uuid, :priority]).first)
+ return c
+ end
+ end
end
include CanBeAnOwner
include Trashable
+ serialize :properties, Hash
+
after_create :invalidate_permissions_cache
after_update :maybe_invalidate_permissions_cache
before_create :assign_name
t.add :delete_at
t.add :trash_at
t.add :is_trashed
+ t.add :properties
end
def maybe_invalidate_permissions_cache
before_create :set_initial_username, :if => Proc.new { |user|
user.username.nil? and user.email
}
+ after_create :setup_on_activate
after_create :add_system_group_permission_link
after_create :invalidate_permissions_cache
after_create :auto_setup_new_user, :if => Proc.new { |user|
if !oid_login_perms.any?
# create openid login permission
- oid_login_perm = Link.create(link_class: 'permission',
+ oid_login_perm = Link.create!(link_class: 'permission',
name: 'can_login',
tail_uuid: self.email,
head_uuid: self.uuid,
### New user and & email settings
###
- # Config parameters to automatically setup new users.
+ # Config parameters to automatically setup new users. If enabled,
+ # this users will be able to self-activate. Enable this if you want
+ # to run an open instance where anyone can create an account and use
+ # the system without requiring manual approval.
+ #
# The params auto_setup_new_users_with_* are meaningful only when auto_setup_new_users is turned on.
# auto_setup_name_blacklist is a list of usernames to be blacklisted for auto setup.
auto_setup_new_users: false
auto_setup_new_users_with_repository: false
auto_setup_name_blacklist: [arvados, git, gitolite, gitolite-admin, root, syslog]
- # When new_users_are_active is set to true, the user agreement check is skipped.
+ # When new_users_are_active is set to true, new users will be active
+ # immediately. This skips the "self-activate" step which enforces
+ # user agreements. Should only be enabled for development.
new_users_are_active: false
# The e-mail address of the user you would like to become marked as an admin
### Crunch, DNS & compute node management
###
+ # Preemptible instance support (e.g. AWS Spot Instances)
+ # When true, child containers will get created with the preemptible
+ # scheduling parameter parameter set.
+ preemptible_instances: false
+
# Docker image to be used when none found in runtime_constraints of a job
default_docker_image_for_jobs: false
# remote_hosts above.
remote_hosts_via_dns: false
+ # List of cluster prefixes. These are "trusted" clusters, users
+ # from the clusters listed here will be automatically setup and
+ # activated. This is separate from the settings
+ # auto_setup_new_users and new_users_are_active.
+ auto_activate_users_from: []
+
###
### Remaining assorted configuration options.
###
# "git log".
source_version: false
+ # Override the automatic package version string. With the default version of
+ # false, the package version is read from package-build.version in Rails.root
+ # (included in vendor packages).
+ package_version: false
+
# Enable asynchronous permission graph rebuild. Must run
# script/permission-updater.rb as a separate process. When the permission
# cache is invalidated, the background process will update the permission
module Server
class Application < Rails::Application
+ # The following is to avoid SafeYAML's warning message
+ SafeYAML::OPTIONS[:default_mode] = :safe
+
# Settings in config/environments/* take precedence over those specified here.
# Application configuration should go into files in config/initializers
# -- all .rb files in that directory are automatically loaded.
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'oj'
+
+Oj::Rails.set_encoder()
+Oj::Rails.set_decoder()
+Oj::Rails.optimize()
+Oj::Rails.mimic_JSON()
+
#
# SPDX-License-Identifier: AGPL-3.0
+ActiveSupport::JSON::Encoding.time_precision = 9
+
class ActiveSupport::TimeWithZone
remove_method :as_json
def as_json *args
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require './db/migrate/20161213172944_full_text_search_indexes'
+
+class PropertiesToJsonb < ActiveRecord::Migration
+
+ @@tables_columns = [["nodes", "properties"],
+ ["nodes", "info"],
+ ["container_requests", "properties"],
+ ["links", "properties"]]
+
+ def up
+ @@tables_columns.each do |table, column|
+ # Drop the FT index before changing column type to avoid
+ # "PG::DatatypeMismatch: ERROR: COALESCE types jsonb and text
+ # cannot be matched".
+ ActiveRecord::Base.connection.execute "DROP INDEX IF EXISTS #{table}_full_text_search_idx"
+ ActiveRecord::Base.connection.execute "ALTER TABLE #{table} ALTER COLUMN #{column} TYPE jsonb USING #{column}::jsonb"
+ ActiveRecord::Base.connection.execute "CREATE INDEX #{table}_index_on_#{column} ON #{table} USING gin (#{column})"
+ end
+ FullTextSearchIndexes.new.replace_index("container_requests")
+ end
+
+ def down
+ @@tables_columns.each do |table, column|
+ ActiveRecord::Base.connection.execute "DROP INDEX IF EXISTS #{table}_index_on_#{column}"
+ ActiveRecord::Base.connection.execute "ALTER TABLE #{table} ALTER COLUMN #{column} TYPE text"
+ end
+ end
+end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require './db/migrate/20161213172944_full_text_search_indexes'
+
+class AddPropertiesToGroups < ActiveRecord::Migration
+ def up
+ add_column :groups, :properties, :jsonb, default: {}
+ ActiveRecord::Base.connection.execute("CREATE INDEX group_index_on_properties ON groups USING gin (properties);")
+ FullTextSearchIndexes.new.replace_index('groups')
+ end
+
+ def down
+ ActiveRecord::Base.connection.execute("DROP INDEX IF EXISTS group_index_on_properties")
+ remove_column :groups, :properties
+ end
+end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+class IndexAllFilenames < ActiveRecord::Migration
+ def up
+ ActiveRecord::Base.connection.execute 'ALTER TABLE collections ALTER COLUMN file_names TYPE text'
+ Collection.find_each(batch_size: 20) do |c|
+ ActiveRecord::Base.connection.execute "UPDATE collections
+ SET file_names = #{ActiveRecord::Base.connection.quote(c.manifest_files)}
+ WHERE uuid = #{ActiveRecord::Base.connection.quote(c.uuid)}
+ AND portable_data_hash = #{ActiveRecord::Base.connection.quote(c.portable_data_hash)}"
+ end
+ end
+ def down
+ ActiveRecord::Base.connection.execute 'ALTER TABLE collections ALTER COLUMN file_names TYPE varchar(8192)'
+ end
+end
SET statement_timeout = 0;
SET client_encoding = 'UTF8';
SET standard_conforming_strings = on;
+SELECT pg_catalog.set_config('search_path', '', false);
SET check_function_bodies = false;
SET client_min_messages = warning;
-- COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
-SET search_path = public, pg_catalog;
-
SET default_tablespace = '';
SET default_with_oids = false;
-- Name: api_client_authorizations; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE api_client_authorizations (
+CREATE TABLE public.api_client_authorizations (
id integer NOT NULL,
api_token character varying(255) NOT NULL,
api_client_id integer NOT NULL,
-- Name: api_client_authorizations_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE api_client_authorizations_id_seq
+CREATE SEQUENCE public.api_client_authorizations_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: api_client_authorizations_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE api_client_authorizations_id_seq OWNED BY api_client_authorizations.id;
+ALTER SEQUENCE public.api_client_authorizations_id_seq OWNED BY public.api_client_authorizations.id;
--
-- Name: api_clients; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE api_clients (
+CREATE TABLE public.api_clients (
id integer NOT NULL,
uuid character varying(255),
owner_uuid character varying(255),
-- Name: api_clients_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE api_clients_id_seq
+CREATE SEQUENCE public.api_clients_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: api_clients_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE api_clients_id_seq OWNED BY api_clients.id;
+ALTER SEQUENCE public.api_clients_id_seq OWNED BY public.api_clients.id;
--
-- Name: authorized_keys; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE authorized_keys (
+CREATE TABLE public.authorized_keys (
id integer NOT NULL,
uuid character varying(255) NOT NULL,
owner_uuid character varying(255) NOT NULL,
-- Name: authorized_keys_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE authorized_keys_id_seq
+CREATE SEQUENCE public.authorized_keys_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: authorized_keys_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE authorized_keys_id_seq OWNED BY authorized_keys.id;
+ALTER SEQUENCE public.authorized_keys_id_seq OWNED BY public.authorized_keys.id;
--
-- Name: collections; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE collections (
+CREATE TABLE public.collections (
id integer NOT NULL,
owner_uuid character varying(255),
created_at timestamp without time zone NOT NULL,
description character varying(524288),
properties jsonb,
delete_at timestamp without time zone,
- file_names character varying(8192),
+ file_names text,
trash_at timestamp without time zone,
is_trashed boolean DEFAULT false NOT NULL,
storage_classes_desired jsonb DEFAULT '["default"]'::jsonb,
-- Name: collections_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE collections_id_seq
+CREATE SEQUENCE public.collections_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: collections_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE collections_id_seq OWNED BY collections.id;
+ALTER SEQUENCE public.collections_id_seq OWNED BY public.collections.id;
--
-- Name: commit_ancestors; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE commit_ancestors (
+CREATE TABLE public.commit_ancestors (
id integer NOT NULL,
repository_name character varying(255),
descendant character varying(255) NOT NULL,
-- Name: commit_ancestors_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE commit_ancestors_id_seq
+CREATE SEQUENCE public.commit_ancestors_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: commit_ancestors_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE commit_ancestors_id_seq OWNED BY commit_ancestors.id;
+ALTER SEQUENCE public.commit_ancestors_id_seq OWNED BY public.commit_ancestors.id;
--
-- Name: commits; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE commits (
+CREATE TABLE public.commits (
id integer NOT NULL,
repository_name character varying(255),
sha1 character varying(255),
-- Name: commits_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE commits_id_seq
+CREATE SEQUENCE public.commits_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: commits_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE commits_id_seq OWNED BY commits.id;
+ALTER SEQUENCE public.commits_id_seq OWNED BY public.commits.id;
--
-- Name: container_requests; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE container_requests (
+CREATE TABLE public.container_requests (
id integer NOT NULL,
uuid character varying(255),
owner_uuid character varying(255),
modified_by_user_uuid character varying(255),
name character varying(255),
description text,
- properties text,
+ properties jsonb,
state character varying(255),
requesting_container_uuid character varying(255),
container_uuid character varying(255),
-- Name: container_requests_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE container_requests_id_seq
+CREATE SEQUENCE public.container_requests_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: container_requests_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE container_requests_id_seq OWNED BY container_requests.id;
+ALTER SEQUENCE public.container_requests_id_seq OWNED BY public.container_requests.id;
--
-- Name: containers; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE containers (
+CREATE TABLE public.containers (
id integer NOT NULL,
uuid character varying(255),
owner_uuid character varying(255),
-- Name: containers_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE containers_id_seq
+CREATE SEQUENCE public.containers_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: containers_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE containers_id_seq OWNED BY containers.id;
+ALTER SEQUENCE public.containers_id_seq OWNED BY public.containers.id;
--
-- Name: groups; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE groups (
+CREATE TABLE public.groups (
id integer NOT NULL,
uuid character varying(255),
owner_uuid character varying(255),
group_class character varying(255),
trash_at timestamp without time zone,
is_trashed boolean DEFAULT false NOT NULL,
- delete_at timestamp without time zone
+ delete_at timestamp without time zone,
+ properties jsonb DEFAULT '{}'::jsonb
);
-- Name: groups_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE groups_id_seq
+CREATE SEQUENCE public.groups_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: groups_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE groups_id_seq OWNED BY groups.id;
+ALTER SEQUENCE public.groups_id_seq OWNED BY public.groups.id;
--
-- Name: humans; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE humans (
+CREATE TABLE public.humans (
id integer NOT NULL,
uuid character varying(255) NOT NULL,
owner_uuid character varying(255) NOT NULL,
-- Name: humans_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE humans_id_seq
+CREATE SEQUENCE public.humans_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: humans_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE humans_id_seq OWNED BY humans.id;
+ALTER SEQUENCE public.humans_id_seq OWNED BY public.humans.id;
--
-- Name: job_tasks; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE job_tasks (
+CREATE TABLE public.job_tasks (
id integer NOT NULL,
uuid character varying(255),
owner_uuid character varying(255),
-- Name: job_tasks_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE job_tasks_id_seq
+CREATE SEQUENCE public.job_tasks_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: job_tasks_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE job_tasks_id_seq OWNED BY job_tasks.id;
+ALTER SEQUENCE public.job_tasks_id_seq OWNED BY public.job_tasks.id;
--
-- Name: job_tasks_qsequence_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE job_tasks_qsequence_seq
+CREATE SEQUENCE public.job_tasks_qsequence_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: job_tasks_qsequence_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE job_tasks_qsequence_seq OWNED BY job_tasks.qsequence;
+ALTER SEQUENCE public.job_tasks_qsequence_seq OWNED BY public.job_tasks.qsequence;
--
-- Name: jobs; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE jobs (
+CREATE TABLE public.jobs (
id integer NOT NULL,
uuid character varying(255),
owner_uuid character varying(255),
-- Name: jobs_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE jobs_id_seq
+CREATE SEQUENCE public.jobs_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: jobs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE jobs_id_seq OWNED BY jobs.id;
+ALTER SEQUENCE public.jobs_id_seq OWNED BY public.jobs.id;
--
-- Name: keep_disks; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE keep_disks (
+CREATE TABLE public.keep_disks (
id integer NOT NULL,
uuid character varying(255) NOT NULL,
owner_uuid character varying(255) NOT NULL,
-- Name: keep_disks_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE keep_disks_id_seq
+CREATE SEQUENCE public.keep_disks_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: keep_disks_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE keep_disks_id_seq OWNED BY keep_disks.id;
+ALTER SEQUENCE public.keep_disks_id_seq OWNED BY public.keep_disks.id;
--
-- Name: keep_services; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE keep_services (
+CREATE TABLE public.keep_services (
id integer NOT NULL,
uuid character varying(255) NOT NULL,
owner_uuid character varying(255) NOT NULL,
-- Name: keep_services_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE keep_services_id_seq
+CREATE SEQUENCE public.keep_services_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: keep_services_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE keep_services_id_seq OWNED BY keep_services.id;
+ALTER SEQUENCE public.keep_services_id_seq OWNED BY public.keep_services.id;
--
-- Name: links; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE links (
+CREATE TABLE public.links (
id integer NOT NULL,
uuid character varying(255),
owner_uuid character varying(255),
link_class character varying(255),
name character varying(255),
head_uuid character varying(255),
- properties text,
+ properties jsonb,
updated_at timestamp without time zone NOT NULL
);
-- Name: links_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE links_id_seq
+CREATE SEQUENCE public.links_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: links_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE links_id_seq OWNED BY links.id;
+ALTER SEQUENCE public.links_id_seq OWNED BY public.links.id;
--
-- Name: logs; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE logs (
+CREATE TABLE public.logs (
id integer NOT NULL,
uuid character varying(255),
owner_uuid character varying(255),
-- Name: logs_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE logs_id_seq
+CREATE SEQUENCE public.logs_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE logs_id_seq OWNED BY logs.id;
+ALTER SEQUENCE public.logs_id_seq OWNED BY public.logs.id;
--
-- Name: users; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE users (
+CREATE TABLE public.users (
id integer NOT NULL,
uuid character varying(255),
owner_uuid character varying(255) NOT NULL,
-- Name: materialized_permission_view; Type: MATERIALIZED VIEW; Schema: public; Owner: -
--
-CREATE MATERIALIZED VIEW materialized_permission_view AS
+CREATE MATERIALIZED VIEW public.materialized_permission_view AS
WITH RECURSIVE perm_value(name, val) AS (
VALUES ('can_read'::text,(1)::smallint), ('can_login'::text,1), ('can_write'::text,2), ('can_manage'::text,3)
), perm_edges(tail_uuid, head_uuid, val, follow, trashed) AS (
((pv.val = 3) OR (groups.uuid IS NOT NULL)) AS follow,
(0)::smallint AS trashed,
(0)::smallint AS followtrash
- FROM ((links
+ FROM ((public.links
LEFT JOIN perm_value pv ON ((pv.name = (links.name)::text)))
- LEFT JOIN groups ON (((pv.val < 3) AND ((groups.uuid)::text = (links.head_uuid)::text))))
+ LEFT JOIN public.groups ON (((pv.val < 3) AND ((groups.uuid)::text = (links.head_uuid)::text))))
WHERE ((links.link_class)::text = 'permission'::text)
UNION ALL
SELECT groups.owner_uuid,
ELSE 0
END AS "case",
1
- FROM groups
+ FROM public.groups
), perm(val, follow, user_uuid, target_uuid, trashed) AS (
SELECT (3)::smallint AS val,
true AS follow,
(users.uuid)::character varying(32) AS user_uuid,
(users.uuid)::character varying(32) AS target_uuid,
(0)::smallint AS trashed
- FROM users
+ FROM public.users
UNION
SELECT (LEAST((perm_1.val)::integer, edges.val))::smallint AS val,
edges.follow,
-- Name: nodes; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE nodes (
+CREATE TABLE public.nodes (
id integer NOT NULL,
uuid character varying(255),
owner_uuid character varying(255),
ip_address character varying(255),
first_ping_at timestamp without time zone,
last_ping_at timestamp without time zone,
- info text,
+ info jsonb,
updated_at timestamp without time zone NOT NULL,
- properties text,
+ properties jsonb,
job_uuid character varying(255)
);
-- Name: nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE nodes_id_seq
+CREATE SEQUENCE public.nodes_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: nodes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE nodes_id_seq OWNED BY nodes.id;
+ALTER SEQUENCE public.nodes_id_seq OWNED BY public.nodes.id;
--
-- Name: permission_refresh_lock; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE permission_refresh_lock (
+CREATE TABLE public.permission_refresh_lock (
id integer NOT NULL
);
-- Name: permission_refresh_lock_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE permission_refresh_lock_id_seq
+CREATE SEQUENCE public.permission_refresh_lock_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: permission_refresh_lock_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE permission_refresh_lock_id_seq OWNED BY permission_refresh_lock.id;
+ALTER SEQUENCE public.permission_refresh_lock_id_seq OWNED BY public.permission_refresh_lock.id;
--
-- Name: pipeline_instances; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE pipeline_instances (
+CREATE TABLE public.pipeline_instances (
id integer NOT NULL,
uuid character varying(255),
owner_uuid character varying(255),
-- Name: pipeline_instances_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE pipeline_instances_id_seq
+CREATE SEQUENCE public.pipeline_instances_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: pipeline_instances_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE pipeline_instances_id_seq OWNED BY pipeline_instances.id;
+ALTER SEQUENCE public.pipeline_instances_id_seq OWNED BY public.pipeline_instances.id;
--
-- Name: pipeline_templates; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE pipeline_templates (
+CREATE TABLE public.pipeline_templates (
id integer NOT NULL,
uuid character varying(255),
owner_uuid character varying(255),
-- Name: pipeline_templates_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE pipeline_templates_id_seq
+CREATE SEQUENCE public.pipeline_templates_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: pipeline_templates_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE pipeline_templates_id_seq OWNED BY pipeline_templates.id;
+ALTER SEQUENCE public.pipeline_templates_id_seq OWNED BY public.pipeline_templates.id;
--
-- Name: repositories; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE repositories (
+CREATE TABLE public.repositories (
id integer NOT NULL,
uuid character varying(255) NOT NULL,
owner_uuid character varying(255) NOT NULL,
-- Name: repositories_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE repositories_id_seq
+CREATE SEQUENCE public.repositories_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: repositories_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE repositories_id_seq OWNED BY repositories.id;
+ALTER SEQUENCE public.repositories_id_seq OWNED BY public.repositories.id;
--
-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE schema_migrations (
+CREATE TABLE public.schema_migrations (
version character varying(255) NOT NULL
);
-- Name: specimens; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE specimens (
+CREATE TABLE public.specimens (
id integer NOT NULL,
uuid character varying(255),
owner_uuid character varying(255),
-- Name: specimens_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE specimens_id_seq
+CREATE SEQUENCE public.specimens_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: specimens_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE specimens_id_seq OWNED BY specimens.id;
+ALTER SEQUENCE public.specimens_id_seq OWNED BY public.specimens.id;
--
-- Name: traits; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE traits (
+CREATE TABLE public.traits (
id integer NOT NULL,
uuid character varying(255) NOT NULL,
owner_uuid character varying(255) NOT NULL,
-- Name: traits_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE traits_id_seq
+CREATE SEQUENCE public.traits_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: traits_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE traits_id_seq OWNED BY traits.id;
+ALTER SEQUENCE public.traits_id_seq OWNED BY public.traits.id;
--
-- Name: users_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE users_id_seq
+CREATE SEQUENCE public.users_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: users_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE users_id_seq OWNED BY users.id;
+ALTER SEQUENCE public.users_id_seq OWNED BY public.users.id;
--
-- Name: virtual_machines; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE virtual_machines (
+CREATE TABLE public.virtual_machines (
id integer NOT NULL,
uuid character varying(255) NOT NULL,
owner_uuid character varying(255) NOT NULL,
-- Name: virtual_machines_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE virtual_machines_id_seq
+CREATE SEQUENCE public.virtual_machines_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: virtual_machines_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE virtual_machines_id_seq OWNED BY virtual_machines.id;
+ALTER SEQUENCE public.virtual_machines_id_seq OWNED BY public.virtual_machines.id;
--
-- Name: workflows; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE workflows (
+CREATE TABLE public.workflows (
id integer NOT NULL,
uuid character varying(255),
owner_uuid character varying(255),
-- Name: workflows_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE workflows_id_seq
+CREATE SEQUENCE public.workflows_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
-- Name: workflows_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE workflows_id_seq OWNED BY workflows.id;
+ALTER SEQUENCE public.workflows_id_seq OWNED BY public.workflows.id;
--
-- Name: api_client_authorizations id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY api_client_authorizations ALTER COLUMN id SET DEFAULT nextval('api_client_authorizations_id_seq'::regclass);
+ALTER TABLE ONLY public.api_client_authorizations ALTER COLUMN id SET DEFAULT nextval('public.api_client_authorizations_id_seq'::regclass);
--
-- Name: api_clients id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY api_clients ALTER COLUMN id SET DEFAULT nextval('api_clients_id_seq'::regclass);
+ALTER TABLE ONLY public.api_clients ALTER COLUMN id SET DEFAULT nextval('public.api_clients_id_seq'::regclass);
--
-- Name: authorized_keys id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY authorized_keys ALTER COLUMN id SET DEFAULT nextval('authorized_keys_id_seq'::regclass);
+ALTER TABLE ONLY public.authorized_keys ALTER COLUMN id SET DEFAULT nextval('public.authorized_keys_id_seq'::regclass);
--
-- Name: collections id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY collections ALTER COLUMN id SET DEFAULT nextval('collections_id_seq'::regclass);
+ALTER TABLE ONLY public.collections ALTER COLUMN id SET DEFAULT nextval('public.collections_id_seq'::regclass);
--
-- Name: commit_ancestors id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY commit_ancestors ALTER COLUMN id SET DEFAULT nextval('commit_ancestors_id_seq'::regclass);
+ALTER TABLE ONLY public.commit_ancestors ALTER COLUMN id SET DEFAULT nextval('public.commit_ancestors_id_seq'::regclass);
--
-- Name: commits id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY commits ALTER COLUMN id SET DEFAULT nextval('commits_id_seq'::regclass);
+ALTER TABLE ONLY public.commits ALTER COLUMN id SET DEFAULT nextval('public.commits_id_seq'::regclass);
--
-- Name: container_requests id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY container_requests ALTER COLUMN id SET DEFAULT nextval('container_requests_id_seq'::regclass);
+ALTER TABLE ONLY public.container_requests ALTER COLUMN id SET DEFAULT nextval('public.container_requests_id_seq'::regclass);
--
-- Name: containers id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY containers ALTER COLUMN id SET DEFAULT nextval('containers_id_seq'::regclass);
+ALTER TABLE ONLY public.containers ALTER COLUMN id SET DEFAULT nextval('public.containers_id_seq'::regclass);
--
-- Name: groups id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY groups ALTER COLUMN id SET DEFAULT nextval('groups_id_seq'::regclass);
+ALTER TABLE ONLY public.groups ALTER COLUMN id SET DEFAULT nextval('public.groups_id_seq'::regclass);
--
-- Name: humans id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY humans ALTER COLUMN id SET DEFAULT nextval('humans_id_seq'::regclass);
+ALTER TABLE ONLY public.humans ALTER COLUMN id SET DEFAULT nextval('public.humans_id_seq'::regclass);
--
-- Name: job_tasks id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY job_tasks ALTER COLUMN id SET DEFAULT nextval('job_tasks_id_seq'::regclass);
+ALTER TABLE ONLY public.job_tasks ALTER COLUMN id SET DEFAULT nextval('public.job_tasks_id_seq'::regclass);
--
-- Name: jobs id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY jobs ALTER COLUMN id SET DEFAULT nextval('jobs_id_seq'::regclass);
+ALTER TABLE ONLY public.jobs ALTER COLUMN id SET DEFAULT nextval('public.jobs_id_seq'::regclass);
--
-- Name: keep_disks id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY keep_disks ALTER COLUMN id SET DEFAULT nextval('keep_disks_id_seq'::regclass);
+ALTER TABLE ONLY public.keep_disks ALTER COLUMN id SET DEFAULT nextval('public.keep_disks_id_seq'::regclass);
--
-- Name: keep_services id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY keep_services ALTER COLUMN id SET DEFAULT nextval('keep_services_id_seq'::regclass);
+ALTER TABLE ONLY public.keep_services ALTER COLUMN id SET DEFAULT nextval('public.keep_services_id_seq'::regclass);
--
-- Name: links id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY links ALTER COLUMN id SET DEFAULT nextval('links_id_seq'::regclass);
+ALTER TABLE ONLY public.links ALTER COLUMN id SET DEFAULT nextval('public.links_id_seq'::regclass);
--
-- Name: logs id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY logs ALTER COLUMN id SET DEFAULT nextval('logs_id_seq'::regclass);
+ALTER TABLE ONLY public.logs ALTER COLUMN id SET DEFAULT nextval('public.logs_id_seq'::regclass);
--
-- Name: nodes id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY nodes ALTER COLUMN id SET DEFAULT nextval('nodes_id_seq'::regclass);
+ALTER TABLE ONLY public.nodes ALTER COLUMN id SET DEFAULT nextval('public.nodes_id_seq'::regclass);
--
-- Name: permission_refresh_lock id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY permission_refresh_lock ALTER COLUMN id SET DEFAULT nextval('permission_refresh_lock_id_seq'::regclass);
+ALTER TABLE ONLY public.permission_refresh_lock ALTER COLUMN id SET DEFAULT nextval('public.permission_refresh_lock_id_seq'::regclass);
--
-- Name: pipeline_instances id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY pipeline_instances ALTER COLUMN id SET DEFAULT nextval('pipeline_instances_id_seq'::regclass);
+ALTER TABLE ONLY public.pipeline_instances ALTER COLUMN id SET DEFAULT nextval('public.pipeline_instances_id_seq'::regclass);
--
-- Name: pipeline_templates id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY pipeline_templates ALTER COLUMN id SET DEFAULT nextval('pipeline_templates_id_seq'::regclass);
+ALTER TABLE ONLY public.pipeline_templates ALTER COLUMN id SET DEFAULT nextval('public.pipeline_templates_id_seq'::regclass);
--
-- Name: repositories id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY repositories ALTER COLUMN id SET DEFAULT nextval('repositories_id_seq'::regclass);
+ALTER TABLE ONLY public.repositories ALTER COLUMN id SET DEFAULT nextval('public.repositories_id_seq'::regclass);
--
-- Name: specimens id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY specimens ALTER COLUMN id SET DEFAULT nextval('specimens_id_seq'::regclass);
+ALTER TABLE ONLY public.specimens ALTER COLUMN id SET DEFAULT nextval('public.specimens_id_seq'::regclass);
--
-- Name: traits id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY traits ALTER COLUMN id SET DEFAULT nextval('traits_id_seq'::regclass);
+ALTER TABLE ONLY public.traits ALTER COLUMN id SET DEFAULT nextval('public.traits_id_seq'::regclass);
--
-- Name: users id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY users ALTER COLUMN id SET DEFAULT nextval('users_id_seq'::regclass);
+ALTER TABLE ONLY public.users ALTER COLUMN id SET DEFAULT nextval('public.users_id_seq'::regclass);
--
-- Name: virtual_machines id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY virtual_machines ALTER COLUMN id SET DEFAULT nextval('virtual_machines_id_seq'::regclass);
+ALTER TABLE ONLY public.virtual_machines ALTER COLUMN id SET DEFAULT nextval('public.virtual_machines_id_seq'::regclass);
--
-- Name: workflows id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY workflows ALTER COLUMN id SET DEFAULT nextval('workflows_id_seq'::regclass);
+ALTER TABLE ONLY public.workflows ALTER COLUMN id SET DEFAULT nextval('public.workflows_id_seq'::regclass);
--
-- Name: api_client_authorizations api_client_authorizations_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY api_client_authorizations
+ALTER TABLE ONLY public.api_client_authorizations
ADD CONSTRAINT api_client_authorizations_pkey PRIMARY KEY (id);
-- Name: api_clients api_clients_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY api_clients
+ALTER TABLE ONLY public.api_clients
ADD CONSTRAINT api_clients_pkey PRIMARY KEY (id);
-- Name: authorized_keys authorized_keys_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY authorized_keys
+ALTER TABLE ONLY public.authorized_keys
ADD CONSTRAINT authorized_keys_pkey PRIMARY KEY (id);
-- Name: collections collections_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY collections
+ALTER TABLE ONLY public.collections
ADD CONSTRAINT collections_pkey PRIMARY KEY (id);
-- Name: commit_ancestors commit_ancestors_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY commit_ancestors
+ALTER TABLE ONLY public.commit_ancestors
ADD CONSTRAINT commit_ancestors_pkey PRIMARY KEY (id);
-- Name: commits commits_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY commits
+ALTER TABLE ONLY public.commits
ADD CONSTRAINT commits_pkey PRIMARY KEY (id);
-- Name: container_requests container_requests_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY container_requests
+ALTER TABLE ONLY public.container_requests
ADD CONSTRAINT container_requests_pkey PRIMARY KEY (id);
-- Name: containers containers_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY containers
+ALTER TABLE ONLY public.containers
ADD CONSTRAINT containers_pkey PRIMARY KEY (id);
-- Name: groups groups_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY groups
+ALTER TABLE ONLY public.groups
ADD CONSTRAINT groups_pkey PRIMARY KEY (id);
-- Name: humans humans_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY humans
+ALTER TABLE ONLY public.humans
ADD CONSTRAINT humans_pkey PRIMARY KEY (id);
-- Name: job_tasks job_tasks_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY job_tasks
+ALTER TABLE ONLY public.job_tasks
ADD CONSTRAINT job_tasks_pkey PRIMARY KEY (id);
-- Name: jobs jobs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY jobs
+ALTER TABLE ONLY public.jobs
ADD CONSTRAINT jobs_pkey PRIMARY KEY (id);
-- Name: keep_disks keep_disks_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY keep_disks
+ALTER TABLE ONLY public.keep_disks
ADD CONSTRAINT keep_disks_pkey PRIMARY KEY (id);
-- Name: keep_services keep_services_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY keep_services
+ALTER TABLE ONLY public.keep_services
ADD CONSTRAINT keep_services_pkey PRIMARY KEY (id);
-- Name: links links_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY links
+ALTER TABLE ONLY public.links
ADD CONSTRAINT links_pkey PRIMARY KEY (id);
-- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY logs
+ALTER TABLE ONLY public.logs
ADD CONSTRAINT logs_pkey PRIMARY KEY (id);
-- Name: nodes nodes_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY nodes
+ALTER TABLE ONLY public.nodes
ADD CONSTRAINT nodes_pkey PRIMARY KEY (id);
-- Name: permission_refresh_lock permission_refresh_lock_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY permission_refresh_lock
+ALTER TABLE ONLY public.permission_refresh_lock
ADD CONSTRAINT permission_refresh_lock_pkey PRIMARY KEY (id);
-- Name: pipeline_instances pipeline_instances_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY pipeline_instances
+ALTER TABLE ONLY public.pipeline_instances
ADD CONSTRAINT pipeline_instances_pkey PRIMARY KEY (id);
-- Name: pipeline_templates pipeline_templates_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY pipeline_templates
+ALTER TABLE ONLY public.pipeline_templates
ADD CONSTRAINT pipeline_templates_pkey PRIMARY KEY (id);
-- Name: repositories repositories_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY repositories
+ALTER TABLE ONLY public.repositories
ADD CONSTRAINT repositories_pkey PRIMARY KEY (id);
-- Name: specimens specimens_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY specimens
+ALTER TABLE ONLY public.specimens
ADD CONSTRAINT specimens_pkey PRIMARY KEY (id);
-- Name: traits traits_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY traits
+ALTER TABLE ONLY public.traits
ADD CONSTRAINT traits_pkey PRIMARY KEY (id);
-- Name: users users_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY users
+ALTER TABLE ONLY public.users
ADD CONSTRAINT users_pkey PRIMARY KEY (id);
-- Name: virtual_machines virtual_machines_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY virtual_machines
+ALTER TABLE ONLY public.virtual_machines
ADD CONSTRAINT virtual_machines_pkey PRIMARY KEY (id);
-- Name: workflows workflows_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY workflows
+ALTER TABLE ONLY public.workflows
ADD CONSTRAINT workflows_pkey PRIMARY KEY (id);
-- Name: api_client_authorizations_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX api_client_authorizations_search_index ON api_client_authorizations USING btree (api_token, created_by_ip_address, last_used_by_ip_address, default_owner_uuid, uuid);
+CREATE INDEX api_client_authorizations_search_index ON public.api_client_authorizations USING btree (api_token, created_by_ip_address, last_used_by_ip_address, default_owner_uuid, uuid);
--
-- Name: api_clients_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX api_clients_search_index ON api_clients USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, name, url_prefix);
+CREATE INDEX api_clients_search_index ON public.api_clients USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, name, url_prefix);
--
-- Name: authorized_keys_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX authorized_keys_search_index ON authorized_keys USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, name, key_type, authorized_user_uuid);
+CREATE INDEX authorized_keys_search_index ON public.authorized_keys USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, name, key_type, authorized_user_uuid);
--
-- Name: collection_index_on_properties; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX collection_index_on_properties ON collections USING gin (properties);
+CREATE INDEX collection_index_on_properties ON public.collections USING gin (properties);
--
-- Name: collections_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX collections_full_text_search_idx ON collections USING gin (to_tsvector('english'::regconfig, (((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE((properties)::text, ''::text)) || ' '::text) || (COALESCE(file_names, ''::character varying))::text)));
+CREATE INDEX collections_full_text_search_idx ON public.collections USING gin (to_tsvector('english'::regconfig, (((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE((properties)::text, ''::text)) || ' '::text) || COALESCE(file_names, (''::character varying)::text))));
--
-- Name: collections_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX collections_search_index ON collections USING btree (owner_uuid, modified_by_client_uuid, modified_by_user_uuid, portable_data_hash, uuid, name);
+CREATE INDEX collections_search_index ON public.collections USING btree (owner_uuid, modified_by_client_uuid, modified_by_user_uuid, portable_data_hash, uuid, name);
--
-- Name: container_requests_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX container_requests_full_text_search_idx ON container_requests USING gin (to_tsvector('english'::regconfig, (((((((((((((((((((((((((((((((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || COALESCE(description, ''::text)) || ' '::text) || COALESCE(properties, ''::text)) || ' '::text) || (COALESCE(state, ''::character varying))::text) || ' '::text) || (COALESCE(requesting_container_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(container_uuid, ''::character varying))::text) || ' '::text) || COALESCE(runtime_constraints, ''::text)) || ' '::text) || (COALESCE(container_image, ''::character varying))::text) || ' '::text) || COALESCE(environment, ''::text)) || ' '::text) || (COALESCE(cwd, ''::character varying))::text) || ' '::text) || COALESCE(command, ''::text)) || ' '::text) || (COALESCE(output_path, ''::character varying))::text) || ' '::text) || COALESCE(filters, ''::text)) || ' '::text) || COALESCE(scheduling_parameters, ''::text)) || ' '::text) || (COALESCE(output_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(log_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(output_name, ''::character varying))::text)));
+CREATE INDEX container_requests_full_text_search_idx ON public.container_requests USING gin (to_tsvector('english'::regconfig, (((((((((((((((((((((((((((((((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || COALESCE(description, ''::text)) || ' '::text) || COALESCE((properties)::text, ''::text)) || ' '::text) || (COALESCE(state, ''::character varying))::text) || ' '::text) || (COALESCE(requesting_container_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(container_uuid, ''::character varying))::text) || ' '::text) || COALESCE(runtime_constraints, ''::text)) || ' '::text) || (COALESCE(container_image, ''::character varying))::text) || ' '::text) || COALESCE(environment, ''::text)) || ' '::text) || (COALESCE(cwd, ''::character varying))::text) || ' '::text) || COALESCE(command, ''::text)) || ' '::text) || (COALESCE(output_path, ''::character varying))::text) || ' '::text) || COALESCE(filters, ''::text)) || ' '::text) || COALESCE(scheduling_parameters, ''::text)) || ' '::text) || (COALESCE(output_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(log_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(output_name, ''::character varying))::text)));
+
+
+--
+-- Name: container_requests_index_on_properties; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX container_requests_index_on_properties ON public.container_requests USING gin (properties);
--
-- Name: container_requests_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX container_requests_search_index ON container_requests USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, name, state, requesting_container_uuid, container_uuid, container_image, cwd, output_path, output_uuid, log_uuid, output_name);
+CREATE INDEX container_requests_search_index ON public.container_requests USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, name, state, requesting_container_uuid, container_uuid, container_image, cwd, output_path, output_uuid, log_uuid, output_name);
--
-- Name: containers_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX containers_search_index ON containers USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, state, log, cwd, output_path, output, container_image, auth_uuid, locked_by_uuid);
+CREATE INDEX containers_search_index ON public.containers USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, state, log, cwd, output_path, output, container_image, auth_uuid, locked_by_uuid);
+
+
+--
+-- Name: group_index_on_properties; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX group_index_on_properties ON public.groups USING gin (properties);
--
-- Name: groups_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX groups_full_text_search_idx ON groups USING gin (to_tsvector('english'::regconfig, (((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || (COALESCE(group_class, ''::character varying))::text)));
+CREATE INDEX groups_full_text_search_idx ON public.groups USING gin (to_tsvector('english'::regconfig, (((((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || (COALESCE(group_class, ''::character varying))::text) || ' '::text) || COALESCE((properties)::text, ''::text))));
--
-- Name: groups_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX groups_search_index ON groups USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, name, group_class);
+CREATE INDEX groups_search_index ON public.groups USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, name, group_class);
--
-- Name: humans_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX humans_search_index ON humans USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid);
+CREATE INDEX humans_search_index ON public.humans USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid);
--
-- Name: index_api_client_authorizations_on_api_client_id; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_api_client_authorizations_on_api_client_id ON api_client_authorizations USING btree (api_client_id);
+CREATE INDEX index_api_client_authorizations_on_api_client_id ON public.api_client_authorizations USING btree (api_client_id);
--
-- Name: index_api_client_authorizations_on_api_token; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_api_client_authorizations_on_api_token ON api_client_authorizations USING btree (api_token);
+CREATE UNIQUE INDEX index_api_client_authorizations_on_api_token ON public.api_client_authorizations USING btree (api_token);
--
-- Name: index_api_client_authorizations_on_expires_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_api_client_authorizations_on_expires_at ON api_client_authorizations USING btree (expires_at);
+CREATE INDEX index_api_client_authorizations_on_expires_at ON public.api_client_authorizations USING btree (expires_at);
--
-- Name: index_api_client_authorizations_on_user_id; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_api_client_authorizations_on_user_id ON api_client_authorizations USING btree (user_id);
+CREATE INDEX index_api_client_authorizations_on_user_id ON public.api_client_authorizations USING btree (user_id);
--
-- Name: index_api_client_authorizations_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_api_client_authorizations_on_uuid ON api_client_authorizations USING btree (uuid);
+CREATE UNIQUE INDEX index_api_client_authorizations_on_uuid ON public.api_client_authorizations USING btree (uuid);
--
-- Name: index_api_clients_on_created_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_api_clients_on_created_at ON api_clients USING btree (created_at);
+CREATE INDEX index_api_clients_on_created_at ON public.api_clients USING btree (created_at);
--
-- Name: index_api_clients_on_modified_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_api_clients_on_modified_at ON api_clients USING btree (modified_at);
+CREATE INDEX index_api_clients_on_modified_at ON public.api_clients USING btree (modified_at);
--
-- Name: index_api_clients_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_api_clients_on_owner_uuid ON api_clients USING btree (owner_uuid);
+CREATE INDEX index_api_clients_on_owner_uuid ON public.api_clients USING btree (owner_uuid);
--
-- Name: index_api_clients_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_api_clients_on_uuid ON api_clients USING btree (uuid);
+CREATE UNIQUE INDEX index_api_clients_on_uuid ON public.api_clients USING btree (uuid);
--
-- Name: index_authkeys_on_user_and_expires_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_authkeys_on_user_and_expires_at ON authorized_keys USING btree (authorized_user_uuid, expires_at);
+CREATE INDEX index_authkeys_on_user_and_expires_at ON public.authorized_keys USING btree (authorized_user_uuid, expires_at);
--
-- Name: index_authorized_keys_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_authorized_keys_on_owner_uuid ON authorized_keys USING btree (owner_uuid);
+CREATE INDEX index_authorized_keys_on_owner_uuid ON public.authorized_keys USING btree (owner_uuid);
--
-- Name: index_authorized_keys_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_authorized_keys_on_uuid ON authorized_keys USING btree (uuid);
+CREATE UNIQUE INDEX index_authorized_keys_on_uuid ON public.authorized_keys USING btree (uuid);
--
-- Name: index_collections_on_created_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_collections_on_created_at ON collections USING btree (created_at);
+CREATE INDEX index_collections_on_created_at ON public.collections USING btree (created_at);
--
-- Name: index_collections_on_delete_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_collections_on_delete_at ON collections USING btree (delete_at);
+CREATE INDEX index_collections_on_delete_at ON public.collections USING btree (delete_at);
--
-- Name: index_collections_on_is_trashed; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_collections_on_is_trashed ON collections USING btree (is_trashed);
+CREATE INDEX index_collections_on_is_trashed ON public.collections USING btree (is_trashed);
--
-- Name: index_collections_on_modified_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_collections_on_modified_at ON collections USING btree (modified_at);
+CREATE INDEX index_collections_on_modified_at ON public.collections USING btree (modified_at);
--
-- Name: index_collections_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_collections_on_modified_at_uuid ON collections USING btree (modified_at DESC, uuid);
+CREATE INDEX index_collections_on_modified_at_uuid ON public.collections USING btree (modified_at DESC, uuid);
--
-- Name: index_collections_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_collections_on_owner_uuid ON collections USING btree (owner_uuid);
+CREATE INDEX index_collections_on_owner_uuid ON public.collections USING btree (owner_uuid);
--
-- Name: index_collections_on_owner_uuid_and_name; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_collections_on_owner_uuid_and_name ON collections USING btree (owner_uuid, name) WHERE (is_trashed = false);
+CREATE UNIQUE INDEX index_collections_on_owner_uuid_and_name ON public.collections USING btree (owner_uuid, name) WHERE (is_trashed = false);
--
-- Name: index_collections_on_portable_data_hash; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_collections_on_portable_data_hash ON collections USING btree (portable_data_hash);
+CREATE INDEX index_collections_on_portable_data_hash ON public.collections USING btree (portable_data_hash);
--
-- Name: index_collections_on_trash_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_collections_on_trash_at ON collections USING btree (trash_at);
+CREATE INDEX index_collections_on_trash_at ON public.collections USING btree (trash_at);
--
-- Name: index_collections_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_collections_on_uuid ON collections USING btree (uuid);
+CREATE UNIQUE INDEX index_collections_on_uuid ON public.collections USING btree (uuid);
--
-- Name: index_commit_ancestors_on_descendant_and_ancestor; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_commit_ancestors_on_descendant_and_ancestor ON commit_ancestors USING btree (descendant, ancestor);
+CREATE UNIQUE INDEX index_commit_ancestors_on_descendant_and_ancestor ON public.commit_ancestors USING btree (descendant, ancestor);
--
-- Name: index_commits_on_repository_name_and_sha1; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_commits_on_repository_name_and_sha1 ON commits USING btree (repository_name, sha1);
+CREATE UNIQUE INDEX index_commits_on_repository_name_and_sha1 ON public.commits USING btree (repository_name, sha1);
--
-- Name: index_container_requests_on_container_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_container_requests_on_container_uuid ON container_requests USING btree (container_uuid);
+CREATE INDEX index_container_requests_on_container_uuid ON public.container_requests USING btree (container_uuid);
--
-- Name: index_container_requests_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_container_requests_on_modified_at_uuid ON container_requests USING btree (modified_at DESC, uuid);
+CREATE INDEX index_container_requests_on_modified_at_uuid ON public.container_requests USING btree (modified_at DESC, uuid);
--
-- Name: index_container_requests_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_container_requests_on_owner_uuid ON container_requests USING btree (owner_uuid);
+CREATE INDEX index_container_requests_on_owner_uuid ON public.container_requests USING btree (owner_uuid);
--
-- Name: index_container_requests_on_requesting_container_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_container_requests_on_requesting_container_uuid ON container_requests USING btree (requesting_container_uuid);
+CREATE INDEX index_container_requests_on_requesting_container_uuid ON public.container_requests USING btree (requesting_container_uuid);
--
-- Name: index_container_requests_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_container_requests_on_uuid ON container_requests USING btree (uuid);
+CREATE UNIQUE INDEX index_container_requests_on_uuid ON public.container_requests USING btree (uuid);
--
-- Name: index_containers_on_auth_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_containers_on_auth_uuid ON containers USING btree (auth_uuid);
+CREATE INDEX index_containers_on_auth_uuid ON public.containers USING btree (auth_uuid);
--
-- Name: index_containers_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_containers_on_modified_at_uuid ON containers USING btree (modified_at DESC, uuid);
+CREATE INDEX index_containers_on_modified_at_uuid ON public.containers USING btree (modified_at DESC, uuid);
--
-- Name: index_containers_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_containers_on_owner_uuid ON containers USING btree (owner_uuid);
+CREATE INDEX index_containers_on_owner_uuid ON public.containers USING btree (owner_uuid);
--
-- Name: index_containers_on_secret_mounts_md5; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_containers_on_secret_mounts_md5 ON containers USING btree (secret_mounts_md5);
+CREATE INDEX index_containers_on_secret_mounts_md5 ON public.containers USING btree (secret_mounts_md5);
--
-- Name: index_containers_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_containers_on_uuid ON containers USING btree (uuid);
+CREATE UNIQUE INDEX index_containers_on_uuid ON public.containers USING btree (uuid);
--
-- Name: index_groups_on_created_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_groups_on_created_at ON groups USING btree (created_at);
+CREATE INDEX index_groups_on_created_at ON public.groups USING btree (created_at);
--
-- Name: index_groups_on_delete_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_groups_on_delete_at ON groups USING btree (delete_at);
+CREATE INDEX index_groups_on_delete_at ON public.groups USING btree (delete_at);
--
-- Name: index_groups_on_group_class; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_groups_on_group_class ON groups USING btree (group_class);
+CREATE INDEX index_groups_on_group_class ON public.groups USING btree (group_class);
--
-- Name: index_groups_on_is_trashed; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_groups_on_is_trashed ON groups USING btree (is_trashed);
+CREATE INDEX index_groups_on_is_trashed ON public.groups USING btree (is_trashed);
--
-- Name: index_groups_on_modified_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_groups_on_modified_at ON groups USING btree (modified_at);
+CREATE INDEX index_groups_on_modified_at ON public.groups USING btree (modified_at);
--
-- Name: index_groups_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_groups_on_modified_at_uuid ON groups USING btree (modified_at DESC, uuid);
+CREATE INDEX index_groups_on_modified_at_uuid ON public.groups USING btree (modified_at DESC, uuid);
--
-- Name: index_groups_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_groups_on_owner_uuid ON groups USING btree (owner_uuid);
+CREATE INDEX index_groups_on_owner_uuid ON public.groups USING btree (owner_uuid);
--
-- Name: index_groups_on_owner_uuid_and_name; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_groups_on_owner_uuid_and_name ON groups USING btree (owner_uuid, name) WHERE (is_trashed = false);
+CREATE UNIQUE INDEX index_groups_on_owner_uuid_and_name ON public.groups USING btree (owner_uuid, name) WHERE (is_trashed = false);
--
-- Name: index_groups_on_trash_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_groups_on_trash_at ON groups USING btree (trash_at);
+CREATE INDEX index_groups_on_trash_at ON public.groups USING btree (trash_at);
--
-- Name: index_groups_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_groups_on_uuid ON groups USING btree (uuid);
+CREATE UNIQUE INDEX index_groups_on_uuid ON public.groups USING btree (uuid);
--
-- Name: index_humans_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_humans_on_owner_uuid ON humans USING btree (owner_uuid);
+CREATE INDEX index_humans_on_owner_uuid ON public.humans USING btree (owner_uuid);
--
-- Name: index_humans_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_humans_on_uuid ON humans USING btree (uuid);
+CREATE UNIQUE INDEX index_humans_on_uuid ON public.humans USING btree (uuid);
--
-- Name: index_job_tasks_on_created_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_job_tasks_on_created_at ON job_tasks USING btree (created_at);
+CREATE INDEX index_job_tasks_on_created_at ON public.job_tasks USING btree (created_at);
--
-- Name: index_job_tasks_on_created_by_job_task_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_job_tasks_on_created_by_job_task_uuid ON job_tasks USING btree (created_by_job_task_uuid);
+CREATE INDEX index_job_tasks_on_created_by_job_task_uuid ON public.job_tasks USING btree (created_by_job_task_uuid);
--
-- Name: index_job_tasks_on_job_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_job_tasks_on_job_uuid ON job_tasks USING btree (job_uuid);
+CREATE INDEX index_job_tasks_on_job_uuid ON public.job_tasks USING btree (job_uuid);
--
-- Name: index_job_tasks_on_modified_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_job_tasks_on_modified_at ON job_tasks USING btree (modified_at);
+CREATE INDEX index_job_tasks_on_modified_at ON public.job_tasks USING btree (modified_at);
--
-- Name: index_job_tasks_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_job_tasks_on_owner_uuid ON job_tasks USING btree (owner_uuid);
+CREATE INDEX index_job_tasks_on_owner_uuid ON public.job_tasks USING btree (owner_uuid);
--
-- Name: index_job_tasks_on_sequence; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_job_tasks_on_sequence ON job_tasks USING btree (sequence);
+CREATE INDEX index_job_tasks_on_sequence ON public.job_tasks USING btree (sequence);
--
-- Name: index_job_tasks_on_success; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_job_tasks_on_success ON job_tasks USING btree (success);
+CREATE INDEX index_job_tasks_on_success ON public.job_tasks USING btree (success);
--
-- Name: index_job_tasks_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_job_tasks_on_uuid ON job_tasks USING btree (uuid);
+CREATE UNIQUE INDEX index_job_tasks_on_uuid ON public.job_tasks USING btree (uuid);
--
-- Name: index_jobs_on_created_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_jobs_on_created_at ON jobs USING btree (created_at);
+CREATE INDEX index_jobs_on_created_at ON public.jobs USING btree (created_at);
--
-- Name: index_jobs_on_finished_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_jobs_on_finished_at ON jobs USING btree (finished_at);
+CREATE INDEX index_jobs_on_finished_at ON public.jobs USING btree (finished_at);
--
-- Name: index_jobs_on_modified_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_jobs_on_modified_at ON jobs USING btree (modified_at);
+CREATE INDEX index_jobs_on_modified_at ON public.jobs USING btree (modified_at);
--
-- Name: index_jobs_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_jobs_on_modified_at_uuid ON jobs USING btree (modified_at DESC, uuid);
+CREATE INDEX index_jobs_on_modified_at_uuid ON public.jobs USING btree (modified_at DESC, uuid);
--
-- Name: index_jobs_on_output; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_jobs_on_output ON jobs USING btree (output);
+CREATE INDEX index_jobs_on_output ON public.jobs USING btree (output);
--
-- Name: index_jobs_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_jobs_on_owner_uuid ON jobs USING btree (owner_uuid);
+CREATE INDEX index_jobs_on_owner_uuid ON public.jobs USING btree (owner_uuid);
--
-- Name: index_jobs_on_script; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_jobs_on_script ON jobs USING btree (script);
+CREATE INDEX index_jobs_on_script ON public.jobs USING btree (script);
--
-- Name: index_jobs_on_script_parameters_digest; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_jobs_on_script_parameters_digest ON jobs USING btree (script_parameters_digest);
+CREATE INDEX index_jobs_on_script_parameters_digest ON public.jobs USING btree (script_parameters_digest);
--
-- Name: index_jobs_on_started_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_jobs_on_started_at ON jobs USING btree (started_at);
+CREATE INDEX index_jobs_on_started_at ON public.jobs USING btree (started_at);
--
-- Name: index_jobs_on_submit_id; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_jobs_on_submit_id ON jobs USING btree (submit_id);
+CREATE UNIQUE INDEX index_jobs_on_submit_id ON public.jobs USING btree (submit_id);
--
-- Name: index_jobs_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_jobs_on_uuid ON jobs USING btree (uuid);
+CREATE UNIQUE INDEX index_jobs_on_uuid ON public.jobs USING btree (uuid);
--
-- Name: index_keep_disks_on_filesystem_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_keep_disks_on_filesystem_uuid ON keep_disks USING btree (filesystem_uuid);
+CREATE INDEX index_keep_disks_on_filesystem_uuid ON public.keep_disks USING btree (filesystem_uuid);
--
-- Name: index_keep_disks_on_last_ping_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_keep_disks_on_last_ping_at ON keep_disks USING btree (last_ping_at);
+CREATE INDEX index_keep_disks_on_last_ping_at ON public.keep_disks USING btree (last_ping_at);
--
-- Name: index_keep_disks_on_node_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_keep_disks_on_node_uuid ON keep_disks USING btree (node_uuid);
+CREATE INDEX index_keep_disks_on_node_uuid ON public.keep_disks USING btree (node_uuid);
--
-- Name: index_keep_disks_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_keep_disks_on_owner_uuid ON keep_disks USING btree (owner_uuid);
+CREATE INDEX index_keep_disks_on_owner_uuid ON public.keep_disks USING btree (owner_uuid);
--
-- Name: index_keep_disks_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_keep_disks_on_uuid ON keep_disks USING btree (uuid);
+CREATE UNIQUE INDEX index_keep_disks_on_uuid ON public.keep_disks USING btree (uuid);
--
-- Name: index_keep_services_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_keep_services_on_owner_uuid ON keep_services USING btree (owner_uuid);
+CREATE INDEX index_keep_services_on_owner_uuid ON public.keep_services USING btree (owner_uuid);
--
-- Name: index_keep_services_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_keep_services_on_uuid ON keep_services USING btree (uuid);
+CREATE UNIQUE INDEX index_keep_services_on_uuid ON public.keep_services USING btree (uuid);
--
-- Name: index_links_on_created_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_links_on_created_at ON links USING btree (created_at);
+CREATE INDEX index_links_on_created_at ON public.links USING btree (created_at);
--
-- Name: index_links_on_head_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_links_on_head_uuid ON links USING btree (head_uuid);
+CREATE INDEX index_links_on_head_uuid ON public.links USING btree (head_uuid);
--
-- Name: index_links_on_modified_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_links_on_modified_at ON links USING btree (modified_at);
+CREATE INDEX index_links_on_modified_at ON public.links USING btree (modified_at);
--
-- Name: index_links_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_links_on_modified_at_uuid ON links USING btree (modified_at DESC, uuid);
+CREATE INDEX index_links_on_modified_at_uuid ON public.links USING btree (modified_at DESC, uuid);
--
-- Name: index_links_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_links_on_owner_uuid ON links USING btree (owner_uuid);
+CREATE INDEX index_links_on_owner_uuid ON public.links USING btree (owner_uuid);
--
-- Name: index_links_on_tail_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_links_on_tail_uuid ON links USING btree (tail_uuid);
+CREATE INDEX index_links_on_tail_uuid ON public.links USING btree (tail_uuid);
--
-- Name: index_links_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_links_on_uuid ON links USING btree (uuid);
+CREATE UNIQUE INDEX index_links_on_uuid ON public.links USING btree (uuid);
--
-- Name: index_logs_on_created_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_logs_on_created_at ON logs USING btree (created_at);
+CREATE INDEX index_logs_on_created_at ON public.logs USING btree (created_at);
--
-- Name: index_logs_on_event_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_logs_on_event_at ON logs USING btree (event_at);
+CREATE INDEX index_logs_on_event_at ON public.logs USING btree (event_at);
--
-- Name: index_logs_on_event_type; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_logs_on_event_type ON logs USING btree (event_type);
+CREATE INDEX index_logs_on_event_type ON public.logs USING btree (event_type);
--
-- Name: index_logs_on_modified_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_logs_on_modified_at ON logs USING btree (modified_at);
+CREATE INDEX index_logs_on_modified_at ON public.logs USING btree (modified_at);
--
-- Name: index_logs_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_logs_on_modified_at_uuid ON logs USING btree (modified_at DESC, uuid);
+CREATE INDEX index_logs_on_modified_at_uuid ON public.logs USING btree (modified_at DESC, uuid);
--
-- Name: index_logs_on_object_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_logs_on_object_owner_uuid ON logs USING btree (object_owner_uuid);
+CREATE INDEX index_logs_on_object_owner_uuid ON public.logs USING btree (object_owner_uuid);
--
-- Name: index_logs_on_object_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_logs_on_object_uuid ON logs USING btree (object_uuid);
+CREATE INDEX index_logs_on_object_uuid ON public.logs USING btree (object_uuid);
--
-- Name: index_logs_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_logs_on_owner_uuid ON logs USING btree (owner_uuid);
+CREATE INDEX index_logs_on_owner_uuid ON public.logs USING btree (owner_uuid);
--
-- Name: index_logs_on_summary; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_logs_on_summary ON logs USING btree (summary);
+CREATE INDEX index_logs_on_summary ON public.logs USING btree (summary);
--
-- Name: index_logs_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_logs_on_uuid ON logs USING btree (uuid);
+CREATE UNIQUE INDEX index_logs_on_uuid ON public.logs USING btree (uuid);
--
-- Name: index_nodes_on_created_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_nodes_on_created_at ON nodes USING btree (created_at);
+CREATE INDEX index_nodes_on_created_at ON public.nodes USING btree (created_at);
--
-- Name: index_nodes_on_hostname; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_nodes_on_hostname ON nodes USING btree (hostname);
+CREATE INDEX index_nodes_on_hostname ON public.nodes USING btree (hostname);
--
-- Name: index_nodes_on_modified_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_nodes_on_modified_at ON nodes USING btree (modified_at);
+CREATE INDEX index_nodes_on_modified_at ON public.nodes USING btree (modified_at);
--
-- Name: index_nodes_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_nodes_on_owner_uuid ON nodes USING btree (owner_uuid);
+CREATE INDEX index_nodes_on_owner_uuid ON public.nodes USING btree (owner_uuid);
--
-- Name: index_nodes_on_slot_number; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_nodes_on_slot_number ON nodes USING btree (slot_number);
+CREATE UNIQUE INDEX index_nodes_on_slot_number ON public.nodes USING btree (slot_number);
--
-- Name: index_nodes_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_nodes_on_uuid ON nodes USING btree (uuid);
+CREATE UNIQUE INDEX index_nodes_on_uuid ON public.nodes USING btree (uuid);
--
-- Name: index_pipeline_instances_on_created_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_pipeline_instances_on_created_at ON pipeline_instances USING btree (created_at);
+CREATE INDEX index_pipeline_instances_on_created_at ON public.pipeline_instances USING btree (created_at);
--
-- Name: index_pipeline_instances_on_modified_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_pipeline_instances_on_modified_at ON pipeline_instances USING btree (modified_at);
+CREATE INDEX index_pipeline_instances_on_modified_at ON public.pipeline_instances USING btree (modified_at);
--
-- Name: index_pipeline_instances_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_pipeline_instances_on_modified_at_uuid ON pipeline_instances USING btree (modified_at DESC, uuid);
+CREATE INDEX index_pipeline_instances_on_modified_at_uuid ON public.pipeline_instances USING btree (modified_at DESC, uuid);
--
-- Name: index_pipeline_instances_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_pipeline_instances_on_owner_uuid ON pipeline_instances USING btree (owner_uuid);
+CREATE INDEX index_pipeline_instances_on_owner_uuid ON public.pipeline_instances USING btree (owner_uuid);
--
-- Name: index_pipeline_instances_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_pipeline_instances_on_uuid ON pipeline_instances USING btree (uuid);
+CREATE UNIQUE INDEX index_pipeline_instances_on_uuid ON public.pipeline_instances USING btree (uuid);
--
-- Name: index_pipeline_templates_on_created_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_pipeline_templates_on_created_at ON pipeline_templates USING btree (created_at);
+CREATE INDEX index_pipeline_templates_on_created_at ON public.pipeline_templates USING btree (created_at);
--
-- Name: index_pipeline_templates_on_modified_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_pipeline_templates_on_modified_at ON pipeline_templates USING btree (modified_at);
+CREATE INDEX index_pipeline_templates_on_modified_at ON public.pipeline_templates USING btree (modified_at);
--
-- Name: index_pipeline_templates_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_pipeline_templates_on_modified_at_uuid ON pipeline_templates USING btree (modified_at DESC, uuid);
+CREATE INDEX index_pipeline_templates_on_modified_at_uuid ON public.pipeline_templates USING btree (modified_at DESC, uuid);
--
-- Name: index_pipeline_templates_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_pipeline_templates_on_owner_uuid ON pipeline_templates USING btree (owner_uuid);
+CREATE INDEX index_pipeline_templates_on_owner_uuid ON public.pipeline_templates USING btree (owner_uuid);
--
-- Name: index_pipeline_templates_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_pipeline_templates_on_uuid ON pipeline_templates USING btree (uuid);
+CREATE UNIQUE INDEX index_pipeline_templates_on_uuid ON public.pipeline_templates USING btree (uuid);
--
-- Name: index_repositories_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_repositories_on_modified_at_uuid ON repositories USING btree (modified_at DESC, uuid);
+CREATE INDEX index_repositories_on_modified_at_uuid ON public.repositories USING btree (modified_at DESC, uuid);
--
-- Name: index_repositories_on_name; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_repositories_on_name ON repositories USING btree (name);
+CREATE UNIQUE INDEX index_repositories_on_name ON public.repositories USING btree (name);
--
-- Name: index_repositories_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_repositories_on_owner_uuid ON repositories USING btree (owner_uuid);
+CREATE INDEX index_repositories_on_owner_uuid ON public.repositories USING btree (owner_uuid);
--
-- Name: index_repositories_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_repositories_on_uuid ON repositories USING btree (uuid);
+CREATE UNIQUE INDEX index_repositories_on_uuid ON public.repositories USING btree (uuid);
--
-- Name: index_specimens_on_created_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_specimens_on_created_at ON specimens USING btree (created_at);
+CREATE INDEX index_specimens_on_created_at ON public.specimens USING btree (created_at);
--
-- Name: index_specimens_on_modified_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_specimens_on_modified_at ON specimens USING btree (modified_at);
+CREATE INDEX index_specimens_on_modified_at ON public.specimens USING btree (modified_at);
--
-- Name: index_specimens_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_specimens_on_owner_uuid ON specimens USING btree (owner_uuid);
+CREATE INDEX index_specimens_on_owner_uuid ON public.specimens USING btree (owner_uuid);
--
-- Name: index_specimens_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_specimens_on_uuid ON specimens USING btree (uuid);
+CREATE UNIQUE INDEX index_specimens_on_uuid ON public.specimens USING btree (uuid);
--
-- Name: index_traits_on_name; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_traits_on_name ON traits USING btree (name);
+CREATE INDEX index_traits_on_name ON public.traits USING btree (name);
--
-- Name: index_traits_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_traits_on_owner_uuid ON traits USING btree (owner_uuid);
+CREATE INDEX index_traits_on_owner_uuid ON public.traits USING btree (owner_uuid);
--
-- Name: index_traits_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_traits_on_uuid ON traits USING btree (uuid);
+CREATE UNIQUE INDEX index_traits_on_uuid ON public.traits USING btree (uuid);
--
-- Name: index_users_on_created_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_users_on_created_at ON users USING btree (created_at);
+CREATE INDEX index_users_on_created_at ON public.users USING btree (created_at);
--
-- Name: index_users_on_modified_at; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_users_on_modified_at ON users USING btree (modified_at);
+CREATE INDEX index_users_on_modified_at ON public.users USING btree (modified_at);
--
-- Name: index_users_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_users_on_modified_at_uuid ON users USING btree (modified_at DESC, uuid);
+CREATE INDEX index_users_on_modified_at_uuid ON public.users USING btree (modified_at DESC, uuid);
--
-- Name: index_users_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_users_on_owner_uuid ON users USING btree (owner_uuid);
+CREATE INDEX index_users_on_owner_uuid ON public.users USING btree (owner_uuid);
--
-- Name: index_users_on_username; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_users_on_username ON users USING btree (username);
+CREATE UNIQUE INDEX index_users_on_username ON public.users USING btree (username);
--
-- Name: index_users_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_users_on_uuid ON users USING btree (uuid);
+CREATE UNIQUE INDEX index_users_on_uuid ON public.users USING btree (uuid);
--
-- Name: index_virtual_machines_on_hostname; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_virtual_machines_on_hostname ON virtual_machines USING btree (hostname);
+CREATE INDEX index_virtual_machines_on_hostname ON public.virtual_machines USING btree (hostname);
--
-- Name: index_virtual_machines_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_virtual_machines_on_modified_at_uuid ON virtual_machines USING btree (modified_at DESC, uuid);
+CREATE INDEX index_virtual_machines_on_modified_at_uuid ON public.virtual_machines USING btree (modified_at DESC, uuid);
--
-- Name: index_virtual_machines_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_virtual_machines_on_owner_uuid ON virtual_machines USING btree (owner_uuid);
+CREATE INDEX index_virtual_machines_on_owner_uuid ON public.virtual_machines USING btree (owner_uuid);
--
-- Name: index_virtual_machines_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_virtual_machines_on_uuid ON virtual_machines USING btree (uuid);
+CREATE UNIQUE INDEX index_virtual_machines_on_uuid ON public.virtual_machines USING btree (uuid);
--
-- Name: index_workflows_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_workflows_on_modified_at_uuid ON workflows USING btree (modified_at DESC, uuid);
+CREATE INDEX index_workflows_on_modified_at_uuid ON public.workflows USING btree (modified_at DESC, uuid);
--
-- Name: index_workflows_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_workflows_on_owner_uuid ON workflows USING btree (owner_uuid);
+CREATE INDEX index_workflows_on_owner_uuid ON public.workflows USING btree (owner_uuid);
--
-- Name: index_workflows_on_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX index_workflows_on_uuid ON workflows USING btree (uuid);
+CREATE UNIQUE INDEX index_workflows_on_uuid ON public.workflows USING btree (uuid);
--
-- Name: job_tasks_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX job_tasks_search_index ON job_tasks USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, job_uuid, created_by_job_task_uuid);
+CREATE INDEX job_tasks_search_index ON public.job_tasks USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, job_uuid, created_by_job_task_uuid);
--
-- Name: jobs_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX jobs_full_text_search_idx ON jobs USING gin (to_tsvector('english'::regconfig, (((((((((((((((((((((((((((((((((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(submit_id, ''::character varying))::text) || ' '::text) || (COALESCE(script, ''::character varying))::text) || ' '::text) || (COALESCE(script_version, ''::character varying))::text) || ' '::text) || COALESCE(script_parameters, ''::text)) || ' '::text) || (COALESCE(cancelled_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(cancelled_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(output, ''::character varying))::text) || ' '::text) || (COALESCE(is_locked_by_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(log, ''::character varying))::text) || ' '::text) || COALESCE(tasks_summary, ''::text)) || ' '::text) || COALESCE(runtime_constraints, ''::text)) || ' '::text) || (COALESCE(repository, ''::character varying))::text) || ' '::text) || (COALESCE(supplied_script_version, ''::character varying))::text) || ' '::text) || (COALESCE(docker_image_locator, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || (COALESCE(state, ''::character varying))::text) || ' '::text) || (COALESCE(arvados_sdk_version, ''::character varying))::text) || ' '::text) || COALESCE(components, ''::text))));
+CREATE INDEX jobs_full_text_search_idx ON public.jobs USING gin (to_tsvector('english'::regconfig, (((((((((((((((((((((((((((((((((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(submit_id, ''::character varying))::text) || ' '::text) || (COALESCE(script, ''::character varying))::text) || ' '::text) || (COALESCE(script_version, ''::character varying))::text) || ' '::text) || COALESCE(script_parameters, ''::text)) || ' '::text) || (COALESCE(cancelled_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(cancelled_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(output, ''::character varying))::text) || ' '::text) || (COALESCE(is_locked_by_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(log, ''::character varying))::text) || ' '::text) || COALESCE(tasks_summary, ''::text)) || ' '::text) || COALESCE(runtime_constraints, ''::text)) || ' '::text) || (COALESCE(repository, ''::character varying))::text) || ' '::text) || (COALESCE(supplied_script_version, ''::character varying))::text) || ' '::text) || (COALESCE(docker_image_locator, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || (COALESCE(state, ''::character varying))::text) || ' '::text) || (COALESCE(arvados_sdk_version, ''::character varying))::text) || ' '::text) || COALESCE(components, ''::text))));
--
-- Name: jobs_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX jobs_search_index ON jobs USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, submit_id, script, script_version, cancelled_by_client_uuid, cancelled_by_user_uuid, output, is_locked_by_uuid, log, repository, supplied_script_version, docker_image_locator, state, arvados_sdk_version);
+CREATE INDEX jobs_search_index ON public.jobs USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, submit_id, script, script_version, cancelled_by_client_uuid, cancelled_by_user_uuid, output, is_locked_by_uuid, log, repository, supplied_script_version, docker_image_locator, state, arvados_sdk_version);
--
-- Name: keep_disks_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX keep_disks_search_index ON keep_disks USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, ping_secret, node_uuid, filesystem_uuid, keep_service_uuid);
+CREATE INDEX keep_disks_search_index ON public.keep_disks USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, ping_secret, node_uuid, filesystem_uuid, keep_service_uuid);
--
-- Name: keep_services_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX keep_services_search_index ON keep_services USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, service_host, service_type);
+CREATE INDEX keep_services_search_index ON public.keep_services USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, service_host, service_type);
+
+
+--
+-- Name: links_index_on_properties; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX links_index_on_properties ON public.links USING gin (properties);
--
-- Name: links_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX links_search_index ON links USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, tail_uuid, link_class, name, head_uuid);
+CREATE INDEX links_search_index ON public.links USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, tail_uuid, link_class, name, head_uuid);
--
-- Name: links_tail_name_unique_if_link_class_name; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX links_tail_name_unique_if_link_class_name ON links USING btree (tail_uuid, name) WHERE ((link_class)::text = 'name'::text);
+CREATE UNIQUE INDEX links_tail_name_unique_if_link_class_name ON public.links USING btree (tail_uuid, name) WHERE ((link_class)::text = 'name'::text);
--
-- Name: logs_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX logs_search_index ON logs USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, object_uuid, event_type, object_owner_uuid);
+CREATE INDEX logs_search_index ON public.logs USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, object_uuid, event_type, object_owner_uuid);
+
+
+--
+-- Name: nodes_index_on_info; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX nodes_index_on_info ON public.nodes USING gin (info);
+
+
+--
+-- Name: nodes_index_on_properties; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX nodes_index_on_properties ON public.nodes USING gin (properties);
--
-- Name: nodes_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX nodes_search_index ON nodes USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, hostname, domain, ip_address, job_uuid);
+CREATE INDEX nodes_search_index ON public.nodes USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, hostname, domain, ip_address, job_uuid);
--
-- Name: permission_target_trashed; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX permission_target_trashed ON materialized_permission_view USING btree (trashed, target_uuid);
+CREATE INDEX permission_target_trashed ON public.materialized_permission_view USING btree (trashed, target_uuid);
--
-- Name: permission_target_user_trashed_level; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX permission_target_user_trashed_level ON materialized_permission_view USING btree (user_uuid, trashed, perm_level);
+CREATE INDEX permission_target_user_trashed_level ON public.materialized_permission_view USING btree (user_uuid, trashed, perm_level);
--
-- Name: pipeline_instances_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX pipeline_instances_full_text_search_idx ON pipeline_instances USING gin (to_tsvector('english'::regconfig, (((((((((((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(pipeline_template_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || COALESCE(components, ''::text)) || ' '::text) || COALESCE(properties, ''::text)) || ' '::text) || (COALESCE(state, ''::character varying))::text) || ' '::text) || COALESCE(components_summary, ''::text)) || ' '::text) || (COALESCE(description, ''::character varying))::text)));
+CREATE INDEX pipeline_instances_full_text_search_idx ON public.pipeline_instances USING gin (to_tsvector('english'::regconfig, (((((((((((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(pipeline_template_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || COALESCE(components, ''::text)) || ' '::text) || COALESCE(properties, ''::text)) || ' '::text) || (COALESCE(state, ''::character varying))::text) || ' '::text) || COALESCE(components_summary, ''::text)) || ' '::text) || (COALESCE(description, ''::character varying))::text)));
--
-- Name: pipeline_instances_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX pipeline_instances_search_index ON pipeline_instances USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, pipeline_template_uuid, name, state);
+CREATE INDEX pipeline_instances_search_index ON public.pipeline_instances USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, pipeline_template_uuid, name, state);
--
-- Name: pipeline_template_owner_uuid_name_unique; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX pipeline_template_owner_uuid_name_unique ON pipeline_templates USING btree (owner_uuid, name);
+CREATE UNIQUE INDEX pipeline_template_owner_uuid_name_unique ON public.pipeline_templates USING btree (owner_uuid, name);
--
-- Name: pipeline_templates_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX pipeline_templates_full_text_search_idx ON pipeline_templates USING gin (to_tsvector('english'::regconfig, (((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || COALESCE(components, ''::text)) || ' '::text) || (COALESCE(description, ''::character varying))::text)));
+CREATE INDEX pipeline_templates_full_text_search_idx ON public.pipeline_templates USING gin (to_tsvector('english'::regconfig, (((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || COALESCE(components, ''::text)) || ' '::text) || (COALESCE(description, ''::character varying))::text)));
--
-- Name: pipeline_templates_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX pipeline_templates_search_index ON pipeline_templates USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, name);
+CREATE INDEX pipeline_templates_search_index ON public.pipeline_templates USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, name);
--
-- Name: repositories_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX repositories_search_index ON repositories USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, name);
+CREATE INDEX repositories_search_index ON public.repositories USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, name);
--
-- Name: specimens_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX specimens_search_index ON specimens USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, material);
+CREATE INDEX specimens_search_index ON public.specimens USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, material);
--
-- Name: traits_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX traits_search_index ON traits USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, name);
+CREATE INDEX traits_search_index ON public.traits USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, name);
--
-- Name: unique_schema_migrations; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX unique_schema_migrations ON schema_migrations USING btree (version);
+CREATE UNIQUE INDEX unique_schema_migrations ON public.schema_migrations USING btree (version);
--
-- Name: users_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX users_search_index ON users USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, email, first_name, last_name, identity_url, default_owner_uuid, username, redirect_to_user_uuid);
+CREATE INDEX users_search_index ON public.users USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, email, first_name, last_name, identity_url, default_owner_uuid, username, redirect_to_user_uuid);
--
-- Name: virtual_machines_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX virtual_machines_search_index ON virtual_machines USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, hostname);
+CREATE INDEX virtual_machines_search_index ON public.virtual_machines USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, hostname);
--
-- Name: workflows_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX workflows_full_text_search_idx ON workflows USING gin (to_tsvector('english'::regconfig, (((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || COALESCE(description, ''::text))));
+CREATE INDEX workflows_full_text_search_idx ON public.workflows USING gin (to_tsvector('english'::regconfig, (((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || COALESCE(description, ''::text))));
--
-- Name: workflows_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX workflows_search_idx ON workflows USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, name);
+CREATE INDEX workflows_search_idx ON public.workflows USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, name);
--
INSERT INTO schema_migrations (version) VALUES ('20180514135529');
+INSERT INTO schema_migrations (version) VALUES ('20180607175050');
+
+INSERT INTO schema_migrations (version) VALUES ('20180608123145');
+
+INSERT INTO schema_migrations (version) VALUES ('20180806133039');
+
def self.forget
@hash = nil
+ @package_version = nil
end
# Return abbrev commit hash for current code version: "abc1234", or
@hash || "unknown"
end
+
+ def self.package_version
+ if (cached = Rails.configuration.package_version || @package_version)
+ return cached
+ end
+
+ begin
+ @package_version = IO.read(Rails.root.join("package-build.version")).strip
+ rescue Errno::ENOENT
+ @package_version = "unknown"
+ end
+
+ @package_version
+ end
end
@docker_bin = ENV['CRUNCH_JOB_DOCKER_BIN']
@docker_run_args = ENV['CRUNCH_JOB_DOCKER_RUN_ARGS']
@cgroup_root = ENV['CRUNCH_CGROUP_ROOT']
+ @srun_sync_timeout = ENV['CRUNCH_SRUN_SYNC_TIMEOUT']
@arvados_internal = Rails.configuration.git_internal_dir
if not File.exist? @arvados_internal
@fetched_commits[sha1] = ($? == 0)
end
- def tag_commit(commit_hash, tag_name)
+ def tag_commit(job, commit_hash, tag_name)
# @git_tags[T]==V if we know commit V has been tagged T in the
# arvados_internal repository.
if not @git_tags[tag_name]
next
end
ready &&= get_commit repo.server_path, job.script_version
- ready &&= tag_commit job.script_version, job.uuid
+ ready &&= tag_commit job, job.script_version, job.uuid
end
# This should be unnecessary, because API server does it during
# job create/update, but it's still not a bad idea to verify the
# tag is correct before starting the job:
- ready &&= tag_commit job.script_version, job.uuid
+ ready &&= tag_commit job, job.script_version, job.uuid
# The arvados_sdk_version doesn't support use of arbitrary
# remote URLs, so the requested version isn't necessarily copied
# into the internal repository yet.
if job.arvados_sdk_version
ready &&= get_commit @arvados_repo_path, job.arvados_sdk_version
- ready &&= tag_commit job.arvados_sdk_version, "#{job.uuid}-arvados-sdk"
+ ready &&= tag_commit job, job.arvados_sdk_version, "#{job.uuid}-arvados-sdk"
end
if not ready
cmd_args += ['--docker-run-args', @docker_run_args]
end
+ if @srun_sync_timeout
+ cmd_args += ['--srun-sync-timeout', @srun_sync_timeout]
+ end
+
if have_job_lock?(job)
cmd_args << "--force-unlock"
end
# Load params[:limit], params[:offset] and params[:order]
# into @limit, @offset, @orders
- def load_limit_offset_order_params
+ def load_limit_offset_order_params(fill_table_names: true)
if params[:limit]
unless params[:limit].to_s.match(/^\d+$/)
raise ArgumentError.new("Invalid value for limit parameter")
# has used set_table_name to use an alternate table name from the Rails standard.
# I could not find a perfect way to handle this well, but ActiveRecord::Base.send(:descendants)
# would be a place to start if this ever becomes necessary.
- if attr.match(/^[a-z][_a-z0-9]+$/) and
- model_class.columns.collect(&:name).index(attr) and
- ['asc','desc'].index direction.downcase
- @orders << "#{table_name}.#{attr} #{direction.downcase}"
+ if (attr.match(/^[a-z][_a-z0-9]+$/) &&
+ model_class.columns.collect(&:name).index(attr) &&
+ ['asc','desc'].index(direction.downcase))
+ if fill_table_names
+ @orders << "#{table_name}.#{attr} #{direction.downcase}"
+ else
+ @orders << "#{attr} #{direction.downcase}"
+ end
elsif attr.match(/^([a-z][_a-z0-9]+)\.([a-z][_a-z0-9]+)$/) and
['asc','desc'].index(direction.downcase) and
ActiveRecord::Base.connection.tables.include?($1) and
return Oj.dump(o, mode: :compat)
end
def self.load(s)
+ if s.nil? or s == ''
+ # Oj 2.18.5 used to return nil. Not anymore on 3.6.4.
+ # Upgraded for performance issues (see #13803 and
+ # https://github.com/ohler55/oj/issues/441)
+ return nil
+ end
Oj.strict_load(s, symbol_keys: false)
end
end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+module UpdatePriority
+ extend CurrentApiClient
+
+ # Clean up after races.
+ #
+ # If container priority>0 but there are no committed container
+ # requests for it, reset priority to 0.
+ #
+ # If container priority=0 but there are committed container requests
+ # for it with priority>0, update priority.
+ def self.update_priority
+ if !File.owned?(Rails.root.join('tmp'))
+ Rails.logger.warn("UpdatePriority: not owner of #{Rails.root}/tmp, skipping")
+ return
+ end
+ lockfile = Rails.root.join('tmp', 'update_priority.lock')
+ File.open(lockfile, File::RDWR|File::CREAT, 0600) do |f|
+ return unless f.flock(File::LOCK_NB|File::LOCK_EX)
+
+ # priority>0 but should be 0:
+ ActiveRecord::Base.connection.
+ exec_query("UPDATE containers AS c SET priority=0 WHERE state IN ('Queued', 'Locked', 'Running') AND priority>0 AND uuid NOT IN (SELECT container_uuid FROM container_requests WHERE priority>0 AND state='Committed');", 'UpdatePriority')
+
+ # priority==0 but should be >0:
+ act_as_system_user do
+ Container.
+ joins("JOIN container_requests ON container_requests.container_uuid=containers.uuid AND container_requests.state=#{Container.sanitize(ContainerRequest::Committed)} AND container_requests.priority>0").
+ where('containers.state IN (?) AND containers.priority=0 AND container_requests.uuid IS NOT NULL',
+ [Container::Queued, Container::Locked, Container::Running]).
+ map(&:update_priority!)
+ end
+ end
+ end
+
+ def self.run_update_thread
+ need = false
+ Rails.cache.fetch('UpdatePriority', expires_in: 5.seconds) do
+ need = true
+ end
+ return if !need
+
+ Thread.new do
+ Thread.current.abort_on_exception = false
+ begin
+ update_priority
+ rescue => e
+ Rails.logger.error "#{e.class}: #{e}\n#{e.backtrace.join("\n\t")}"
+ ensure
+ ActiveRecord::Base.connection.close
+ end
+ end
+ end
+end
def check_update_whitelist permitted_fields
attribute_names.each do |field|
if !permitted_fields.include?(field.to_sym) && really_changed(field)
- errors.add field, "cannot be modified in this state (#{send(field+"_was").inspect}, #{send(field).inspect})"
+ errors.add field, "cannot be modified in state '#{self.state}' (#{send(field+"_was").inspect}, #{send(field).inspect})"
end
end
end
assert_includes ids, collections(:baz_file_in_asubproject).uuid
end
- [['asc', :<=],
- ['desc', :>=]].each do |order, operator|
- test "user with project read permission can sort project collections #{order}" do
+ [
+ ['collections.name', 'asc', :<=, "name"],
+ ['collections.name', 'desc', :>=, "name"],
+ ['name', 'asc', :<=, "name"],
+ ['name', 'desc', :>=, "name"],
+ ['collections.created_at', 'asc', :<=, "created_at"],
+ ['collections.created_at', 'desc', :>=, "created_at"],
+ ['created_at', 'asc', :<=, "created_at"],
+ ['created_at', 'desc', :>=, "created_at"],
+ ].each do |column, order, operator, field|
+ test "user with project read permission can sort projects on #{column} #{order}" do
authorize_with :project_viewer
get :contents, {
id: groups(:asubproject).uuid,
format: :json,
filters: [['uuid', 'is_a', "arvados#collection"]],
- order: "collections.name #{order}"
+ order: "#{column} #{order}"
}
- sorted_names = json_response['items'].collect { |item| item["name"] }
- # Here we avoid assuming too much about the database
- # collation. Both "alice"<"Bob" and "alice">"Bob" can be
- # correct. Hopefully it _is_ safe to assume that if "a" comes
- # before "b" in the ascii alphabet, "aX">"bY" is never true for
- # any strings X and Y.
- reliably_sortable_names = sorted_names.select do |name|
- name[0] >= 'a' and name[0] <= 'z'
- end.uniq do |name|
- name[0]
- end
- # Preserve order of sorted_names. But do not use &=. If
- # sorted_names has out-of-order duplicates, we want to preserve
- # them here, so we can detect them and fail the test below.
- sorted_names.select! do |name|
- reliably_sortable_names.include? name
- end
- actually_checked_anything = false
- previous = nil
- sorted_names.each do |entry|
- if previous
- assert_operator(previous, operator, entry,
- "Entries sorted incorrectly.")
- actually_checked_anything = true
+ sorted_values = json_response['items'].collect { |item| item[field] }
+ if field == "name"
+ # Here we avoid assuming too much about the database
+ # collation. Both "alice"<"Bob" and "alice">"Bob" can be
+ # correct. Hopefully it _is_ safe to assume that if "a" comes
+ # before "b" in the ascii alphabet, "aX">"bY" is never true for
+ # any strings X and Y.
+ reliably_sortable_names = sorted_values.select do |name|
+ name[0] >= 'a' && name[0] <= 'z'
+ end.uniq do |name|
+ name[0]
+ end
+ # Preserve order of sorted_values. But do not use &=. If
+ # sorted_values has out-of-order duplicates, we want to preserve
+ # them here, so we can detect them and fail the test below.
+ sorted_values.select! do |name|
+ reliably_sortable_names.include? name
end
- previous = entry
end
- assert actually_checked_anything, "Didn't even find two names to compare."
+ assert_sorted(operator, sorted_values)
+ end
+ end
+
+ def assert_sorted(operator, sorted_items)
+ actually_checked_anything = false
+ previous = nil
+ sorted_items.each do |entry|
+ if !previous.nil?
+ assert_operator(previous, operator, entry,
+ "Entries sorted incorrectly.")
+ actually_checked_anything = true
+ end
+ previous = entry
end
+ assert actually_checked_anything, "Didn't even find two items to compare."
end
test 'list objects across multiple projects' do
assert_includes discovery_doc, 'defaultTrashLifetime'
assert_equal discovery_doc['defaultTrashLifetime'], Rails.application.config.default_trash_lifetime
assert_match(/^[0-9a-f]+(-modified)?$/, discovery_doc['source_version'])
+ assert_match(/^[0-9a-f]+(-modified)?$/, discovery_doc['sourceVersion'])
+ assert_match(/^unknown$/, discovery_doc['packageVersion'])
assert_equal discovery_doc['websocketUrl'], Rails.application.config.websocket_address
assert_equal discovery_doc['workbenchUrl'], Rails.application.config.workbench_address
assert_equal('zzzzz', discovery_doc['uuidPrefix'])
end
- test "discovery document overrides source_version with config" do
+ test "discovery document overrides source_version & sourceVersion with config" do
Rails.configuration.source_version = 'aaa888fff'
get :index
assert_response :success
discovery_doc = JSON.parse(@response.body)
+ # Key source_version will be replaced with sourceVersion
assert_equal 'aaa888fff', discovery_doc['source_version']
+ assert_equal 'aaa888fff', discovery_doc['sourceVersion']
+ end
+
+ test "discovery document overrides packageVersion with config" do
+ Rails.configuration.package_version = '1.0.0-stable'
+ get :index
+ assert_response :success
+ discovery_doc = JSON.parse(@response.body)
+ assert_equal '1.0.0-stable', discovery_doc['packageVersion']
end
test "empty disable_api_methods" do
['/arvados/v1/collections',
'/arvados/v1/users',
'/arvados/v1/api_client_authorizations'].each do |path|
- test "CORS headers are set and body is stub at OPTIONS #{path}" do
+ test "CORS headers are set and body is empty at OPTIONS #{path}" do
options path, {}, {}
assert_response :success
assert_cors_headers
- assert_equal '-', response.body
+ assert_equal '', response.body
end
test "CORS headers are set at authenticated GET #{path}" do
assert_response :success
assert_equal 'zbbbb-tpzed-000000000000000', json_response['uuid']
assert_equal false, json_response['is_admin']
+ assert_equal false, json_response['is_active']
assert_equal 'foo@example.com', json_response['email']
assert_equal 'barney', json_response['username']
refute_includes(group_uuids, groups(:trashed_project).uuid)
refute_includes(group_uuids, groups(:testusergroup_admins).uuid)
end
+
+ test 'auto-activate user from trusted cluster' do
+ Rails.configuration.auto_activate_users_from = ['zbbbb']
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response :success
+ assert_equal 'zbbbb-tpzed-000000000000000', json_response['uuid']
+ assert_equal false, json_response['is_admin']
+ assert_equal true, json_response['is_active']
+ assert_equal 'foo@example.com', json_response['email']
+ assert_equal 'barney', json_response['username']
+ end
+
+ test 'pre-activate remote user' do
+ post '/arvados/v1/users', {
+ "user" => {
+ "uuid" => "zbbbb-tpzed-000000000000000",
+ "email" => 'foo@example.com',
+ "username" => 'barney',
+ "is_active" => true
+ }
+ }, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_token(:admin)}"}
+ assert_response :success
+
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response :success
+ assert_equal 'zbbbb-tpzed-000000000000000', json_response['uuid']
+ assert_equal nil, json_response['is_admin']
+ assert_equal true, json_response['is_active']
+ assert_equal 'foo@example.com', json_response['email']
+ assert_equal 'barney', json_response['username']
+ end
+
end
end
def self.slow_test(name, &block)
- define_method(name, block) unless skip_slow_tests?
+ test(name, &block) unless skip_slow_tests?
end
end
properties: {'foo' => 'bar'}.with_indifferent_access)
raw = ActiveRecord::Base.connection.
select_value("select properties from links where uuid='#{link.uuid}'")
- assert_equal '{"foo":"bar"}', raw
+ assert_equal '{"foo": "bar"}', raw
end
test "store long string" do
assert_equal ContainerRequest::Final, cr3.state
end
+ [
+ [false, ActiveRecord::RecordInvalid],
+ [true, nil],
+ ].each do |preemptible_conf, expected|
+ test "having Rails.configuration.preemptible_instances=#{preemptible_conf}, create preemptible container request and verify #{expected}" do
+ sp = {"preemptible" => true}
+ common_attrs = {cwd: "test",
+ priority: 1,
+ command: ["echo", "hello"],
+ output_path: "test",
+ scheduling_parameters: sp,
+ mounts: {"test" => {"kind" => "json"}}}
+ Rails.configuration.preemptible_instances = preemptible_conf
+ set_user_from_auth :active
+
+ cr = create_minimal_req!(common_attrs)
+ cr.state = ContainerRequest::Committed
+
+ if !expected.nil?
+ assert_raises(expected) do
+ cr.save!
+ end
+ else
+ cr.save!
+ assert_equal sp, cr.scheduling_parameters
+ end
+ end
+ end
+
+ [
+ 'zzzzz-dz642-runningcontainr',
+ nil,
+ ].each do |requesting_c|
+ test "having preemptible instances active on the API server, a committed #{requesting_c.nil? ? 'non-':''}child CR should not ask for preemptible instance if parameter already set to false" do
+ common_attrs = {cwd: "test",
+ priority: 1,
+ command: ["echo", "hello"],
+ output_path: "test",
+ scheduling_parameters: {"preemptible" => false},
+ mounts: {"test" => {"kind" => "json"}}}
+
+ Rails.configuration.preemptible_instances = true
+ set_user_from_auth :active
+
+ if requesting_c
+ cr = with_container_auth(Container.find_by_uuid requesting_c) do
+ create_minimal_req!(common_attrs)
+ end
+ assert_not_nil cr.requesting_container_uuid
+ else
+ cr = create_minimal_req!(common_attrs)
+ end
+
+ cr.state = ContainerRequest::Committed
+ cr.save!
+
+ assert_equal false, cr.scheduling_parameters['preemptible']
+ end
+ end
+
+ [
+ [true, 'zzzzz-dz642-runningcontainr', true],
+ [true, nil, nil],
+ [false, 'zzzzz-dz642-runningcontainr', nil],
+ [false, nil, nil],
+ ].each do |preemptible_conf, requesting_c, schedule_preemptible|
+ test "having Rails.configuration.preemptible_instances=#{preemptible_conf}, #{requesting_c.nil? ? 'non-':''}child CR should #{schedule_preemptible ? '':'not'} ask for preemptible instance by default" do
+ common_attrs = {cwd: "test",
+ priority: 1,
+ command: ["echo", "hello"],
+ output_path: "test",
+ mounts: {"test" => {"kind" => "json"}}}
+
+ Rails.configuration.preemptible_instances = preemptible_conf
+ set_user_from_auth :active
+
+ if requesting_c
+ cr = with_container_auth(Container.find_by_uuid requesting_c) do
+ create_minimal_req!(common_attrs)
+ end
+ assert_not_nil cr.requesting_container_uuid
+ else
+ cr = create_minimal_req!(common_attrs)
+ end
+
+ cr.state = ContainerRequest::Committed
+ cr.save!
+
+ assert_equal schedule_preemptible, cr.scheduling_parameters['preemptible']
+ end
+ end
+
[
[{"partitions" => ["fastcpu","vfastcpu", 100]}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
[{"partitions" => ["fastcpu","vfastcpu", 100]}, ContainerRequest::Uncommitted],
[{"partitions" => "fastcpu"}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
[{"partitions" => "fastcpu"}, ContainerRequest::Uncommitted],
[{"partitions" => ["fastcpu","vfastcpu"]}, ContainerRequest::Committed],
+ [{"max_run_time" => "one day"}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
+ [{"max_run_time" => "one day"}, ContainerRequest::Uncommitted],
+ [{"max_run_time" => -1}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
+ [{"max_run_time" => -1}, ContainerRequest::Uncommitted],
+ [{"max_run_time" => 86400}, ContainerRequest::Committed],
].each do |sp, state, expected|
test "create container request with scheduling_parameters #{sp} in state #{state} and verify #{expected}" do
common_attrs = {cwd: "test",
end
end
+ test "Having preemptible_instances=true create a committed child container request and verify the scheduling parameter of its container" do
+ common_attrs = {cwd: "test",
+ priority: 1,
+ command: ["echo", "hello"],
+ output_path: "test",
+ state: ContainerRequest::Committed,
+ mounts: {"test" => {"kind" => "json"}}}
+ set_user_from_auth :active
+ Rails.configuration.preemptible_instances = true
+
+ cr = with_container_auth(Container.find_by_uuid 'zzzzz-dz642-runningcontainr') do
+ create_minimal_req!(common_attrs)
+ end
+ assert_equal 'zzzzz-dz642-runningcontainr', cr.requesting_container_uuid
+ assert_equal true, cr.scheduling_parameters["preemptible"]
+
+ c = Container.find_by_uuid(cr.container_uuid)
+ assert_equal true, c.scheduling_parameters["preemptible"]
+ end
+
[['Committed', true, {name: "foobar", priority: 123}],
['Committed', false, {container_count: 2}],
['Committed', false, {container_count: 0}],
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'test_helper'
+require 'update_priority'
+
+class UpdatePriorityTest < ActiveSupport::TestCase
+ test 'priority 0 but should be >0' do
+ uuid = containers(:running).uuid
+ ActiveRecord::Base.connection.exec_query('UPDATE containers SET priority=0 WHERE uuid=$1', 'test-setup', [[nil, uuid]])
+ assert_equal 0, Container.find_by_uuid(uuid).priority
+ UpdatePriority.update_priority
+ assert_operator 0, :<, Container.find_by_uuid(uuid).priority
+
+ uuid = containers(:queued).uuid
+ ActiveRecord::Base.connection.exec_query('UPDATE containers SET priority=0 WHERE uuid=$1', 'test-setup', [[nil, uuid]])
+ assert_equal 0, Container.find_by_uuid(uuid).priority
+ UpdatePriority.update_priority
+ assert_operator 0, :<, Container.find_by_uuid(uuid).priority
+ end
+
+ test 'priority>0 but should be 0' do
+ uuid = containers(:running).uuid
+ ActiveRecord::Base.connection.exec_query('DELETE FROM container_requests WHERE container_uuid=$1', 'test-setup', [[nil, uuid]])
+ assert_operator 0, :<, Container.find_by_uuid(uuid).priority
+ UpdatePriority.update_priority
+ assert_equal 0, Container.find_by_uuid(uuid).priority
+ end
+end
assert_equal(expect_username, user.username)
# check user setup
- verify_link_exists(Rails.configuration.auto_setup_new_users,
+ verify_link_exists(Rails.configuration.auto_setup_new_users || active,
groups(:all_users).uuid, user.uuid,
"permission", "can_read")
# Check for OID login link.
- verify_link_exists(Rails.configuration.auto_setup_new_users,
+ verify_link_exists(Rails.configuration.auto_setup_new_users || active,
user.uuid, user.email, "permission", "can_login")
# Check for repository.
if named_repo = (prior_repo or
// Dispatcher service for Crunch that submits containers to the slurm queue.
import (
+ "bytes"
"context"
"flag"
"fmt"
crArgs = append(crArgs, container.UUID)
crScript := strings.NewReader(execScript(crArgs))
- disp.sqCheck.L.Lock()
- defer disp.sqCheck.L.Unlock()
-
sbArgs, err := disp.sbatchArgs(container)
if err != nil {
return err
log.Printf("Submitting container %s to slurm", ctr.UUID)
if err := disp.submit(ctr, disp.CrunchRunCommand); err != nil {
var text string
- if err == dispatchcloud.ErrConstraintsNotSatisfiable {
- text = fmt.Sprintf("cannot run container %s: %s", ctr.UUID, err)
+ if err, ok := err.(dispatchcloud.ConstraintsNotSatisfiableError); ok {
+ var logBuf bytes.Buffer
+ fmt.Fprintf(&logBuf, "cannot run container %s: %s\n", ctr.UUID, err)
+ if len(err.AvailableTypes) == 0 {
+ fmt.Fprint(&logBuf, "No instance types are configured.\n")
+ } else {
+ fmt.Fprint(&logBuf, "Available instance types:\n")
+ for _, t := range err.AvailableTypes {
+ fmt.Fprintf(&logBuf,
+ "Type %q: %d VCPUs, %d RAM, %d Scratch, %f Price\n",
+ t.Name, t.VCPUs, t.RAM, t.Scratch, t.Price,
+ )
+ }
+ }
+ text = logBuf.String()
disp.UpdateState(ctr.UUID, dispatch.Cancelled)
} else {
text = fmt.Sprintf("Error submitting container %s to slurm: %s", ctr.UUID, err)
}
}
func (disp *Dispatcher) scancel(ctr arvados.Container) {
- disp.sqCheck.L.Lock()
err := disp.slurm.Cancel(ctr.UUID)
- disp.sqCheck.L.Unlock()
-
if err != nil {
log.Printf("scancel: %s", err)
time.Sleep(time.Second)
}
type slurmFake struct {
- didBatch [][]string
- didCancel []string
- didRelease []string
- didRenice [][]string
- queue string
+ didBatch [][]string
+ didCancel []string
+ didRelease []string
+ didRenice [][]string
+ queue string
+ rejectNice10K bool
// If non-nil, run this func during the 2nd+ call to Cancel()
onCancel func()
// Error returned by Batch()
func (sf *slurmFake) Renice(name string, nice int64) error {
sf.didRenice = append(sf.didRenice, []string{name, fmt.Sprintf("%d", nice)})
+ if sf.rejectNice10K && nice > 10000 {
+ return errors.New("scontrol: error: Invalid nice value, must be between -10000 and 10000")
+ }
return nil
}
var containers arvados.ContainerList
err = arv.List("containers", params, &containers)
c.Check(err, IsNil)
- c.Check(len(containers.Items), Equals, 1)
+ c.Assert(len(containers.Items), Equals, 1)
s.disp.CrunchRunCommand = []string{"echo"}
}
for _, trial := range []struct {
- types []arvados.InstanceType
+ types map[string]arvados.InstanceType
sbatchArgs []string
err error
}{
// Choose node type => use --constraint arg
{
- types: []arvados.InstanceType{
- {Name: "a1.tiny", Price: 0.02, RAM: 128000000, VCPUs: 1},
- {Name: "a1.small", Price: 0.04, RAM: 256000000, VCPUs: 2},
- {Name: "a1.medium", Price: 0.08, RAM: 512000000, VCPUs: 4},
- {Name: "a1.large", Price: 0.16, RAM: 1024000000, VCPUs: 8},
+ types: map[string]arvados.InstanceType{
+ "a1.tiny": {Name: "a1.tiny", Price: 0.02, RAM: 128000000, VCPUs: 1},
+ "a1.small": {Name: "a1.small", Price: 0.04, RAM: 256000000, VCPUs: 2},
+ "a1.medium": {Name: "a1.medium", Price: 0.08, RAM: 512000000, VCPUs: 4},
+ "a1.large": {Name: "a1.large", Price: 0.16, RAM: 1024000000, VCPUs: 8},
},
sbatchArgs: []string{"--constraint=instancetype=a1.medium"},
},
},
// No node type is big enough => error
{
- types: []arvados.InstanceType{
- {Name: "a1.tiny", Price: 0.02, RAM: 128000000, VCPUs: 1},
+ types: map[string]arvados.InstanceType{
+ "a1.tiny": {Name: "a1.tiny", Price: 0.02, RAM: 128000000, VCPUs: 1},
},
- err: dispatchcloud.ErrConstraintsNotSatisfiable,
+ err: dispatchcloud.ConstraintsNotSatisfiableError{},
},
} {
c.Logf("%#v", trial)
s.disp.cluster = &arvados.Cluster{InstanceTypes: trial.types}
args, err := s.disp.sbatchArgs(container)
- c.Check(err, Equals, trial.err)
+ c.Check(err == nil, Equals, trial.err == nil)
if trial.err == nil {
c.Check(args, DeepEquals, append([]string{"--job-name=123", "--nice=10000"}, trial.sbatchArgs...))
+ } else {
+ c.Check(len(err.(dispatchcloud.ConstraintsNotSatisfiableError).AvailableTypes), Equals, len(trial.types))
}
}
}
"time"
)
+const slurm15NiceLimit int64 = 10000
+
type slurmJob struct {
uuid string
wantPriority int64
priority int64 // current slurm priority (incorporates nice value)
nice int64 // current slurm nice value
+ hitNiceLimit bool
}
// Squeue implements asynchronous polling monitor of the SLURM queue using the
queue map[string]*slurmJob
startOnce sync.Once
done chan struct{}
- sync.Cond
+ lock sync.RWMutex
+ notify sync.Cond
}
// HasUUID checks if a given container UUID is in the slurm queue.
func (sqc *SqueueChecker) HasUUID(uuid string) bool {
sqc.startOnce.Do(sqc.start)
- sqc.L.Lock()
- defer sqc.L.Unlock()
+ sqc.lock.RLock()
+ defer sqc.lock.RUnlock()
// block until next squeue broadcast signaling an update.
- sqc.Wait()
+ sqc.notify.Wait()
_, exists := sqc.queue[uuid]
return exists
}
// container.
func (sqc *SqueueChecker) SetPriority(uuid string, want int64) {
sqc.startOnce.Do(sqc.start)
- sqc.L.Lock()
- defer sqc.L.Unlock()
- job, ok := sqc.queue[uuid]
- if !ok {
+
+ sqc.lock.RLock()
+ job := sqc.queue[uuid]
+ if job == nil {
// Wait in case the slurm job was just submitted and
// will appear in the next squeue update.
- sqc.Wait()
- if job, ok = sqc.queue[uuid]; !ok {
- return
- }
+ sqc.notify.Wait()
+ job = sqc.queue[uuid]
+ }
+ needUpdate := job != nil && job.wantPriority != want
+ sqc.lock.RUnlock()
+
+ if needUpdate {
+ sqc.lock.Lock()
+ job.wantPriority = want
+ sqc.lock.Unlock()
}
- job.wantPriority = want
}
// adjust slurm job nice values as needed to ensure slurm priority
// order matches Arvados priority order.
func (sqc *SqueueChecker) reniceAll() {
- sqc.L.Lock()
- defer sqc.L.Unlock()
+ sqc.lock.RLock()
+ defer sqc.lock.RUnlock()
jobs := make([]*slurmJob, 0, len(sqc.queue))
for _, j := range sqc.queue {
// (perhaps it's not an Arvados job)
continue
}
- if j.priority == 0 {
+ if j.priority <= 2*slurm15NiceLimit {
// SLURM <= 15.x implements "hold" by setting
// priority to 0. If we include held jobs
// here, we'll end up trying to push other
})
renice := wantNice(jobs, sqc.PrioritySpread)
for i, job := range jobs {
- if renice[i] == job.nice {
+ niceNew := renice[i]
+ if job.hitNiceLimit && niceNew > slurm15NiceLimit {
+ niceNew = slurm15NiceLimit
+ }
+ if niceNew == job.nice {
continue
}
- sqc.Slurm.Renice(job.uuid, renice[i])
+ err := sqc.Slurm.Renice(job.uuid, niceNew)
+ if err != nil && niceNew > slurm15NiceLimit && strings.Contains(err.Error(), "Invalid nice value") {
+ log.Printf("container %q clamping nice values at %d, priority order will not be correct -- see https://dev.arvados.org/projects/arvados/wiki/SLURM_integration#Limited-nice-values-SLURM-15", job.uuid, slurm15NiceLimit)
+ job.hitNiceLimit = true
+ }
}
}
// queued). If it succeeds, it updates sqc.queue and wakes up any
// goroutines that are waiting in HasUUID() or All().
func (sqc *SqueueChecker) check() {
- // Mutex between squeue sync and running sbatch or scancel. This
- // establishes a sequence so that squeue doesn't run concurrently with
- // sbatch or scancel; the next update of squeue will occur only after
- // sbatch or scancel has completed.
- sqc.L.Lock()
- defer sqc.L.Unlock()
+ sqc.lock.Lock()
+ defer sqc.lock.Unlock()
cmd := sqc.Slurm.QueueCommand([]string{"--all", "--noheader", "--format=%j %y %Q %T %r"})
stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
replacing.nice = n
newq[uuid] = replacing
- if state == "PENDING" && ((reason == "BadConstraints" && p == 0) || reason == "launch failed requeued held") && replacing.wantPriority > 0 {
+ if state == "PENDING" && ((reason == "BadConstraints" && p <= 2*slurm15NiceLimit) || reason == "launch failed requeued held") && replacing.wantPriority > 0 {
// When using SLURM 14.x or 15.x, our queued
// jobs land in this state when "scontrol
// reconfigure" invalidates their feature
// constraints by clearing all node features.
// They stay in this state even after the
// features reappear, until we run "scontrol
- // release {jobid}".
+ // release {jobid}". Priority is usually 0 in
+ // this state, but sometimes (due to a race
+ // with nice adjustments?) it's a small
+ // positive value.
//
// "scontrol release" is silent and successful
// regardless of whether the features have
// "launch failed requeued held" seems to be
// another manifestation of this problem,
// resolved the same way.
- log.Printf("releasing held job %q", uuid)
+ log.Printf("releasing held job %q (priority=%d, state=%q, reason=%q)", uuid, p, state, reason)
sqc.Slurm.Release(uuid)
- } else if p < 1<<20 && replacing.wantPriority > 0 {
+ } else if state != "RUNNING" && p <= 2*slurm15NiceLimit && replacing.wantPriority > 0 {
log.Printf("warning: job %q has low priority %d, nice %d, state %q, reason %q", uuid, p, n, state, reason)
}
}
sqc.queue = newq
- sqc.Broadcast()
+ sqc.notify.Broadcast()
}
// Initialize, and start a goroutine to call check() once per
// squeue.Period until terminated by calling Stop().
func (sqc *SqueueChecker) start() {
- sqc.L = &sync.Mutex{}
+ sqc.notify.L = sqc.lock.RLocker()
sqc.done = make(chan struct{})
go func() {
ticker := time.NewTicker(sqc.Period)
case <-ticker.C:
sqc.check()
sqc.reniceAll()
+ select {
+ case <-ticker.C:
+ // If this iteration took
+ // longer than sqc.Period,
+ // consume the next tick and
+ // wait. Otherwise we would
+ // starve other goroutines.
+ default:
+ }
}
}
}()
// names reported by squeue.
func (sqc *SqueueChecker) All() []string {
sqc.startOnce.Do(sqc.start)
- sqc.L.Lock()
- defer sqc.L.Unlock()
- sqc.Wait()
+ sqc.lock.RLock()
+ defer sqc.lock.RUnlock()
+ sqc.notify.Wait()
var uuids []string
for u := range sqc.queue {
uuids = append(uuids, u)
}
}
+// If a limited nice range prevents desired priority adjustments, give
+// up and clamp nice to 10K.
+func (s *SqueueSuite) TestReniceInvalidNiceValue(c *C) {
+ uuids := []string{"zzzzz-dz642-fake0fake0fake0", "zzzzz-dz642-fake1fake1fake1", "zzzzz-dz642-fake2fake2fake2"}
+ slurm := &slurmFake{
+ queue: uuids[0] + " 0 4294000222 PENDING Resources\n" + uuids[1] + " 0 4294555222 PENDING Resources\n",
+ rejectNice10K: true,
+ }
+ sqc := &SqueueChecker{
+ Slurm: slurm,
+ PrioritySpread: 1,
+ Period: time.Hour,
+ }
+ sqc.startOnce.Do(sqc.start)
+ sqc.check()
+ sqc.SetPriority(uuids[0], 2)
+ sqc.SetPriority(uuids[1], 1)
+
+ // First attempt should renice to 555001, which will fail
+ sqc.reniceAll()
+ c.Check(slurm.didRenice, DeepEquals, [][]string{{uuids[1], "555001"}})
+
+ // Next attempt should renice to 10K, which will succeed
+ sqc.reniceAll()
+ c.Check(slurm.didRenice, DeepEquals, [][]string{{uuids[1], "555001"}, {uuids[1], "10000"}})
+ // ...so we'll change the squeue response to reflect the
+ // updated priority+nice, and make sure sqc sees that...
+ slurm.queue = uuids[0] + " 0 4294000222 PENDING Resources\n" + uuids[1] + " 10000 4294545222 PENDING Resources\n"
+ sqc.check()
+
+ // Next attempt should leave nice alone because it's already
+ // at the 10K limit
+ sqc.reniceAll()
+ c.Check(slurm.didRenice, DeepEquals, [][]string{{uuids[1], "555001"}, {uuids[1], "10000"}})
+
+ // Back to normal if desired nice value falls below 10K
+ slurm.queue = uuids[0] + " 0 4294000222 PENDING Resources\n" + uuids[1] + " 10000 4294000111 PENDING Resources\n"
+ sqc.check()
+ sqc.reniceAll()
+ c.Check(slurm.didRenice, DeepEquals, [][]string{{uuids[1], "555001"}, {uuids[1], "10000"}, {uuids[1], "9890"}})
+
+ sqc.Stop()
+}
+
// If the given UUID isn't in the slurm queue yet, SetPriority()
// should wait for it to appear on the very next poll, then give up.
func (s *SqueueSuite) TestSetPriorityBeforeQueued(c *C) {
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
"git.curoverse.com/arvados.git/sdk/go/manifest"
+ "github.com/shirou/gopsutil/process"
"golang.org/x/net/context"
dockertypes "github.com/docker/docker/api/types"
ImageRemove(ctx context.Context, image string, options dockertypes.ImageRemoveOptions) ([]dockertypes.ImageDeleteResponseItem, error)
}
+type PsProcess interface {
+ CmdlineSlice() ([]string, error)
+}
+
// ContainerRunner is the main stateful struct used for a single execution of a
// container.
type ContainerRunner struct {
finalState string
parentTemp string
+ ListProcesses func() ([]PsProcess, error)
+
statLogger io.WriteCloser
statReporter *crunchstat.Reporter
hoststatLogger io.WriteCloser
cStateLock sync.Mutex
cCancelled bool // StopContainer() invoked
- enableNetwork string // one of "default" or "always"
- networkMode string // passed through to HostConfig.NetworkMode
- arvMountLog *ThrottledLogger
+ enableNetwork string // one of "default" or "always"
+ networkMode string // passed through to HostConfig.NetworkMode
+ arvMountLog *ThrottledLogger
+ checkContainerd time.Duration
}
// setupSignals sets up signal handling to gracefully terminate the underlying
var errorBlacklist = []string{
"(?ms).*[Cc]annot connect to the Docker daemon.*",
"(?ms).*oci runtime error.*starting container process.*container init.*mounting.*to rootfs.*no such file or directory.*",
+ "(?ms).*grpc: the connection is unavailable.*",
}
var brokenNodeHook *string = flag.String("broken-node-hook", "", "Script to run if node is detected to be broken (for example, Docker daemon is not running)")
+func (runner *ContainerRunner) runBrokenNodeHook() {
+ if *brokenNodeHook == "" {
+ runner.CrunchLog.Printf("No broken node hook provided, cannot mark node as broken.")
+ } else {
+ runner.CrunchLog.Printf("Running broken node hook %q", *brokenNodeHook)
+ // run killme script
+ c := exec.Command(*brokenNodeHook)
+ c.Stdout = runner.CrunchLog
+ c.Stderr = runner.CrunchLog
+ err := c.Run()
+ if err != nil {
+ runner.CrunchLog.Printf("Error running broken node hook: %v", err)
+ }
+ }
+}
+
func (runner *ContainerRunner) checkBrokenNode(goterr error) bool {
for _, d := range errorBlacklist {
if m, e := regexp.MatchString(d, goterr.Error()); m && e == nil {
runner.CrunchLog.Printf("Error suggests node is unable to run containers: %v", goterr)
- if *brokenNodeHook == "" {
- runner.CrunchLog.Printf("No broken node hook provided, cannot mark node as broken.")
- } else {
- runner.CrunchLog.Printf("Running broken node hook %q", *brokenNodeHook)
- // run killme script
- c := exec.Command(*brokenNodeHook)
- c.Stdout = runner.CrunchLog
- c.Stderr = runner.CrunchLog
- err := c.Run()
- if err != nil {
- runner.CrunchLog.Printf("Error running broken node hook: %v", err)
- }
- }
+ runner.runBrokenNodeHook()
return true
}
}
CgroupParent: runner.expectCgroupParent,
CgroupRoot: runner.cgroupRoot,
PollPeriod: runner.statInterval,
+ TempDir: runner.parentTemp,
}
runner.statReporter.Start()
return nil
runner.ContainerConfig.Volumes = runner.Volumes
maxRAM := int64(runner.Container.RuntimeConstraints.RAM)
+ if maxRAM < 4*1024*1024 {
+ // Docker daemon won't let you set a limit less than 4 MiB
+ maxRAM = 4 * 1024 * 1024
+ }
runner.HostConfig = dockercontainer.HostConfig{
Binds: runner.Binds,
LogConfig: dockercontainer.LogConfig{
return nil
}
+// checkContainerd checks if "containerd" is present in the process list.
+func (runner *ContainerRunner) CheckContainerd() error {
+ if runner.checkContainerd == 0 {
+ return nil
+ }
+ p, _ := runner.ListProcesses()
+ for _, i := range p {
+ e, _ := i.CmdlineSlice()
+ if len(e) > 0 {
+ if strings.Index(e[0], "containerd") > -1 {
+ return nil
+ }
+ }
+ }
+
+ // Not found
+ runner.runBrokenNodeHook()
+ runner.stop(nil)
+ return fmt.Errorf("'containerd' not found in process list.")
+}
+
// WaitFinish waits for the container to terminate, capture the exit code, and
// close the stdout/stderr logging.
func (runner *ContainerRunner) WaitFinish() error {
+ var runTimeExceeded <-chan time.Time
runner.CrunchLog.Print("Waiting for container to finish")
waitOk, waitErr := runner.Docker.ContainerWait(context.TODO(), runner.ContainerID, dockercontainer.WaitConditionNotRunning)
arvMountExit := runner.ArvMountExit
+ if timeout := runner.Container.SchedulingParameters.MaxRunTime; timeout > 0 {
+ runTimeExceeded = time.After(time.Duration(timeout) * time.Second)
+ }
+
+ containerdGone := make(chan error)
+ defer close(containerdGone)
+ if runner.checkContainerd > 0 {
+ go func() {
+ ticker := time.NewTicker(time.Duration(runner.checkContainerd))
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ticker.C:
+ if ck := runner.CheckContainerd(); ck != nil {
+ containerdGone <- ck
+ return
+ }
+ case <-containerdGone:
+ // Channel closed, quit goroutine
+ return
+ }
+ }
+ }()
+ }
+
for {
select {
case waitBody := <-waitOk:
// arvMountExit will always be ready now that
// it's closed, but that doesn't interest us.
arvMountExit = nil
+
+ case <-runTimeExceeded:
+ runner.CrunchLog.Printf("maximum run time exceeded. Stopping container.")
+ runner.stop(nil)
+ runTimeExceeded = nil
+
+ case err := <-containerdGone:
+ return err
}
}
}
return
}
+ // Sanity check that containerd is running.
+ err = runner.CheckContainerd()
+ if err != nil {
+ return
+ }
+
// check for and/or load image
err = runner.LoadImage()
if err != nil {
cr.NewLogWriter = cr.NewArvLogWriter
cr.RunArvMount = cr.ArvMountCmd
cr.MkTempDir = ioutil.TempDir
+ cr.ListProcesses = func() ([]PsProcess, error) {
+ pr, err := process.Processes()
+ if err != nil {
+ return nil, err
+ }
+ ps := make([]PsProcess, len(pr))
+ for i, j := range pr {
+ ps[i] = j
+ }
+ return ps, nil
+ }
cr.MkArvClient = func(token string) (IArvadosClient, error) {
cl, err := arvadosclient.MakeArvadosClient()
if err != nil {
`)
memprofile := flag.String("memprofile", "", "write memory profile to `file` after running container")
getVersion := flag.Bool("version", false, "Print version information and exit.")
+ checkContainerd := flag.Duration("check-containerd", 60*time.Second, "Periodic check if (docker-)containerd is running (use 0s to disable).")
flag.Parse()
// Print version information if requested
cr.expectCgroupParent = *cgroupParent
cr.enableNetwork = *enableNetwork
cr.networkMode = *networkMode
+ cr.checkContainerd = *checkContainerd
if *cgroupParentSubsystem != "" {
p := findCgroup(*cgroupParentSubsystem)
cr.setCgroupParent = p
"mounts": {"/tmp": {"kind": "tmp"} },
"output_path": "/tmp",
"priority": 1,
- "runtime_constraints": {}
+ "runtime_constraints": {}
}`, nil, 0, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, "hello world\n"))
t.logWriter.Close()
}
+func (s *TestSuite) TestRunTimeExceeded(c *C) {
+ api, _, _ := s.fullRunHelper(c, `{
+ "command": ["sleep", "3"],
+ "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "cwd": ".",
+ "environment": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
+ "output_path": "/tmp",
+ "priority": 1,
+ "runtime_constraints": {},
+ "scheduling_parameters":{"max_run_time": 1}
+}`, nil, 0, func(t *TestDockerClient) {
+ time.Sleep(3 * time.Second)
+ t.logWriter.Close()
+ })
+
+ c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
+ c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
+}
+
func (s *TestSuite) TestCrunchstat(c *C) {
api, _, _ := s.fullRunHelper(c, `{
"command": ["sleep", "1"],
c.Check(api.CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
c.Check(api.CalledWith("collection.manifest_text", ""), NotNil)
}
+
+type FakeProcess struct {
+ cmdLine []string
+}
+
+func (fp FakeProcess) CmdlineSlice() ([]string, error) {
+ return fp.cmdLine, nil
+}
+
+func (s *TestSuite) helpCheckContainerd(c *C, lp func() ([]PsProcess, error)) error {
+ kc := &KeepTestClient{}
+ defer kc.Close()
+ cr, err := NewContainerRunner(s.client, &ArvTestClient{callraw: true}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ cr.checkContainerd = time.Duration(100 * time.Millisecond)
+ c.Assert(err, IsNil)
+ cr.ListProcesses = lp
+
+ s.docker.fn = func(t *TestDockerClient) {
+ time.Sleep(1 * time.Second)
+ t.logWriter.Close()
+ }
+
+ err = cr.CreateContainer()
+ c.Check(err, IsNil)
+
+ err = cr.StartContainer()
+ c.Check(err, IsNil)
+
+ err = cr.WaitFinish()
+ return err
+
+}
+
+func (s *TestSuite) TestCheckContainerdPresent(c *C) {
+ err := s.helpCheckContainerd(c, func() ([]PsProcess, error) {
+ return []PsProcess{FakeProcess{[]string{"docker-containerd"}}}, nil
+ })
+ c.Check(err, IsNil)
+}
+
+func (s *TestSuite) TestCheckContainerdMissing(c *C) {
+ err := s.helpCheckContainerd(c, func() ([]PsProcess, error) {
+ return []PsProcess{FakeProcess{[]string{"abc"}}}, nil
+ })
+ c.Check(err, ErrorMatches, `'containerd' not found in process list.`)
+}
cr.CrunchLog.Print("Goodbye")
cr.CrunchLog.Close()
- c.Check(api.Calls > 1, Equals, true)
+ c.Check(api.Calls > 0, Equals, true)
c.Check(api.Calls < 2000000, Equals, true)
mt, err := cr.LogCollection.MarshalManifest(".")
}
// Funnel stderr through our channel
- stderr_pipe, err := cmd.StderrPipe()
+ stderrPipe, err := cmd.StderrPipe()
if err != nil {
logger.Fatalln("error in StderrPipe:", err)
}
os.Stdin.Close()
os.Stdout.Close()
- copyPipeToChildLog(stderr_pipe, log.New(os.Stderr, "", 0))
+ copyPipeToChildLog(stderrPipe, log.New(os.Stderr, "", 0))
return cmd.Wait()
}
# llfuse 1.3.4 fails to install via pip
'llfuse >=1.2, <1.3.4',
'python-daemon',
- 'ciso8601 >=1.0.6',
+ 'ciso8601 >=1.0.6, <2.0.0',
'setuptools'
],
test_suite='tests',
if err != nil {
log.Fatal(err)
}
- nodeCfg, err := clusterCfg.GetThisSystemNode()
+ nodeCfg, err := clusterCfg.GetNodeProfile("")
if err != nil {
log.Fatal(err)
}
return
}
}
+ bal.cleanupMounts()
if err = bal.CheckSanityEarly(&config.Client); err != nil {
return
})
}
+func (bal *Balancer) cleanupMounts() {
+ rwdev := map[string]*KeepService{}
+ for _, srv := range bal.KeepServices {
+ for _, mnt := range srv.mounts {
+ if !mnt.ReadOnly && mnt.DeviceID != "" {
+ rwdev[mnt.DeviceID] = srv
+ }
+ }
+ }
+ // Drop the readonly mounts whose device is mounted RW
+ // elsewhere.
+ for _, srv := range bal.KeepServices {
+ var dedup []*KeepMount
+ for _, mnt := range srv.mounts {
+ if mnt.ReadOnly && rwdev[mnt.DeviceID] != nil {
+ bal.logf("skipping srv %s readonly mount %q because same device %q is mounted read-write on srv %s", srv, mnt.UUID, mnt.DeviceID, rwdev[mnt.DeviceID])
+ } else {
+ dedup = append(dedup, mnt)
+ }
+ }
+ srv.mounts = dedup
+ }
+ for _, srv := range bal.KeepServices {
+ for _, mnt := range srv.mounts {
+ if mnt.Replication <= 0 {
+ log.Printf("%s: mount %s reports replication=%d, using replication=1", srv, mnt.UUID, mnt.Replication)
+ mnt.Replication = 1
+ }
+ }
+ }
+}
+
// CheckSanityEarly checks for configuration and runtime errors that
// can be detected before GetCurrentState() and ComputeChangeSets()
// are called.
errs := make(chan error, 2+len(bal.KeepServices))
wg := sync.WaitGroup{}
- // Start one goroutine for each KeepService: retrieve the
- // index, and add the returned blocks to BlockStateMap.
+ // When a device is mounted more than once, we will get its
+ // index only once, and call AddReplicas on all of the mounts.
+ // equivMount keys are the mounts that will be indexed, and
+ // each value is a list of mounts to apply the received index
+ // to.
+ equivMount := map[*KeepMount][]*KeepMount{}
+ // deviceMount maps each device ID to the one mount that will
+ // be indexed for that device.
+ deviceMount := map[string]*KeepMount{}
for _, srv := range bal.KeepServices {
+ for _, mnt := range srv.mounts {
+ equiv := deviceMount[mnt.DeviceID]
+ if equiv == nil {
+ equiv = mnt
+ if mnt.DeviceID != "" {
+ deviceMount[mnt.DeviceID] = equiv
+ }
+ }
+ equivMount[equiv] = append(equivMount[equiv], mnt)
+ }
+ }
+
+ // Start one goroutine for each (non-redundant) mount:
+ // retrieve the index, and add the returned blocks to
+ // BlockStateMap.
+ for _, mounts := range equivMount {
wg.Add(1)
- go func(srv *KeepService) {
+ go func(mounts []*KeepMount) {
defer wg.Done()
- bal.logf("%s: retrieve indexes", srv)
- for _, mount := range srv.mounts {
- bal.logf("%s: retrieve index", mount)
- idx, err := srv.IndexMount(c, mount.UUID, "")
- if err != nil {
- errs <- fmt.Errorf("%s: retrieve index: %v", mount, err)
- return
- }
- if len(errs) > 0 {
- // Some other goroutine encountered an
- // error -- any further effort here
- // will be wasted.
- return
- }
+ bal.logf("mount %s: retrieve index from %s", mounts[0], mounts[0].KeepService)
+ idx, err := mounts[0].KeepService.IndexMount(c, mounts[0].UUID, "")
+ if err != nil {
+ errs <- fmt.Errorf("%s: retrieve index: %v", mounts[0], err)
+ return
+ }
+ if len(errs) > 0 {
+ // Some other goroutine encountered an
+ // error -- any further effort here
+ // will be wasted.
+ return
+ }
+ for _, mount := range mounts {
bal.logf("%s: add %d replicas to map", mount, len(idx))
bal.BlockStateMap.AddReplicas(mount, idx)
- bal.logf("%s: done", mount)
+ bal.logf("%s: added %d replicas", mount, len(idx))
}
- bal.logf("%s: done", srv)
- }(srv)
+ bal.logf("mount %s: index done", mounts[0])
+ }(mounts)
}
// collQ buffers incoming collections so we can start fetching
for _, class := range bal.classes {
desired := blk.Desired[class]
+ countedDev := map[string]bool{}
have := 0
for _, slot := range slots {
- if slot.repl != nil && bal.mountsByClass[class][slot.mnt] {
- have++
+ if slot.repl != nil && bal.mountsByClass[class][slot.mnt] && !countedDev[slot.mnt.DeviceID] {
+ have += slot.mnt.Replication
+ if slot.mnt.DeviceID != "" {
+ countedDev[slot.mnt.DeviceID] = true
+ }
}
}
classState[class] = balancedBlockState{
}
})
- // Servers and mounts (with or without existing
+ // Servers/mounts/devices (with or without existing
// replicas) that are part of the best achievable
// layout for this storage class.
wantSrv := map[*KeepService]bool{}
wantMnt := map[*KeepMount]bool{}
+ wantDev := map[string]bool{}
// Positions (with existing replicas) that have been
// protected (via unsafeToDelete) to ensure we don't
// reduce replication below desired level when
// trashing replicas that aren't optimal positions for
// any storage class.
protMnt := map[*KeepMount]bool{}
+ // Replication planned so far (corresponds to wantMnt).
+ replWant := 0
+ // Protected replication (corresponds to protMnt).
+ replProt := 0
// trySlot tries using a slot to meet requirements,
// and returns true if all requirements are met.
trySlot := func(i int) bool {
slot := slots[i]
- if len(protMnt) < desired && slot.repl != nil {
+ if wantMnt[slot.mnt] || wantDev[slot.mnt.DeviceID] {
+ // Already allocated a replica to this
+ // backend device, possibly on a
+ // different server.
+ return false
+ }
+ if replProt < desired && slot.repl != nil && !protMnt[slot.mnt] {
unsafeToDelete[slot.repl.Mtime] = true
protMnt[slot.mnt] = true
+ replProt += slot.mnt.Replication
}
- if len(wantMnt) < desired && (slot.repl != nil || !slot.mnt.ReadOnly) {
+ if replWant < desired && (slot.repl != nil || !slot.mnt.ReadOnly) {
slots[i].want = true
wantSrv[slot.mnt.KeepService] = true
wantMnt[slot.mnt] = true
+ if slot.mnt.DeviceID != "" {
+ wantDev[slot.mnt.DeviceID] = true
+ }
+ replWant += slot.mnt.Replication
}
- return len(protMnt) >= desired && len(wantMnt) >= desired
+ return replProt >= desired && replWant >= desired
}
// First try to achieve desired replication without
if slot.repl == nil || !bal.mountsByClass[class][slot.mnt] {
continue
}
- if safe++; safe >= desired {
+ if safe += slot.mnt.Replication; safe >= desired {
break
}
}
cs.unachievable = true
classState[class] = cs
}
+
+ // Avoid deleting wanted replicas from devices that
+ // are mounted on multiple servers -- even if they
+ // haven't already been added to unsafeToDelete
+ // because the servers report different Mtimes.
+ for _, slot := range slots {
+ if slot.repl != nil && wantDev[slot.mnt.DeviceID] {
+ unsafeToDelete[slot.repl.Mtime] = true
+ }
+ }
}
// TODO: If multiple replicas are trashable, prefer the oldest
// replica that doesn't have a timestamp collision with
// others.
+ countedDev := map[string]bool{}
var have, want int
for _, slot := range slots {
+ if countedDev[slot.mnt.DeviceID] {
+ continue
+ }
if slot.want {
- want++
+ want += slot.mnt.Replication
}
if slot.repl != nil {
- have++
+ have += slot.mnt.Replication
+ }
+ if slot.mnt.DeviceID != "" {
+ countedDev[slot.mnt.DeviceID] = true
}
}
case surplus > 0:
s.overrep.replicas += surplus
s.overrep.blocks++
- s.overrep.bytes += bytes * int64(len(result.blk.Replicas)-result.want)
+ s.overrep.bytes += bytes * int64(result.have-result.want)
default:
s.justright.replicas += result.want
s.justright.blocks++
s.desired.blocks++
s.desired.bytes += bytes * int64(result.want)
}
- if len(result.blk.Replicas) > 0 {
- s.current.replicas += len(result.blk.Replicas)
+ if result.have > 0 {
+ s.current.replicas += result.have
s.current.blocks++
- s.current.bytes += bytes * int64(len(result.blk.Replicas))
+ s.current.bytes += bytes * int64(result.have)
}
- for len(s.replHistogram) <= len(result.blk.Replicas) {
+ for len(s.replHistogram) <= result.have {
s.replHistogram = append(s.replHistogram, 0)
}
- s.replHistogram[len(result.blk.Replicas)]++
+ s.replHistogram[result.have]++
}
for _, srv := range bal.KeepServices {
s.pulls += len(srv.ChangeSet.Pulls)
shouldPullMounts []string
shouldTrashMounts []string
+
+ expectResult balanceResult
}
func (bal *balancerSuite) SetUpSuite(c *check.C) {
}
bal.MinMtime = time.Now().UnixNano() - bal.signatureTTL*1e9
+ bal.cleanupMounts()
}
func (bal *balancerSuite) TestPerfect(c *check.C) {
shouldTrash: slots{2}})
}
+func (bal *balancerSuite) TestCleanupMounts(c *check.C) {
+ bal.srvs[3].mounts[0].KeepMount.ReadOnly = true
+ bal.srvs[3].mounts[0].KeepMount.DeviceID = "abcdef"
+ bal.srvs[14].mounts[0].KeepMount.DeviceID = "abcdef"
+ c.Check(len(bal.srvs[3].mounts), check.Equals, 1)
+ bal.cleanupMounts()
+ c.Check(len(bal.srvs[3].mounts), check.Equals, 0)
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 2},
+ current: slots{1},
+ shouldPull: slots{2}})
+}
+
+func (bal *balancerSuite) TestVolumeReplication(c *check.C) {
+ bal.srvs[0].mounts[0].KeepMount.Replication = 2 // srv 0
+ bal.srvs[14].mounts[0].KeepMount.Replication = 2 // srv e
+ bal.cleanupMounts()
+ // block 0 rendezvous is 3,e,a -- so slot 1 has repl=2
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 2},
+ current: slots{1},
+ shouldPull: slots{0}})
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 2},
+ current: slots{0, 1},
+ shouldPull: nil})
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 2},
+ current: slots{0, 1, 2},
+ shouldTrash: slots{2}})
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 3},
+ current: slots{0, 2, 3, 4},
+ shouldPull: slots{1},
+ shouldTrash: slots{4},
+ expectResult: balanceResult{
+ have: 4,
+ want: 3,
+ classState: map[string]balancedBlockState{"default": {
+ desired: 3,
+ surplus: 1,
+ unachievable: false}}}})
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 3},
+ current: slots{0, 1, 2, 3, 4},
+ shouldTrash: slots{2, 3, 4}})
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 4},
+ current: slots{0, 1, 2, 3, 4},
+ shouldTrash: slots{3, 4},
+ expectResult: balanceResult{
+ have: 6,
+ want: 4,
+ classState: map[string]balancedBlockState{"default": {
+ desired: 4,
+ surplus: 2,
+ unachievable: false}}}})
+ // block 1 rendezvous is 0,9,7 -- so slot 0 has repl=2
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 2},
+ current: slots{0},
+ expectResult: balanceResult{
+ have: 2,
+ want: 2,
+ classState: map[string]balancedBlockState{"default": {
+ desired: 2,
+ surplus: 0,
+ unachievable: false}}}})
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 3},
+ current: slots{0},
+ shouldPull: slots{1}})
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 4},
+ current: slots{0},
+ shouldPull: slots{1, 2}})
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 4},
+ current: slots{2},
+ shouldPull: slots{0, 1}})
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 4},
+ current: slots{7},
+ shouldPull: slots{0, 1, 2},
+ expectResult: balanceResult{
+ have: 1,
+ want: 4,
+ classState: map[string]balancedBlockState{"default": {
+ desired: 4,
+ surplus: -3,
+ unachievable: false}}}})
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 2},
+ current: slots{1, 2, 3, 4},
+ shouldPull: slots{0},
+ shouldTrash: slots{3, 4}})
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 2},
+ current: slots{0, 1, 2},
+ shouldTrash: slots{1, 2},
+ expectResult: balanceResult{
+ have: 4,
+ want: 2,
+ classState: map[string]balancedBlockState{"default": {
+ desired: 2,
+ surplus: 2,
+ unachievable: false}}}})
+}
+
+func (bal *balancerSuite) TestDeviceRWMountedByMultipleServers(c *check.C) {
+ bal.srvs[0].mounts[0].KeepMount.DeviceID = "abcdef"
+ bal.srvs[9].mounts[0].KeepMount.DeviceID = "abcdef"
+ bal.srvs[14].mounts[0].KeepMount.DeviceID = "abcdef"
+ // block 0 belongs on servers 3 and e, which have different
+ // device IDs.
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 2},
+ current: slots{1},
+ shouldPull: slots{0}})
+ // block 1 belongs on servers 0 and 9, which both report
+ // having a replica, but the replicas are on the same device
+ // ID -- so we should pull to the third position (7).
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 2},
+ current: slots{0, 1},
+ shouldPull: slots{2}})
+ // block 1 can be pulled to the doubly-mounted device, but the
+ // pull should only be done on the first of the two servers.
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 2},
+ current: slots{2},
+ shouldPull: slots{0}})
+ // block 0 has one replica on a single device mounted on two
+ // servers (e,9 at positions 1,9). Trashing the replica on 9
+ // would lose the block.
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 2},
+ current: slots{1, 9},
+ shouldPull: slots{0},
+ expectResult: balanceResult{
+ have: 1,
+ classState: map[string]balancedBlockState{"default": {
+ desired: 2,
+ surplus: -1,
+ unachievable: false}}}})
+ // block 0 is overreplicated, but the second and third
+ // replicas are the same replica according to DeviceID
+ // (despite different Mtimes). Don't trash the third replica.
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 2},
+ current: slots{0, 1, 9},
+ expectResult: balanceResult{
+ have: 2,
+ classState: map[string]balancedBlockState{"default": {
+ desired: 2,
+ surplus: 0,
+ unachievable: false}}}})
+ // block 0 is overreplicated; the third and fifth replicas are
+ // extra, but the fourth is another view of the second and
+ // shouldn't be trashed.
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 2},
+ current: slots{0, 1, 5, 9, 12},
+ shouldTrash: slots{5, 12},
+ expectResult: balanceResult{
+ have: 4,
+ classState: map[string]balancedBlockState{"default": {
+ desired: 2,
+ surplus: 2,
+ unachievable: false}}}})
+}
+
func (bal *balancerSuite) TestChangeStorageClasses(c *check.C) {
// For known blocks 0/1/2/3, server 9 is slot 9/1/14/0 in
// probe order. For these tests we give it two mounts, one
for _, srv := range bal.srvs {
srv.ChangeSet = &ChangeSet{}
}
- bal.balanceBlock(knownBlkid(t.known), blk)
+ result := bal.balanceBlock(knownBlkid(t.known), blk)
var didPull, didTrash slots
var didPullMounts, didTrashMounts []string
sort.Strings(didTrashMounts)
c.Check(didTrashMounts, check.DeepEquals, t.shouldTrashMounts)
}
+ if t.expectResult.have > 0 {
+ c.Check(result.have, check.Equals, t.expectResult.have)
+ }
+ if t.expectResult.want > 0 {
+ c.Check(result.want, check.Equals, t.expectResult.want)
+ }
+ if t.expectResult.classState != nil {
+ c.Check(result.classState, check.DeepEquals, t.expectResult.classState)
+ }
}
// srvList returns the KeepServices, sorted in rendezvous order and
import (
"sync"
- "sync/atomic"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"github.com/hashicorp/golang-lru"
+ "github.com/prometheus/client_golang/prometheus"
)
+const metricsUpdateInterval = time.Second / 10
+
type cache struct {
TTL arvados.Duration
UUIDTTL arvados.Duration
MaxPermissionEntries int
MaxUUIDEntries int
- stats cacheStats
+ registry *prometheus.Registry
+ metrics cacheMetrics
pdhs *lru.TwoQueueCache
collections *lru.TwoQueueCache
permissions *lru.TwoQueueCache
setupOnce sync.Once
}
-type cacheStats struct {
- Requests uint64 `json:"Cache.Requests"`
- CollectionBytes uint64 `json:"Cache.CollectionBytes"`
- CollectionEntries int `json:"Cache.CollectionEntries"`
- CollectionHits uint64 `json:"Cache.CollectionHits"`
- PDHHits uint64 `json:"Cache.UUIDHits"`
- PermissionHits uint64 `json:"Cache.PermissionHits"`
- APICalls uint64 `json:"Cache.APICalls"`
+type cacheMetrics struct {
+ requests prometheus.Counter
+ collectionBytes prometheus.Gauge
+ collectionEntries prometheus.Gauge
+ collectionHits prometheus.Counter
+ pdhHits prometheus.Counter
+ permissionHits prometheus.Counter
+ apiCalls prometheus.Counter
+}
+
+func (m *cacheMetrics) setup(reg *prometheus.Registry) {
+ m.requests = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "requests",
+ Help: "Number of targetID-to-manifest lookups handled.",
+ })
+ reg.MustRegister(m.requests)
+ m.collectionHits = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "hits",
+ Help: "Number of pdh-to-manifest cache hits.",
+ })
+ reg.MustRegister(m.collectionHits)
+ m.pdhHits = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "pdh_hits",
+ Help: "Number of uuid-to-pdh cache hits.",
+ })
+ reg.MustRegister(m.pdhHits)
+ m.permissionHits = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "permission_hits",
+ Help: "Number of targetID-to-permission cache hits.",
+ })
+ reg.MustRegister(m.permissionHits)
+ m.apiCalls = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "api_calls",
+ Help: "Number of outgoing API calls made by cache.",
+ })
+ reg.MustRegister(m.apiCalls)
+ m.collectionBytes = prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "cached_manifest_bytes",
+ Help: "Total size of all manifests in cache.",
+ })
+ reg.MustRegister(m.collectionBytes)
+ m.collectionEntries = prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "cached_manifests",
+ Help: "Number of manifests in cache.",
+ })
+ reg.MustRegister(m.collectionEntries)
}
type cachedPDH struct {
if err != nil {
panic(err)
}
+
+ reg := c.registry
+ if reg == nil {
+ reg = prometheus.NewRegistry()
+ }
+ c.metrics.setup(reg)
+ go func() {
+ for range time.Tick(metricsUpdateInterval) {
+ c.updateGauges()
+ }
+ }()
}
-var selectPDH = map[string]interface{}{
- "select": []string{"portable_data_hash"},
+func (c *cache) updateGauges() {
+ c.metrics.collectionBytes.Set(float64(c.collectionBytes()))
+ c.metrics.collectionEntries.Set(float64(c.collections.Len()))
}
-func (c *cache) Stats() cacheStats {
- c.setupOnce.Do(c.setup)
- return cacheStats{
- Requests: atomic.LoadUint64(&c.stats.Requests),
- CollectionBytes: c.collectionBytes(),
- CollectionEntries: c.collections.Len(),
- CollectionHits: atomic.LoadUint64(&c.stats.CollectionHits),
- PDHHits: atomic.LoadUint64(&c.stats.PDHHits),
- PermissionHits: atomic.LoadUint64(&c.stats.PermissionHits),
- APICalls: atomic.LoadUint64(&c.stats.APICalls),
- }
+var selectPDH = map[string]interface{}{
+ "select": []string{"portable_data_hash"},
}
// Update saves a modified version (fs) to an existing collection
}
var updated arvados.Collection
defer c.pdhs.Remove(coll.UUID)
- err := client.RequestAndDecode(&updated, "PATCH", "/arvados/v1/collections/"+coll.UUID, client.UpdateBody(coll), nil)
+ err := client.RequestAndDecode(&updated, "PATCH", "arvados/v1/collections/"+coll.UUID, client.UpdateBody(coll), nil)
if err == nil {
c.collections.Add(client.AuthToken+"\000"+coll.PortableDataHash, &cachedCollection{
expire: time.Now().Add(time.Duration(c.TTL)),
func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceReload bool) (*arvados.Collection, error) {
c.setupOnce.Do(c.setup)
-
- atomic.AddUint64(&c.stats.Requests, 1)
+ c.metrics.requests.Inc()
permOK := false
permKey := arv.ApiToken + "\000" + targetID
c.permissions.Remove(permKey)
} else {
permOK = true
- atomic.AddUint64(&c.stats.PermissionHits, 1)
+ c.metrics.permissionHits.Inc()
}
}
c.pdhs.Remove(targetID)
} else {
pdh = ent.pdh
- atomic.AddUint64(&c.stats.PDHHits, 1)
+ c.metrics.pdhHits.Inc()
}
}
// likely, the cached PDH is still correct; if so,
// _and_ the current token has permission, we can
// use our cached manifest.
- atomic.AddUint64(&c.stats.APICalls, 1)
+ c.metrics.apiCalls.Inc()
var current arvados.Collection
err := arv.Get("collections", targetID, selectPDH, ¤t)
if err != nil {
}
// Collection manifest is not cached.
- atomic.AddUint64(&c.stats.APICalls, 1)
+ c.metrics.apiCalls.Inc()
err := arv.Get("collections", targetID, nil, &collection)
if err != nil {
return nil, err
}
func (c *cache) lookupCollection(key string) *arvados.Collection {
- if ent, cached := c.collections.Get(key); !cached {
+ e, cached := c.collections.Get(key)
+ if !cached {
+ return nil
+ }
+ ent := e.(*cachedCollection)
+ if ent.expire.Before(time.Now()) {
+ c.collections.Remove(key)
return nil
- } else {
- ent := ent.(*cachedCollection)
- if ent.expire.Before(time.Now()) {
- c.collections.Remove(key)
- return nil
- } else {
- atomic.AddUint64(&c.stats.CollectionHits, 1)
- return ent.collection
- }
}
+ c.metrics.collectionHits.Inc()
+ return ent.collection
}
package main
import (
+ "bytes"
+
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/common/expfmt"
"gopkg.in/check.v1"
)
+func (s *UnitSuite) checkCacheMetrics(c *check.C, reg *prometheus.Registry, regs ...string) {
+ mfs, err := reg.Gather()
+ c.Check(err, check.IsNil)
+ buf := &bytes.Buffer{}
+ enc := expfmt.NewEncoder(buf, expfmt.FmtText)
+ for _, mf := range mfs {
+ c.Check(enc.Encode(mf), check.IsNil)
+ }
+ mm := buf.String()
+ for _, reg := range regs {
+ c.Check(mm, check.Matches, `(?ms).*collectioncache_`+reg+`\n.*`)
+ }
+}
+
func (s *UnitSuite) TestCache(c *check.C) {
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, check.Equals, nil)
cache := DefaultConfig().Cache
+ cache.registry = prometheus.NewRegistry()
// Hit the same collection 5 times using the same token. Only
// the first req should cause an API call; the next 4 should
c.Check(coll.PortableDataHash, check.Equals, arvadostest.FooPdh)
c.Check(coll.ManifestText[:2], check.Equals, ". ")
}
- c.Check(cache.Stats().Requests, check.Equals, uint64(5))
- c.Check(cache.Stats().CollectionHits, check.Equals, uint64(4))
- c.Check(cache.Stats().PermissionHits, check.Equals, uint64(4))
- c.Check(cache.Stats().PDHHits, check.Equals, uint64(4))
- c.Check(cache.Stats().APICalls, check.Equals, uint64(1))
+ s.checkCacheMetrics(c, cache.registry,
+ "requests 5",
+ "hits 4",
+ "permission_hits 4",
+ "pdh_hits 4",
+ "api_calls 1")
// Hit the same collection 2 more times, this time requesting
// it by PDH and using a different token. The first req should
c.Check(coll2.ManifestText[:2], check.Equals, ". ")
c.Check(coll2.ManifestText, check.Not(check.Equals), coll.ManifestText)
- c.Check(cache.Stats().Requests, check.Equals, uint64(5+1))
- c.Check(cache.Stats().CollectionHits, check.Equals, uint64(4+0))
- c.Check(cache.Stats().PermissionHits, check.Equals, uint64(4+0))
- c.Check(cache.Stats().PDHHits, check.Equals, uint64(4+0))
- c.Check(cache.Stats().APICalls, check.Equals, uint64(1+1))
+ s.checkCacheMetrics(c, cache.registry,
+ "requests 6",
+ "hits 4",
+ "permission_hits 4",
+ "pdh_hits 4",
+ "api_calls 2")
coll2, err = cache.Get(arv, arvadostest.FooPdh, false)
c.Check(err, check.Equals, nil)
c.Check(coll2.PortableDataHash, check.Equals, arvadostest.FooPdh)
c.Check(coll2.ManifestText[:2], check.Equals, ". ")
- c.Check(cache.Stats().Requests, check.Equals, uint64(5+2))
- c.Check(cache.Stats().CollectionHits, check.Equals, uint64(4+1))
- c.Check(cache.Stats().PermissionHits, check.Equals, uint64(4+1))
- c.Check(cache.Stats().PDHHits, check.Equals, uint64(4+0))
- c.Check(cache.Stats().APICalls, check.Equals, uint64(1+1))
+ s.checkCacheMetrics(c, cache.registry,
+ "requests 7",
+ "hits 5",
+ "permission_hits 5",
+ "pdh_hits 4",
+ "api_calls 2")
// Alternating between two collections N times should produce
// only 2 more API calls.
_, err := cache.Get(arv, target, false)
c.Check(err, check.Equals, nil)
}
- c.Check(cache.Stats().Requests, check.Equals, uint64(5+2+20))
- c.Check(cache.Stats().CollectionHits, check.Equals, uint64(4+1+18))
- c.Check(cache.Stats().PermissionHits, check.Equals, uint64(4+1+18))
- c.Check(cache.Stats().PDHHits, check.Equals, uint64(4+0+18))
- c.Check(cache.Stats().APICalls, check.Equals, uint64(1+1+2))
+ s.checkCacheMetrics(c, cache.registry,
+ "requests 27",
+ "hits 23",
+ "permission_hits 23",
+ "pdh_hits 22",
+ "api_calls 4")
}
func (s *UnitSuite) TestCacheForceReloadByPDH(c *check.C) {
c.Assert(err, check.Equals, nil)
cache := DefaultConfig().Cache
+ cache.registry = prometheus.NewRegistry()
for _, forceReload := range []bool{false, true, false, true} {
_, err := cache.Get(arv, arvadostest.FooPdh, forceReload)
c.Check(err, check.Equals, nil)
}
- c.Check(cache.Stats().Requests, check.Equals, uint64(4))
- c.Check(cache.Stats().CollectionHits, check.Equals, uint64(3))
- c.Check(cache.Stats().PermissionHits, check.Equals, uint64(1))
- c.Check(cache.Stats().PDHHits, check.Equals, uint64(0))
- c.Check(cache.Stats().APICalls, check.Equals, uint64(3))
+ s.checkCacheMetrics(c, cache.registry,
+ "requests 4",
+ "hits 3",
+ "permission_hits 1",
+ "pdh_hits 0",
+ "api_calls 3")
}
func (s *UnitSuite) TestCacheForceReloadByUUID(c *check.C) {
c.Assert(err, check.Equals, nil)
cache := DefaultConfig().Cache
+ cache.registry = prometheus.NewRegistry()
for _, forceReload := range []bool{false, true, false, true} {
_, err := cache.Get(arv, arvadostest.FooCollection, forceReload)
c.Check(err, check.Equals, nil)
}
- c.Check(cache.Stats().Requests, check.Equals, uint64(4))
- c.Check(cache.Stats().CollectionHits, check.Equals, uint64(3))
- c.Check(cache.Stats().PermissionHits, check.Equals, uint64(1))
- c.Check(cache.Stats().PDHHits, check.Equals, uint64(3))
- c.Check(cache.Stats().APICalls, check.Equals, uint64(3))
+ s.checkCacheMetrics(c, cache.registry,
+ "requests 4",
+ "hits 3",
+ "permission_hits 1",
+ "pdh_hits 3",
+ "api_calls 3")
}
var newCollection arvados.Collection
arv := arvados.NewClientFromEnv()
arv.AuthToken = arvadostest.ActiveToken
- err = arv.RequestAndDecode(&newCollection, "POST", "/arvados/v1/collections", bytes.NewBufferString(url.Values{"collection": {"{}"}}.Encode()), nil)
+ err = arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", bytes.NewBufferString(url.Values{"collection": {"{}"}}.Encode()), nil)
c.Assert(err, check.IsNil)
readPath, writePath, pdhPath := pathFunc(newCollection)
// avoids redirecting requests to keep-web if they depend on
// TrustAllContent being enabled.
//
+// Metrics
+//
+// Keep-web exposes request metrics in Prometheus text-based format at
+// /metrics. The same information is also available as JSON at
+// /metrics.json.
+//
package main
type handler struct {
Config *Config
+ MetricsAPI http.Handler
clientPool *arvadosclient.ClientPool
setupOnce sync.Once
healthHandler http.Handler
}
func (h *handler) serveStatus(w http.ResponseWriter, r *http.Request) {
- status := struct {
- cacheStats
- Version string
- }{
- cacheStats: h.Config.Cache.Stats(),
- Version: version,
- }
- json.NewEncoder(w).Encode(status)
+ json.NewEncoder(w).Encode(struct{ Version string }{version})
}
// updateOnSuccess wraps httpserver.ResponseWriter. If the handler
if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
remoteAddr = xff + "," + remoteAddr
}
+ if xfp := r.Header.Get("X-Forwarded-Proto"); xfp != "" && xfp != "http" {
+ r.URL.Scheme = xfp
+ }
w := httpserver.WrapResponseWriter(wOrig)
defer func() {
} else if r.URL.Path == "/status.json" {
h.serveStatus(w, r)
return
+ } else if strings.HasPrefix(r.URL.Path, "/metrics") {
+ h.MetricsAPI.ServeHTTP(w, r)
+ return
} else if siteFSDir[pathParts[0]] {
useSiteFS = true
} else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
u = newu
}
redir := (&url.URL{
+ Scheme: r.URL.Scheme,
Host: r.Host,
Path: u.Path,
RawQuery: redirQuery.Encode(),
func (s *UnitSuite) TestCORSPreflight(c *check.C) {
h := handler{Config: DefaultConfig()}
- u, _ := url.Parse("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
+ u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
req := &http.Request{
Method: "OPTIONS",
Host: u.Host,
"http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
} {
c.Log(trial)
- u, err := url.Parse(trial)
- c.Assert(err, check.IsNil)
+ u := mustParseURL(trial)
req := &http.Request{
Method: "GET",
Host: u.Host,
if resp.Code != http.StatusSeeOther {
return resp
}
- c.Check(resp.Body.String(), check.Matches, `.*href="//`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
+ c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
cookies := (&http.Response{Header: resp.Header()}).Cookies()
u, _ = u.Parse(resp.Header().Get("Location"))
package main
import (
+ "net/http"
+
"git.curoverse.com/arvados.git/sdk/go/httpserver"
+ "github.com/prometheus/client_golang/prometheus"
)
type server struct {
}
func (srv *server) Start() error {
- srv.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(&handler{Config: srv.Config}))
+ h := &handler{Config: srv.Config}
+ reg := prometheus.NewRegistry()
+ h.Config.Cache.registry = reg
+ mh := httpserver.Instrument(reg, nil, httpserver.AddRequestIDs(httpserver.LogRequests(nil, h)))
+ h.MetricsAPI = mh.ServeAPI(http.NotFoundHandler())
+ srv.Handler = mh
srv.Addr = srv.Config.Listen
return srv.Server.Start()
}
import (
"crypto/md5"
+ "encoding/json"
"fmt"
"io"
"io/ioutil"
"net"
+ "net/http"
"os"
"os/exec"
"strings"
return
}
+func (s *IntegrationSuite) TestMetrics(c *check.C) {
+ origin := "http://" + s.testServer.Addr
+ req, _ := http.NewRequest("GET", origin+"/notfound", nil)
+ _, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ req, _ = http.NewRequest("GET", origin+"/by_id/", nil)
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ for i := 0; i < 2; i++ {
+ req, _ = http.NewRequest("GET", origin+"/foo", nil)
+ req.Host = arvadostest.FooCollection + ".example.com"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ buf, _ := ioutil.ReadAll(resp.Body)
+ c.Check(buf, check.DeepEquals, []byte("foo"))
+ resp.Body.Close()
+ }
+
+ s.testServer.Config.Cache.updateGauges()
+
+ req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ type summary struct {
+ SampleCount string `json:"sample_count"`
+ SampleSum float64 `json:"sample_sum"`
+ Quantile []struct {
+ Quantile float64
+ Value float64
+ }
+ }
+ type counter struct {
+ Value int64
+ }
+ type gauge struct {
+ Value float64
+ }
+ var ents []struct {
+ Name string
+ Help string
+ Type string
+ Metric []struct {
+ Label []struct {
+ Name string
+ Value string
+ }
+ Counter counter
+ Gauge gauge
+ Summary summary
+ }
+ }
+ json.NewDecoder(resp.Body).Decode(&ents)
+ summaries := map[string]summary{}
+ gauges := map[string]gauge{}
+ counters := map[string]counter{}
+ for _, e := range ents {
+ for _, m := range e.Metric {
+ labels := map[string]string{}
+ for _, lbl := range m.Label {
+ labels[lbl.Name] = lbl.Value
+ }
+ summaries[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
+ counters[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Counter
+ gauges[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Gauge
+ }
+ }
+ c.Check(summaries["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
+ c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
+ c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
+ c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
+ c.Check(counters["arvados_keepweb_collectioncache_requests//"].Value, check.Equals, int64(2))
+ c.Check(counters["arvados_keepweb_collectioncache_api_calls//"].Value, check.Equals, int64(1))
+ c.Check(counters["arvados_keepweb_collectioncache_hits//"].Value, check.Equals, int64(1))
+ c.Check(counters["arvados_keepweb_collectioncache_pdh_hits//"].Value, check.Equals, int64(1))
+ c.Check(counters["arvados_keepweb_collectioncache_permission_hits//"].Value, check.Equals, int64(1))
+ c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1))
+ // FooCollection's cached manifest size is 45 ("1f4b0....+45") plus one 51-byte blob signature
+ c.Check(gauges["arvados_keepweb_collectioncache_cached_manifest_bytes//"].Value, check.Equals, float64(45+51))
+
+ // If the Host header indicates a collection, /metrics.json
+ // refers to a file in the collection -- the metrics handler
+ // must not intercept that route.
+ req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
+ req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
+}
+
func (s *IntegrationSuite) SetUpSuite(c *check.C) {
arvadostest.StartAPI()
arvadostest.StartKeep(2, true)
var status map[string]interface{}
err := json.NewDecoder(resp.Body).Decode(&status)
c.Check(err, check.IsNil)
- c.Check(status["Cache.Requests"], check.Equals, float64(0))
c.Check(status["Version"], check.Not(check.Equals), "")
}
// Start serving requests.
router = MakeRESTRouter(!cfg.DisableGet, !cfg.DisablePut, kc, time.Duration(cfg.Timeout), cfg.ManagementToken)
- http.Serve(listener, httpserver.AddRequestIDs(httpserver.LogRequests(router)))
+ http.Serve(listener, httpserver.AddRequestIDs(httpserver.LogRequests(nil, router)))
log.Println("shutting down")
}
locatorIn := mux.Vars(req)["locator"]
+ // Check if the client specified storage classes
+ if req.Header.Get("X-Keep-Storage-Classes") != "" {
+ var scl []string
+ for _, sc := range strings.Split(req.Header.Get("X-Keep-Storage-Classes"), ",") {
+ scl = append(scl, strings.Trim(sc, " "))
+ }
+ kc.StorageClasses = scl
+ }
+
_, err = fmt.Sscanf(req.Header.Get("Content-Length"), "%d", &expectLength)
if err != nil || expectLength < 0 {
err = LengthRequiredError
c.Check(err, ErrorMatches, `.*loop detected.*`)
}
+func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
+ kc := runProxy(c, nil, false)
+ defer closeListener()
+
+ // Set up fake keepstore to record request headers
+ var hdr http.Header
+ ts := httptest.NewServer(http.HandlerFunc(
+ func(w http.ResponseWriter, r *http.Request) {
+ hdr = r.Header
+ http.Error(w, "Error", http.StatusInternalServerError)
+ }))
+ defer ts.Close()
+
+ // Point keepproxy router's keepclient to the fake keepstore
+ sr := map[string]string{
+ TestProxyUUID: ts.URL,
+ }
+ router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
+
+ // Set up client to ask for storage classes to keepproxy
+ kc.StorageClasses = []string{"secure"}
+ content := []byte("Very important data")
+ _, _, err := kc.PutB(content)
+ c.Check(err, NotNil)
+ c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
+}
+
func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
kc := runProxy(c, nil, false)
defer closeListener()
}
func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
- arv, err := arvadosclient.MakeArvadosClient()
- c.Assert(err, Equals, nil)
+ kc := runProxy(c, nil, false)
+ defer closeListener()
- // keepclient with no such keep server
- kc := keepclient.New(arv)
+ // Point keepproxy at a non-existent keepstore
locals := map[string]string{
TestProxyUUID: "http://localhost:12345",
}
- kc.SetServiceRoots(locals, nil, nil)
+ router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
- // Ask should result in temporary connection refused error
+ // Ask should result in temporary bad gateway error
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
- _, _, err = kc.Ask(hash)
+ _, _, err := kc.Ask(hash)
c.Check(err, NotNil)
errNotFound, _ := err.(*keepclient.ErrNotFound)
c.Check(errNotFound.Temporary(), Equals, true)
- c.Assert(err, ErrorMatches, ".*connection refused.*")
+ c.Assert(err, ErrorMatches, ".*HTTP 502.*")
- // Get should result in temporary connection refused error
+ // Get should result in temporary bad gateway error
_, _, _, err = kc.Get(hash)
c.Check(err, NotNil)
errNotFound, _ = err.(*keepclient.ErrNotFound)
c.Check(errNotFound.Temporary(), Equals, true)
- c.Assert(err, ErrorMatches, ".*connection refused.*")
+ c.Assert(err, ErrorMatches, ".*HTTP 502.*")
}
func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
"encoding/json"
"fmt"
"io/ioutil"
- "net/http"
- "strconv"
"strings"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
- "git.curoverse.com/arvados.git/sdk/go/stats"
"github.com/Sirupsen/logrus"
- "github.com/golang/protobuf/jsonpb"
- "github.com/prometheus/client_golang/prometheus"
- "github.com/prometheus/client_golang/prometheus/promhttp"
)
type Config struct {
systemAuthToken string
debugLogf func(string, ...interface{})
- ManagementToken string
-
- metrics
+ ManagementToken string `doc: The secret key that must be provided by monitoring services
+wishing to access the health check endpoint (/_health).`
}
var (
return nil
}
-type metrics struct {
- registry *prometheus.Registry
- reqDuration *prometheus.SummaryVec
- timeToStatus *prometheus.SummaryVec
- exportProm http.Handler
-}
-
-func (*metrics) Levels() []logrus.Level {
- return logrus.AllLevels
-}
-
-func (m *metrics) Fire(ent *logrus.Entry) error {
- if tts, ok := ent.Data["timeToStatus"].(stats.Duration); !ok {
- } else if method, ok := ent.Data["reqMethod"].(string); !ok {
- } else if code, ok := ent.Data["respStatusCode"].(int); !ok {
- } else {
- m.timeToStatus.WithLabelValues(strconv.Itoa(code), strings.ToLower(method)).Observe(time.Duration(tts).Seconds())
- }
- return nil
-}
-
-func (m *metrics) setup() {
- m.registry = prometheus.NewRegistry()
- m.timeToStatus = prometheus.NewSummaryVec(prometheus.SummaryOpts{
- Name: "time_to_status_seconds",
- Help: "Summary of request TTFB.",
- }, []string{"code", "method"})
- m.reqDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{
- Name: "request_duration_seconds",
- Help: "Summary of request duration.",
- }, []string{"code", "method"})
- m.registry.MustRegister(m.timeToStatus)
- m.registry.MustRegister(m.reqDuration)
- m.exportProm = promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{
- ErrorLog: log,
- })
- log.AddHook(m)
-}
-
-func (m *metrics) exportJSON(w http.ResponseWriter, req *http.Request) {
- jm := jsonpb.Marshaler{Indent: " "}
- mfs, _ := m.registry.Gather()
- w.Write([]byte{'['})
- for i, mf := range mfs {
- if i > 0 {
- w.Write([]byte{','})
- }
- jm.Marshal(w, mf)
- }
- w.Write([]byte{']'})
-}
-
-func (m *metrics) Instrument(next http.Handler) http.Handler {
- return promhttp.InstrumentHandlerDuration(m.reqDuration, next)
-}
-
// VolumeTypes is built up by init() funcs in the source files that
// define the volume types.
var VolumeTypes = []func() VolumeWithExamples{}
// 400 Bad Request.
rtr.NotFoundHandler = http.HandlerFunc(BadRequestHandler)
- theConfig.metrics.setup()
-
rtr.limiter = httpserver.NewRequestLimiter(theConfig.MaxRequests, rtr)
- mux := http.NewServeMux()
- mux.Handle("/", theConfig.metrics.Instrument(
- httpserver.AddRequestIDs(httpserver.LogRequests(rtr.limiter))))
- mux.HandleFunc("/metrics.json", theConfig.metrics.exportJSON)
- mux.Handle("/metrics", theConfig.metrics.exportProm)
-
- return mux
+ stack := httpserver.Instrument(nil, nil,
+ httpserver.AddRequestIDs(httpserver.LogRequests(nil, rtr.limiter)))
+ return stack.ServeAPI(stack)
}
// BadRequestHandler is a HandleFunc to address bad requests.
s.files = ["bin/arvados-login-sync", "agpl-3.0.txt"]
s.executables << "arvados-login-sync"
s.required_ruby_version = '>= 2.1.0'
- s.add_runtime_dependency 'arvados', '~> 0.1', '>= 0.1.20150615153458'
+ s.add_runtime_dependency 'arvados', '~> 1.1.0', '>= 1.1.4'
s.homepage =
'https://arvados.org'
end
@RetryMixin._retry()
def create_cloud_node(self):
self._logger.info("Sending create_node request for node size %s.",
- self.cloud_size.name)
+ self.cloud_size.id)
try:
self.cloud_node = self._cloud.create_node(self.cloud_size,
self.arvados_node)
return super(ComputeNodeShutdownActor, self)._finished()
def cancel_shutdown(self, reason, **kwargs):
+ if not self.cancellable:
+ return False
if self.cancel_reason is not None:
# already cancelled
- return
+ return False
self.cancel_reason = reason
self._logger.info("Shutdown cancelled: %s.", reason)
self._finished(success_flag=False)
+ return True
def _cancel_on_exception(orig_func):
@functools.wraps(orig_func)
self._logger.info("Starting shutdown")
arv_node = self._arvados_node()
if self._cloud.destroy_node(self.cloud_node):
+ self.cancellable = False
self._logger.info("Shutdown success")
if arv_node:
self._later.clean_arvados_node(arv_node)
def __init__(self, cloud_node, cloud_node_start_time, shutdown_timer,
timer_actor, update_actor, cloud_client,
arvados_node=None, poll_stale_after=600, node_stale_after=3600,
- boot_fail_after=1800
+ boot_fail_after=1800, consecutive_idle_count=0
):
super(ComputeNodeMonitorActor, self).__init__()
self._later = self.actor_ref.tell_proxy()
self.boot_fail_after = boot_fail_after
self.subscribers = set()
self.arvados_node = None
+ self.consecutive_idle_count = consecutive_idle_count
+ self.consecutive_idle = 0
self._later.update_arvados_node(arvados_node)
self.last_shutdown_opening = None
self._later.consider_shutdown()
reason for the decision.
"""
+ # If this node's size is invalid (because it has a stale arvados_node_size
+ # tag), return True so that it's properly shut down.
+ if self.cloud_node.size.id == 'invalid':
+ return (True, "node's size tag '%s' not recognizable" % (self.cloud_node.extra['arvados_node_size'],))
+
# Collect states and then consult state transition table whether we
# should shut down. Possible states are:
# crunch_worker_state = ['unpaired', 'busy', 'idle', 'down']
else:
boot_grace = "boot exceeded"
- # API server side not implemented yet.
- idle_grace = 'idle exceeded'
+ if crunch_worker_state == "idle":
+ # Must report as "idle" at least "consecutive_idle_count" times
+ if self.consecutive_idle < self.consecutive_idle_count:
+ idle_grace = 'idle wait'
+ else:
+ idle_grace = 'idle exceeded'
+ else:
+ idle_grace = 'not idle'
node_state = (crunch_worker_state, window, boot_grace, idle_grace)
t = transitions[node_state]
if arvados_node is not None:
self.arvados_node = arvados_node
self._update.sync_node(self.cloud_node, self.arvados_node)
+ if self.arvados_node['crunch_worker_state'] == "idle":
+ self.consecutive_idle += 1
+ else:
+ self.consecutive_idle = 0
self._later.consider_shutdown()
from __future__ import absolute_import, print_function
-import subprocess
+import subprocess32 as subprocess
import time
from . import ComputeNodeMonitorActor
return driver_class(**auth_kwargs)
@RetryMixin._retry()
- def _set_sizes(self):
- self.sizes = {sz.id: sz for sz in self.real.list_sizes()}
+ def sizes(self):
+ if self._sizes is None:
+ self._sizes = {sz.id: sz for sz in self.real.list_sizes()}
+ return self._sizes
def __init__(self, auth_kwargs, list_kwargs, create_kwargs,
driver_class, retry_wait=1, max_retry_wait=180):
if new_pair is not None:
self.create_kwargs[new_pair[0]] = new_pair[1]
- self._set_sizes()
+ self._sizes = None
def _init_ping_host(self, ping_host):
self.ping_host = ping_host
try:
kwargs = self.create_kwargs.copy()
kwargs.update(self.arvados_create_kwargs(size, arvados_node))
- kwargs['size'] = size
+ kwargs['size'] = size.real
return self.real.create_node(**kwargs)
except CLOUD_ERRORS as create_error:
# Workaround for bug #6702: sometimes the create node request
def arvados_create_kwargs(self, size, arvados_node):
tags = {
+ # Set up tag indicating the Arvados assigned Cloud Size id.
+ 'arvados_node_size': size.id,
'booted_at': time.strftime(ARVADOS_TIMEFMT, time.gmtime()),
'arv-ping-url': self._make_ping_url(arvados_node)
}
# Do our own filtering based on tag.
nodes = [node for node in
super(ComputeNodeDriver, self).list_nodes(ex_fetch_nic=False, ex_fetch_power_state=False)
- if node.extra["tags"].get("arvados-class") == self.tags["arvados-class"]]
+ if node.extra.get("tags", {}).get("arvados-class") == self.tags["arvados-class"]]
for n in nodes:
# Need to populate Node.size
if not n.size:
- n.size = self.sizes[n.extra["properties"]["hardwareProfile"]["vmSize"]]
+ n.size = self.sizes()[n.extra["properties"]["hardwareProfile"]["vmSize"]]
+ n.extra['arvados_node_size'] = n.extra.get('tags', {}).get('arvados_node_size') or n.size.id
return nodes
def broken(self, cloud_node):
nodelist = super(ComputeNodeDriver, self).list_nodes()
for node in nodelist:
self._ensure_private_ip(node)
- node.size = self.sizes["1"]
+ node.size = self.sizes()["1"]
return nodelist
def create_node(self, size, arvados_node):
"VolumeSize": volsize,
"VolumeType": "gp2"
}}]
+ if size.preemptible:
+ # Request a Spot instance for this node
+ kw['ex_spot_market'] = True
return kw
def sync_node(self, cloud_node, arvados_node):
self.real.ex_create_tags(cloud_node,
{'Name': arvados_node_fqdn(arvados_node)})
+ def create_node(self, size, arvados_node):
+ # Set up tag indicating the Arvados assigned Cloud Size id.
+ self.create_kwargs['ex_metadata'].update({'arvados_node_size': size.id})
+ return super(ComputeNodeDriver, self).create_node(size, arvados_node)
+
def list_nodes(self):
# Need to populate Node.size
nodes = super(ComputeNodeDriver, self).list_nodes()
for n in nodes:
if not n.size:
- n.size = self.sizes[n.extra["instance_type"]]
+ n.size = self.sizes()[n.extra["instance_type"]]
+ n.extra['arvados_node_size'] = n.extra.get('tags', {}).get('arvados_node_size') or n.size.id
return nodes
@classmethod
super(ComputeNodeDriver, self).__init__(
auth_kwargs, list_kwargs, create_kwargs,
driver_class)
- self._sizes_by_id = {sz.id: sz for sz in self.sizes.itervalues()}
self._disktype_links = {dt.name: self._object_link(dt)
for dt in self.real.ex_list_disktypes()}
'ex_disks_gce_struct': disks,
}
result['ex_metadata'].update({
- 'arv-ping-url': self._make_ping_url(arvados_node),
- 'booted_at': time.strftime(ARVADOS_TIMEFMT, time.gmtime()),
- 'hostname': arvados_node_fqdn(arvados_node),
- })
+ 'arvados_node_size': size.id,
+ 'arv-ping-url': self._make_ping_url(arvados_node),
+ 'booted_at': time.strftime(ARVADOS_TIMEFMT, time.gmtime()),
+ 'hostname': arvados_node_fqdn(arvados_node),
+ })
return result
-
def list_nodes(self):
# The GCE libcloud driver only supports filtering node lists by zone.
# Do our own filtering based on tag list.
nodelist = [node for node in
super(ComputeNodeDriver, self).list_nodes()
if self.node_tags.issubset(node.extra.get('tags', []))]
- # As of 0.18, the libcloud GCE driver sets node.size to the size's name.
- # It's supposed to be the actual size object. Check that it's not,
- # and monkeypatch the results when that's the case.
- if nodelist and not hasattr(nodelist[0].size, 'id'):
- for node in nodelist:
- node.size = self._sizes_by_id[node.size]
+ for node in nodelist:
+ # As of 0.18, the libcloud GCE driver sets node.size to the size's name.
+ # It's supposed to be the actual size object. Check that it's not,
+ # and monkeypatch the results when that's the case.
+ if not hasattr(node.size, 'id'):
+ node.size = self.sizes()[node.size]
+ # Get arvados-assigned cloud size id
+ node.extra['arvados_node_size'] = node.extra.get('metadata', {}).get('arvados_node_size') or node.size.id
return nodelist
@classmethod
from .baseactor import BaseNodeManagerActor
+from functools import partial
from libcloud.common.types import LibcloudError
from libcloud.common.exceptions import BaseHTTPError
'Daemon': {'min_nodes': '0',
'max_nodes': '1',
'poll_time': '60',
+ 'cloudlist_poll_time': '0',
+ 'nodelist_poll_time': '0',
+ 'wishlist_poll_time': '0',
'max_poll_time': '300',
'poll_stale_after': '600',
'max_total_price': '0',
'boot_fail_after': str(sys.maxint),
'node_stale_after': str(60 * 60 * 2),
'watchdog': '600',
- 'node_mem_scaling': '0.95'},
+ 'node_mem_scaling': '0.95',
+ 'consecutive_idle_count': '2'},
'Manage': {'address': '127.0.0.1',
'port': '-1',
'ManagementToken': ''},
if not self.has_option(sec_name, opt_name):
self.set(sec_name, opt_name, value)
- def get_section(self, section, transformer=None):
+ def get_section(self, section, transformers={}, default_transformer=None):
+ transformer_map = {
+ str: self.get,
+ int: self.getint,
+ bool: self.getboolean,
+ float: self.getfloat,
+ }
result = self._dict()
for key, value in self.items(section):
+ transformer = None
+ if transformers.get(key) in transformer_map:
+ transformer = partial(transformer_map[transformers[key]], section)
+ elif default_transformer in transformer_map:
+ transformer = partial(transformer_map[default_transformer], section)
if transformer is not None:
try:
- value = transformer(value)
+ value = transformer(key)
except (TypeError, ValueError):
pass
result[key] = value
self.get_section('Cloud Create'),
driver_class=driver_class)
- def node_sizes(self, all_sizes):
+ def node_sizes(self):
"""Finds all acceptable NodeSizes for our installation.
Returns a list of (NodeSize, kwargs) pairs for each NodeSize object
returned by libcloud that matches a size listed in our config file.
"""
-
+ all_sizes = self.new_cloud_client().list_sizes()
size_kwargs = {}
+ section_types = {
+ 'instance_type': str,
+ 'price': float,
+ 'preemptible': bool,
+ }
for sec_name in self.sections():
sec_words = sec_name.split(None, 2)
if sec_words[0] != 'Size':
continue
- size_spec = self.get_section(sec_name, int)
- if 'price' in size_spec:
- size_spec['price'] = float(size_spec['price'])
+ size_spec = self.get_section(sec_name, section_types, int)
+ if 'preemptible' not in size_spec:
+ size_spec['preemptible'] = False
+ if 'instance_type' not in size_spec:
+ # Assume instance type is Size name if missing
+ size_spec['instance_type'] = sec_words[1]
+ size_spec['id'] = sec_words[1]
size_kwargs[sec_words[1]] = size_spec
# EC2 node sizes are identified by id. GCE sizes are identified by name.
matching_sizes = []
for size in all_sizes:
- if size.id in size_kwargs:
- matching_sizes.append((size, size_kwargs[size.id]))
- elif size.name in size_kwargs:
- matching_sizes.append((size, size_kwargs[size.name]))
+ matching_sizes += [
+ (size, size_kwargs[s]) for s in size_kwargs
+ if size_kwargs[s]['instance_type'] == size.id
+ or size_kwargs[s]['instance_type'] == size.name
+ ]
return matching_sizes
def shutdown_windows(self):
- return [int(n)
+ return [float(n)
for n in self.get('Cloud', 'shutdown_windows').split(',')]
node_setup_class=dispatch.ComputeNodeSetupActor,
node_shutdown_class=dispatch.ComputeNodeShutdownActor,
node_actor_class=dispatch.ComputeNodeMonitorActor,
- max_total_price=0):
+ max_total_price=0,
+ consecutive_idle_count=1):
super(NodeManagerDaemonActor, self).__init__()
self._node_setup = node_setup_class
self._node_shutdown = node_shutdown_class
self.poll_stale_after = poll_stale_after
self.boot_fail_after = boot_fail_after
self.node_stale_after = node_stale_after
+ self.consecutive_idle_count = consecutive_idle_count
self.last_polls = {}
for poll_name in ['server_wishlist', 'arvados_nodes', 'cloud_nodes']:
poll_actor = locals()[poll_name + '_actor']
poll_stale_after=self.poll_stale_after,
node_stale_after=self.node_stale_after,
cloud_client=self._cloud_driver,
- boot_fail_after=self.boot_fail_after)
+ boot_fail_after=self.boot_fail_after,
+ consecutive_idle_count=self.consecutive_idle_count)
actorTell = actor.tell_proxy()
actorTell.subscribe(self._later.node_can_shutdown)
self._cloud_nodes_actor.subscribe_to(cloud_node.id,
busy_count = counts["busy"]
wishlist_count = self._size_wishlist(size)
- self._logger.info("%s: wishlist %i, up %i (booting %i, unpaired %i, idle %i, busy %i), down %i, shutdown %i", size.name,
+ self._logger.info("%s: wishlist %i, up %i (booting %i, unpaired %i, idle %i, busy %i), down %i, shutdown %i", size.id,
wishlist_count,
up_count,
counts["booting"],
can_boot = int((self.max_total_price - total_price) / size.price)
if can_boot == 0:
self._logger.info("Not booting %s (price %s) because with it would exceed max_total_price of %s (current total_price is %s)",
- size.name, size.price, self.max_total_price, total_price)
+ size.id, size.price, self.max_total_price, total_price)
return can_boot
else:
return wanted
nodes_wanted = self._nodes_wanted(cloud_size)
if nodes_wanted < 1:
return None
- arvados_node = self.arvados_nodes.find_stale_node(self.node_stale_after)
- self._logger.info("Want %i more %s nodes. Booting a node.",
- nodes_wanted, cloud_size.name)
- new_setup = self._node_setup.start(
- timer_actor=self._timer,
- arvados_client=self._new_arvados(),
- arvados_node=arvados_node,
- cloud_client=self._new_cloud(),
- cloud_size=self.server_calculator.find_size(cloud_size.id)).proxy()
- self.booting[new_setup.actor_ref.actor_urn] = new_setup
- self.sizes_booting[new_setup.actor_ref.actor_urn] = cloud_size
-
- if arvados_node is not None:
- self.arvados_nodes[arvados_node['uuid']].assignment_time = (
- time.time())
- new_setup.subscribe(self._later.node_setup_finished)
+
+ if not self.cancel_node_shutdown(cloud_size):
+ arvados_node = self.arvados_nodes.find_stale_node(self.node_stale_after)
+ self._logger.info("Want %i more %s nodes. Booting a node.",
+ nodes_wanted, cloud_size.id)
+ new_setup = self._node_setup.start(
+ timer_actor=self._timer,
+ arvados_client=self._new_arvados(),
+ arvados_node=arvados_node,
+ cloud_client=self._new_cloud(),
+ cloud_size=self.server_calculator.find_size(cloud_size.id))
+ self.booting[new_setup.actor_urn] = new_setup.proxy()
+ self.sizes_booting[new_setup.actor_urn] = cloud_size
+
+ if arvados_node is not None:
+ self.arvados_nodes[arvados_node['uuid']].assignment_time = (
+ time.time())
+ new_setup.tell_proxy().subscribe(self._later.node_setup_finished)
+
if nodes_wanted > 1:
self._later.start_node(cloud_size)
if (nodes_excess < 1) or not self.booting:
return None
for key, node in self.booting.iteritems():
- if node and node.cloud_size.get().id == size.id and node.stop_if_no_cloud_node().get():
- del self.booting[key]
- del self.sizes_booting[key]
+ try:
+ if node and node.cloud_size.get().id == size.id and node.stop_if_no_cloud_node().get(2):
+ del self.booting[key]
+ del self.sizes_booting[key]
+ if nodes_excess > 1:
+ self._later.stop_booting_node(size)
+ return
+ except pykka.Timeout:
+ pass
- if nodes_excess > 1:
- self._later.stop_booting_node(size)
- break
+ @_check_poll_freshness
+ def cancel_node_shutdown(self, size):
+ # Go through shutdown actors and see if there are any of the appropriate size that can be cancelled
+ for record in self.cloud_nodes.nodes.itervalues():
+ try:
+ if (record.shutdown_actor is not None and
+ record.cloud_node.size.id == size.id and
+ record.shutdown_actor.cancel_shutdown("Node size is in wishlist").get(2)):
+ return True
+ except (pykka.ActorDeadError, pykka.Timeout) as e:
+ pass
+ return False
def _begin_node_shutdown(self, node_actor, cancellable):
cloud_node_obj = node_actor.cloud_node.get()
import logging
import re
-import subprocess
+import subprocess32 as subprocess
import arvados.util
that would best satisfy the jobs, choosing the cheapest size that
satisfies each job, and ignoring jobs that can't be satisfied.
"""
+ class InvalidCloudSize(object):
+ """
+ Dummy CloudSizeWrapper-like class, to be used when a cloud node doesn't
+ have a recognizable arvados_node_size tag.
+ """
+ def __init__(self):
+ self.id = 'invalid'
+ self.name = 'invalid'
+ self.ram = 0
+ self.disk = 0
+ self.scratch = 0
+ self.cores = 0
+ self.bandwidth = 0
+ # price is multiplied by 1000 to get the node weight
+ # the maximum node weight is 4294967280
+ # so use invalid node weight 4294967 * 1000 = 4294967000
+ self.price = 4294967
+ self.preemptible = False
+ self.extra = {}
+
+ def meets_constraints(self, **kwargs):
+ return False
+
class CloudSizeWrapper(object):
def __init__(self, real_size, node_mem_scaling, **kwargs):
self.disk = 0
self.scratch = self.disk * 1000
self.ram = int(self.ram * node_mem_scaling)
+ self.preemptible = False
for name, override in kwargs.iteritems():
+ if name == 'instance_type': continue
if not hasattr(self, name):
raise ValueError("unrecognized size field '%s'" % (name,))
setattr(self, name, override)
wants = {'cores': want_value('min_cores_per_node'),
'ram': want_value('min_ram_mb_per_node'),
'scratch': want_value('min_scratch_mb_per_node')}
+ # EC2 node sizes are identified by id. GCE sizes are identified by name.
for size in self.cloud_sizes:
if (size.meets_constraints(**wants) and
- (specified_size is None or size.id == specified_size)):
- return size
+ (specified_size is None or
+ size.id == specified_size or size.name == specified_size)):
+ return size
return None
def servers_for_queue(self, queue):
"Job's min_nodes constraint is greater than the configured "
"max_nodes (%d)" % self.max_nodes)
elif (want_count*cloud_size.price <= self.max_price):
- servers.extend([cloud_size.real] * want_count)
+ servers.extend([cloud_size] * want_count)
else:
unsatisfiable_jobs[job['uuid']] = (
"Job's price (%d) is above system's max_price "
for s in self.cloud_sizes:
if s.id == sizeid:
return s
- return None
+ return self.InvalidCloudSize()
class JobQueueMonitorActor(clientactor.RemotePollLoopActor):
job_uuid,
error)
self._logger.debug("Calculated wishlist: %s",
- ', '.join(s.name for s in server_list) or "(empty)")
+ ', '.join(s.id for s in server_list) or "(empty)")
return super(JobQueueMonitorActor, self)._got_response(server_list)
return root_logger
def build_server_calculator(config):
- cloud_size_list = config.node_sizes(config.new_cloud_client().list_sizes())
+ cloud_size_list = config.node_sizes()
if not cloud_size_list:
abort("No valid node sizes configured")
return ServerCalculator(cloud_size_list,
config.getfloat('Daemon', 'node_mem_scaling'))
def launch_pollers(config, server_calculator):
- poll_time = config.getint('Daemon', 'poll_time')
+ poll_time = config.getfloat('Daemon', 'poll_time')
max_poll_time = config.getint('Daemon', 'max_poll_time')
+ cloudlist_poll_time = config.getfloat('Daemon', 'cloudlist_poll_time') or poll_time
+ nodelist_poll_time = config.getfloat('Daemon', 'nodelist_poll_time') or poll_time
+ wishlist_poll_time = config.getfloat('Daemon', 'wishlist_poll_time') or poll_time
+
timer = TimedCallBackActor.start(poll_time / 10.0).tell_proxy()
cloud_node_poller = CloudNodeListMonitorActor.start(
- config.new_cloud_client(), timer, server_calculator, poll_time, max_poll_time).tell_proxy()
+ config.new_cloud_client(), timer, server_calculator, cloudlist_poll_time, max_poll_time).tell_proxy()
arvados_node_poller = ArvadosNodeListMonitorActor.start(
- config.new_arvados_client(), timer, poll_time, max_poll_time).tell_proxy()
+ config.new_arvados_client(), timer, nodelist_poll_time, max_poll_time).tell_proxy()
job_queue_poller = JobQueueMonitorActor.start(
config.new_arvados_client(), timer, server_calculator,
config.getboolean('Arvados', 'jobs_queue'),
config.getboolean('Arvados', 'slurm_queue'),
- poll_time, max_poll_time
+ wishlist_poll_time, max_poll_time
).tell_proxy()
return timer, cloud_node_poller, arvados_node_poller, job_queue_poller
config.getint('Daemon', 'boot_fail_after'),
config.getint('Daemon', 'node_stale_after'),
node_setup, node_shutdown, node_monitor,
- max_total_price=config.getfloat('Daemon', 'max_total_price')).tell_proxy()
+ max_total_price=config.getfloat('Daemon', 'max_total_price'),
+ consecutive_idle_count=config.getint('Daemon', 'consecutive_idle_count'),).tell_proxy()
watchdog = WatchdogActor.start(config.getint('Daemon', 'watchdog'),
cloud_node_poller.actor_ref,
from __future__ import absolute_import, print_function
-import subprocess
+import subprocess32 as subprocess
from . import clientactor
from . import config
def _send_request(self):
nodes = self._client.list_nodes()
for n in nodes:
- # Replace with libcloud NodeSize object with compatible
+ # Replace the libcloud NodeSize object with compatible
# CloudSizeWrapper object which merges the size info reported from
# the cloud with size information from the configuration file.
- n.size = self._calculator.find_size(n.size.id)
+ n.size = self._calculator.find_size(n.extra['arvados_node_size'])
return nodes
global all_nodes, create_calls
create_calls += 1
nodeid = "node%i" % create_calls
+ if ex_tags is None:
+ ex_tags = {}
+ ex_tags.update({'arvados_node_size': size.id})
n = Node(nodeid, nodeid, NodeState.RUNNING, [], [], self, size=size, extra={"tags": ex_tags})
all_nodes.append(n)
if ex_customdata:
ping_url = re.search(r"echo '(.*)' > /var/tmp/arv-node-data/arv-ping-url", ex_customdata).groups(1)[0]
if ex_userdata:
ping_url = ex_userdata
- if ex_metadata:
+ elif ex_metadata:
ping_url = ex_metadata["arv-ping-url"]
ping_url += "&instance_id=" + nodeid
ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
create_calls += 1
if create_calls < 2:
raise RateLimitReachedError(429, "Rate limit exceeded",
- headers={'retry-after': '12'})
+ headers={'retry-after': '2'})
elif create_calls < 3:
raise BaseHTTPError(429, "Rate limit exceeded",
- {'retry-after': '2'})
+ {'retry-after': '1'})
else:
return super(RetryDriver, self).create_node(name=name,
size=size,
auth=auth,
ex_metadata=ex_metadata,
ex_userdata=ex_userdata)
- n.extra = {"launch_time": time.strftime(ARVADOS_TIMEFMT, time.gmtime())[:-1]}
+ n.extra = {
+ "launch_time": time.strftime(ARVADOS_TIMEFMT, time.gmtime())[:-1],
+ "tags" : {
+ "arvados_node_size": size.id
+ }
+ }
return n
def list_sizes(self, **kwargs):
ex_metadata=ex_metadata)
n.extra = {
"metadata": {
- "items": [{"key": k, "value": v} for k,v in ex_metadata.iteritems()]
+ "items": [{"key": k, "value": v} for k,v in ex_metadata.iteritems()],
+ "arvados_node_size": size.id
},
"zone": "fake"
}
# an Arvados node that hasn't been updated for this long.
node_stale_after = 14400
+# Number of consecutive times a node must report as "idle" before it
+# will be considered eligible for shutdown. Node status is checked
+# each poll period, and node can go idle at any point during a poll
+# period (meaning a node could be reported as idle that has only been
+# idle for 1 second). With a 60 second poll period, three consecutive
+# status updates of "idle" suggests the node has been idle at least
+# 121 seconds.
+consecutive_idle_count = 3
+
# Scaling factor to be applied to nodes' available RAM size. Usually there's a
# variable discrepancy between the advertised RAM value on cloud nodes and the
# actual amount available.
# File path for Certificate Authorities
certs_file = /etc/ssl/certs/ca-certificates.crt
+
[Logging]
# Log file path
file = /var/log/arvados/node-manager.log
# an Arvados node that hasn't been updated for this long.
node_stale_after = 14400
+# Number of consecutive times a node must report as "idle" before it
+# will be considered eligible for shutdown. Node status is checked
+# each poll period, and node can go idle at any point during a poll
+# period (meaning a node could be reported as idle that has only been
+# idle for 1 second). With a 60 second poll period, three consecutive
+# status updates of "idle" suggests the node has been idle at least
+# 121 seconds.
+consecutive_idle_count = 3
+
# Scaling factor to be applied to nodes' available RAM size. Usually there's a
# variable discrepancy between the advertised RAM value on cloud nodes and the
# actual amount available.
# You may also want to define the amount of scratch space (expressed
# in GB) for Crunch jobs. You can also override Amazon's provided
# data fields (such as price per hour) by setting them here.
+#
+# Additionally, you can ask for a preemptible instance (AWS's spot instance)
+# by adding the appropriate boolean configuration flag. If you want to have
+# both spot & reserved versions of the same size, you can do so by renaming
+# the Size section and specifying the instance type inside it.
[Size m4.large]
cores = 2
price = 0.126
scratch = 100
+[Size m4.large.spot]
+instance_type = m4.large
+preemptible = true
+cores = 2
+price = 0.126
+scratch = 100
+
[Size m4.xlarge]
cores = 4
price = 0.252
# an Arvados node that hasn't been updated for this long.
node_stale_after = 14400
+# Number of consecutive times a node must report as "idle" before it
+# will be considered eligible for shutdown. Node status is checked
+# each poll period, and node can go idle at any point during a poll
+# period (meaning a node could be reported as idle that has only been
+# idle for 1 second). With a 60 second poll period, three consecutive
+# status updates of "idle" suggests the node has been idle at least
+# 121 seconds.
+consecutive_idle_count = 3
+
# Scaling factor to be applied to nodes' available RAM size. Usually there's a
# variable discrepancy between the advertised RAM value on cloud nodes and the
# actual amount available.
('share/doc/arvados-node-manager', ['agpl-3.0.txt', 'README.rst']),
],
install_requires=[
- 'apache-libcloud>=2.3',
+ 'apache-libcloud>=2.3.1.dev1',
'arvados-python-client>=0.1.20170731145219',
'future',
'pykka',
'python-daemon',
- 'setuptools'
+ 'setuptools',
+ 'subprocess32>=3.5.1',
+ ],
+ dependency_links=[
+ "https://github.com/curoverse/libcloud/archive/apache-libcloud-2.3.1.dev1.zip"
],
test_suite='tests',
tests_require=[
'requests',
'pbr<1.7.0',
'mock>=1.0',
- 'apache-libcloud>=2.3',
+ 'apache-libcloud>=2.3.1.dev1',
+ 'subprocess32>=3.5.1',
],
zip_safe=False
)
# a snapshot of internal state.
# Management server listening address (default 127.0.0.1)
-#address = 0.0.0.0
+address = 0.0.0.0
# Management server port number (default -1, server is disabled)
-#port = 8989
+port = 8989
+
+MangementToken = xxx
[Daemon]
# The dispatcher can customize the start and stop procedure for
max_total_price = 0
# Poll Azure nodes and Arvados for new information every N seconds.
-poll_time = 5
+poll_time = 0.5
# Polls have exponential backoff when services fail to respond.
# This is the longest time to wait between polls.
-max_poll_time = 300
+max_poll_time = 1
# If Node Manager can't succesfully poll a service for this long,
# it will never start or stop compute nodes, on the assumption that its
# information is too outdated.
-poll_stale_after = 600
+poll_stale_after = 1
# If Node Manager boots a cloud node, and it does not pair with an Arvados
# node before this long, assume that there was a cloud bootstrap failure and
# Azure bills by the minute, so it makes sense to agressively shut down idle
# nodes. Specify at least two windows. You can add as many as you need beyond
# that.
-shutdown_windows = 1, 999999
+shutdown_windows = 0.05, 999999
[Cloud Credentials]
# Use "azure account list" with the azure CLI to get these values.
max_total_price = 0
# Poll Azure nodes and Arvados for new information every N seconds.
-poll_time = 5
+poll_time = 0.5
# Polls have exponential backoff when services fail to respond.
# This is the longest time to wait between polls.
-max_poll_time = 300
+max_poll_time = 1
# If Node Manager can't succesfully poll a service for this long,
# it will never start or stop compute nodes, on the assumption that its
# information is too outdated.
-poll_stale_after = 600
+poll_stale_after = 1
# If Node Manager boots a cloud node, and it does not pair with an Arvados
# node before this long, assume that there was a cloud bootstrap failure and
# Azure bills by the minute, so it makes sense to agressively shut down idle
# nodes. Specify at least two windows. You can add as many as you need beyond
# that.
-shutdown_windows = 1, 999999
+shutdown_windows = 0.05, 999999
[Cloud Credentials]
max_total_price = 0
# Poll Azure nodes and Arvados for new information every N seconds.
-poll_time = 5
+poll_time = 0.5
# Polls have exponential backoff when services fail to respond.
# This is the longest time to wait between polls.
-max_poll_time = 300
+max_poll_time = 1
# If Node Manager can't succesfully poll a service for this long,
# it will never start or stop compute nodes, on the assumption that its
# information is too outdated.
-poll_stale_after = 600
+poll_stale_after = 1
# If Node Manager boots a cloud node, and it does not pair with an Arvados
# node before this long, assume that there was a cloud bootstrap failure and
# Azure bills by the minute, so it makes sense to agressively shut down idle
# nodes. Specify at least two windows. You can add as many as you need beyond
# that.
-shutdown_windows = 1, 999999
+shutdown_windows = 0.05, 999999
[Cloud Credentials]
key = 00000000-0000-0000-0000-000000000000
"""
-import subprocess
+import subprocess32 as subprocess
import os
import sys
import re
return 0
-def remaining_jobs(g):
- update_script(os.path.join(fake_slurm, "sinfo"), "#!/bin/sh\n" +
- "\n".join("echo '%s|alloc|(null)'" % (v) for k,v in compute_nodes.items()))
-
- for k,v in all_jobs.items():
- all_jobs[k] = "Running"
-
- set_squeue(g)
-
- return 0
-
-
def node_busy(g):
update_script(os.path.join(fake_slurm, "sinfo"), "#!/bin/sh\n" +
"\n".join("echo '%s|idle|(null)'" % (v) for k,v in compute_nodes.items()))
def node_shutdown(g):
global compute_nodes
- del compute_nodes[g.group(1)]
- return 0
+ if g.group(1) in compute_nodes:
+ del compute_nodes[g.group(1)]
+ return 0
+ else:
+ return 1
+
def jobs_req(g):
global all_jobs
driver_class=driver_class,
ssh_key=os.path.join(fake_slurm, "id_rsa.pub")))
- # Tests must complete in less than 3 minutes.
- timeout = time.time() + 180
+ # Tests must complete in less than 30 seconds.
+ timeout = time.time() + 30
terminated = False
# Now start node manager
if code != 0:
detail.error("Check failed")
if not terminated:
- p.terminate()
+ p.kill()
terminated = True
if terminated:
detail.error("Exceeded timeout with actions remaining: %s", actions)
code += 1
if not terminated:
- p.terminate()
+ p.kill()
terminated = True
k, v = actions[0]
code += v(g)
if code != 0:
detail.error("Action failed")
- p.terminate()
+ p.kill()
terminated = True
if not actions:
- p.terminate()
+ p.kill()
terminated = True
except KeyboardInterrupt:
p.kill()
],
# Checks (things that shouldn't happen)
{
- r".*Suggesting shutdown because node state is \('down', .*\)": fail,
r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)": partial(expect_count, 4),
r".*Setting node quota.*": fail,
},
(r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)", node_paired),
(r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)", node_paired),
(r".*ComputeNodeMonitorActor\..*\.([^[]*).*Not eligible for shut down because node state is \('busy', 'open', .*\)", node_busy),
- (r".*ComputeNodeMonitorActor\..*\.([^[]*).*Suggesting shutdown because node state is \('idle', 'open', .*\)", remaining_jobs),
+ (r".*ComputeNodeMonitorActor\..*\.([^[]*).*Suggesting shutdown because node state is \('idle', 'open', .*\)", noop),
(r".*ComputeNodeShutdownActor\..*\.([^[]*).*Shutdown success", node_shutdown),
(r".*ComputeNodeShutdownActor\..*\.([^[]*).*Shutdown success", node_shutdown)
],
# Checks (things that shouldn't happen)
{
- r".*Suggesting shutdown because node state is \('down', .*\)": fail,
r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)": partial(expect_count, 2),
r".*Sending create_node request.*": partial(expect_count, 5)
},
(r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)", node_paired),
(r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)", node_paired),
(r".*ComputeNodeMonitorActor\..*\.([^[]*).*Not eligible for shut down because node state is \('busy', 'open', .*\)", node_busy),
- (r".*ComputeNodeMonitorActor\..*\.([^[]*).*Suggesting shutdown because node state is \('idle', 'open', .*\)", remaining_jobs),
+ (r".*ComputeNodeMonitorActor\..*\.([^[]*).*Suggesting shutdown because node state is \('idle', 'open', .*\)", noop),
(r".*ComputeNodeShutdownActor\..*\.([^[]*).*Shutdown success", node_shutdown),
(r".*ComputeNodeShutdownActor\..*\.([^[]*).*Shutdown success", node_shutdown),
(r".*sending request", jobs_req),
],
# Checks (things that shouldn't happen)
{
- r".*Suggesting shutdown because node state is \('down', .*\)": fail,
r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)": partial(expect_count, 6),
r".*Sending create_node request.*": partial(expect_count, 9)
},
# Actions (pattern -> action)
[
(r".*Daemon started", set_squeue),
- (r".*Rate limit exceeded - scheduling retry in 12 seconds", noop),
(r".*Rate limit exceeded - scheduling retry in 2 seconds", noop),
+ (r".*Rate limit exceeded - scheduling retry in 1 seconds", noop),
(r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)", noop),
],
# Checks (things that shouldn't happen)
],
# Checks (things that shouldn't happen)
{
- r".*Suggesting shutdown because node state is \('down', .*\)": fail,
r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)": partial(expect_count, 1),
r".*Setting node quota.*": fail,
},
],
# Checks (things that shouldn't happen)
{
- r".*Suggesting shutdown because node state is \('down', .*\)": fail,
r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)": partial(expect_count, 1),
r".*Setting node quota.*": fail,
},
self.make_actor()
self.shutdowns._set_state(True, 600)
self.assertEquals(self.node_actor.shutdown_eligible().get(self.TIMEOUT),
- (False, "node state is ('unpaired', 'open', 'boot wait', 'idle exceeded')"))
+ (False, "node state is ('unpaired', 'open', 'boot wait', 'not idle')"))
+
+ def test_shutdown_when_invalid_cloud_node_size(self):
+ self.make_mocks(1)
+ self.cloud_mock.size.id = 'invalid'
+ self.cloud_mock.extra['arvados_node_size'] = 'stale.type'
+ self.make_actor()
+ self.shutdowns._set_state(True, 600)
+ self.assertEquals((True, "node's size tag 'stale.type' not recognizable"),
+ self.node_actor.shutdown_eligible().get(self.TIMEOUT))
def test_shutdown_without_arvados_node(self):
self.make_actor(start_time=0)
self.shutdowns._set_state(True, 600)
- self.assertEquals((True, "node state is ('down', 'open', 'boot exceeded', 'idle exceeded')"),
+ self.assertEquals((True, "node state is ('down', 'open', 'boot exceeded', 'not idle')"),
self.node_actor.shutdown_eligible().get(self.TIMEOUT))
def test_shutdown_missing(self):
last_ping_at='1970-01-01T01:02:03.04050607Z')
self.make_actor(10, arv_node)
self.shutdowns._set_state(True, 600)
- self.assertEquals((True, "node state is ('down', 'open', 'boot wait', 'idle exceeded')"),
+ self.assertEquals((True, "node state is ('down', 'open', 'boot wait', 'not idle')"),
self.node_actor.shutdown_eligible().get(self.TIMEOUT))
def test_shutdown_running_broken(self):
self.make_actor(12, arv_node)
self.shutdowns._set_state(True, 600)
self.cloud_client.broken.return_value = True
- self.assertEquals((True, "node state is ('down', 'open', 'boot wait', 'idle exceeded')"),
+ self.assertEquals((True, "node state is ('down', 'open', 'boot wait', 'not idle')"),
self.node_actor.shutdown_eligible().get(self.TIMEOUT))
def test_shutdown_missing_broken(self):
self.make_actor(11, arv_node)
self.shutdowns._set_state(True, 600)
self.cloud_client.broken.return_value = True
- self.assertEquals(self.node_actor.shutdown_eligible().get(self.TIMEOUT), (True, "node state is ('down', 'open', 'boot wait', 'idle exceeded')"))
+ self.assertEquals(self.node_actor.shutdown_eligible().get(self.TIMEOUT), (True, "node state is ('down', 'open', 'boot wait', 'not idle')"))
def test_no_shutdown_when_window_closed(self):
self.make_actor(3, testutil.arvados_node_mock(3, job_uuid=None))
def test_no_shutdown_when_node_running_job(self):
self.make_actor(4, testutil.arvados_node_mock(4, job_uuid=True))
self.shutdowns._set_state(True, 600)
- self.assertEquals((False, "node state is ('busy', 'open', 'boot wait', 'idle exceeded')"),
+ self.assertEquals((False, "node state is ('busy', 'open', 'boot wait', 'not idle')"),
self.node_actor.shutdown_eligible().get(self.TIMEOUT))
def test_shutdown_when_node_state_unknown(self):
self.make_actor(5, testutil.arvados_node_mock(
5, crunch_worker_state='fail'))
self.shutdowns._set_state(True, 600)
- self.assertEquals((True, "node state is ('fail', 'open', 'boot wait', 'idle exceeded')"),
+ self.assertEquals((True, "node state is ('fail', 'open', 'boot wait', 'not idle')"),
self.node_actor.shutdown_eligible().get(self.TIMEOUT))
def test_no_shutdown_when_node_state_stale(self):
from __future__ import absolute_import, print_function
-import subprocess
+import subprocess32 as subprocess
import time
import unittest
ComputeNodeSetupActorTestCase, \
ComputeNodeUpdateActorTestCase
-@mock.patch('subprocess.check_output')
+@mock.patch('subprocess32.check_output')
class SLURMComputeNodeShutdownActorTestCase(ComputeNodeShutdownActorMixin,
unittest.TestCase):
ACTOR_CLASS = slurm_dispatch.ComputeNodeShutdownActor
super(SLURMComputeNodeShutdownActorTestCase,
self).test_uncancellable_shutdown()
-@mock.patch('subprocess.check_output')
+@mock.patch('subprocess32.check_output')
class SLURMComputeNodeUpdateActorTestCase(ComputeNodeUpdateActorTestCase):
ACTOR_CLASS = slurm_dispatch.ComputeNodeUpdateActor
class SLURMComputeNodeSetupActorTestCase(ComputeNodeSetupActorTestCase):
ACTOR_CLASS = slurm_dispatch.ComputeNodeSetupActor
- @mock.patch('subprocess.check_output')
+ @mock.patch('subprocess32.check_output')
def test_update_node_features(self, check_output):
# `scontrol update` happens only if the Arvados node record
# has a hostname. ComputeNodeSetupActorTestCase.make_mocks
self.wait_for_assignment(self.setup_actor, 'cloud_node')
check_output.assert_called_with(['scontrol', 'update', 'NodeName=compute99', 'Weight=1000', 'Features=instancetype=z1.test'])
- @mock.patch('subprocess.check_output')
+ @mock.patch('subprocess32.check_output')
def test_failed_arvados_calls_retried(self, check_output):
super(SLURMComputeNodeSetupActorTestCase, self).test_failed_arvados_calls_retried()
- @mock.patch('subprocess.check_output')
+ @mock.patch('subprocess32.check_output')
def test_subscribe(self, check_output):
super(SLURMComputeNodeSetupActorTestCase, self).test_subscribe()
- @mock.patch('subprocess.check_output')
+ @mock.patch('subprocess32.check_output')
def test_creation_with_arvados_node(self, check_output):
super(SLURMComputeNodeSetupActorTestCase, self).test_creation_with_arvados_node()
for an_error, is_cloud_error in errors:
self.driver_mock().create_node.side_effect = an_error
with self.assertRaises(an_error):
- driver.create_node('1', 'id_1')
+ driver.create_node(testutil.MockSize(1), 'id_1')
if is_cloud_error:
error_count += 1
self.assertEqual(error_count, status.tracker.get('create_node_errors'))
self.assertIn('ping_secret=ssshh',
create_method.call_args[1].get('ex_tags', {}).get('arv-ping-url', ""))
+ def test_create_includes_arvados_node_size(self):
+ arv_node = testutil.arvados_node_mock()
+ arv_node["hostname"] = None
+ size = testutil.MockSize(1)
+ driver = self.new_driver()
+ driver.create_node(size, arv_node)
+ create_method = self.driver_mock().create_node
+ self.assertTrue(create_method.called)
+ self.assertIn(
+ ('arvados_node_size', size.id),
+ create_method.call_args[1].get('ex_tags', {'tags': 'missing'}).items()
+ )
+
def test_name_from_new_arvados_node(self):
arv_node = testutil.arvados_node_mock(hostname=None)
driver = self.new_driver()
self.assertEqual('compute-000000000000063-zzzzz',
driver.arvados_create_kwargs(testutil.MockSize(1), arv_node)['name'])
-
-
def check_node_tagged(self, cloud_node, expected_tags):
tag_mock = self.driver_mock().ex_create_tags
self.assertTrue(tag_mock.called)
""",
driver.arvados_create_kwargs(testutil.MockSize(1), arv_node)['ex_customdata'])
+ def test_list_nodes_ignores_nodes_without_tags(self):
+ driver = self.new_driver(create_kwargs={"tag_arvados-class": "dynamic-compute"})
+ # Mock cloud node without tags
+ nodelist = [testutil.cloud_node_mock(1)]
+ self.driver_mock().list_nodes.return_value = nodelist
+ n = driver.list_nodes()
+ self.assertEqual([], n)
+
def test_create_raises_but_actually_succeeded(self):
arv_node = testutil.arvados_node_mock(1, hostname=None)
driver = self.new_driver(create_kwargs={"tag_arvados-class": "dynamic-compute"})
driver.create_node(testutil.MockSize(1), arv_node)
create_method = self.driver_mock().create_node
self.assertTrue(create_method.called)
+ self.assertIn(
+ ('test', 'testvalue'),
+ create_method.call_args[1].get('ex_metadata', {'arg': 'missing'}).items()
+ )
+
+ def test_create_includes_arvados_node_size(self):
+ arv_node = testutil.arvados_node_mock()
+ size = testutil.MockSize(1)
+ driver = self.new_driver()
+ driver.create_node(size, arv_node)
+ create_method = self.driver_mock().create_node
+ self.assertTrue(create_method.called)
+ self.assertIn(
+ ('arvados_node_size', size.id),
+ create_method.call_args[1].get('ex_metadata', {'arg': 'missing'}).items()
+ )
+
+ def test_create_preemptible_instance(self):
+ arv_node = testutil.arvados_node_mock()
+ driver = self.new_driver()
+ driver.create_node(testutil.MockSize(1, preemptible=True), arv_node)
+ create_method = self.driver_mock().create_node
+ self.assertTrue(create_method.called)
self.assertEqual(
- {'test':'testvalue'},
- create_method.call_args[1].get('ex_metadata', {'arg': 'missing'})
+ True,
+ create_method.call_args[1].get('ex_spot_market', 'arg missing')
)
def test_hostname_from_arvados_node(self):
metadata = self.driver_mock().create_node.call_args[1]['ex_metadata']
self.assertIn('ping_secret=ssshh', metadata.get('arv-ping-url'))
+ def test_create_includes_arvados_node_size(self):
+ arv_node = testutil.arvados_node_mock()
+ size = testutil.MockSize(1)
+ driver = self.new_driver()
+ driver.create_node(size, arv_node)
+ create_method = self.driver_mock().create_node
+ self.assertIn(
+ ('arvados_node_size', size.id),
+ create_method.call_args[1].get('ex_metadata', {'metadata':'missing'}).items()
+ )
+
def test_create_raises_but_actually_succeeded(self):
arv_node = testutil.arvados_node_mock(1, hostname=None)
driver = self.new_driver()
cores = 1
price = 0.8
+[Size 1.preemptible]
+instance_type = 1
+preemptible = true
+cores = 1
+price = 0.8
+
[Logging]
file = /dev/null
level = DEBUG
def test_list_sizes(self):
config = self.load_config()
- client = config.new_cloud_client()
- sizes = config.node_sizes(client.list_sizes())
- self.assertEqual(1, len(sizes))
+ sizes = config.node_sizes()
+ self.assertEqual(2, len(sizes))
size, kwargs = sizes[0]
self.assertEqual('Small', size.name)
self.assertEqual(1, kwargs['cores'])
self.assertEqual(0.8, kwargs['price'])
+ # preemptible is False by default
+ self.assertEqual(False, kwargs['preemptible'])
+ # instance_type == arvados node size id by default
+ self.assertEqual(kwargs['id'], kwargs['instance_type'])
+ # Now retrieve the preemptible version
+ size, kwargs = sizes[1]
+ self.assertEqual('Small', size.name)
+ self.assertEqual('1.preemptible', kwargs['id'])
+ self.assertEqual(1, kwargs['cores'])
+ self.assertEqual(0.8, kwargs['price'])
+ self.assertEqual(True, kwargs['preemptible'])
+ self.assertEqual('1', kwargs['instance_type'])
+
def test_default_node_mem_scaling(self):
config = self.load_config()
monitor = self.monitor_list()[0].proxy()
self.daemon.node_can_shutdown(monitor).get(self.TIMEOUT)
self.assertTrue(self.node_shutdown.start.called)
+ getmock = mock.MagicMock()
+ getmock.get.return_value = False
+ self.last_shutdown.cancel_shutdown.return_value = getmock
self.daemon.update_server_wishlist(
[testutil.MockSize(6)]).get(self.TIMEOUT)
self.busywait(lambda: self.node_setup.start.called)
+ def test_nodes_shutting_down_cancelled(self):
+ size = testutil.MockSize(6)
+ cloud_node = testutil.cloud_node_mock(6, size=size)
+ self.make_daemon([cloud_node], [testutil.arvados_node_mock(6, crunch_worker_state='down')],
+ avail_sizes=[(size, {"cores":1})])
+ self.assertEqual(1, self.alive_monitor_count())
+ monitor = self.monitor_list()[0].proxy()
+ self.daemon.node_can_shutdown(monitor).get(self.TIMEOUT)
+ self.assertTrue(self.node_shutdown.start.called)
+ self.daemon.update_server_wishlist(
+ [testutil.MockSize(6)]).get(self.TIMEOUT)
+ self.busywait(lambda: self.last_shutdown.cancel_shutdown.called)
+
def test_nodes_shutting_down_not_replaced_at_max_nodes(self):
cloud_node = testutil.cloud_node_mock(7)
self.make_daemon([cloud_node], [testutil.arvados_node_mock(7)],
super(JobQueueMonitorActorTestCase, self).build_monitor(*args, **kwargs)
self.client.jobs().queue().execute.side_effect = side_effect
- @mock.patch("subprocess.check_call")
- @mock.patch("subprocess.check_output")
+ @mock.patch("subprocess32.check_call")
+ @mock.patch("subprocess32.check_output")
def test_unsatisfiable_jobs(self, mock_squeue, mock_scancel):
job_uuid = 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'
container_uuid = 'yyyyy-dz642-yyyyyyyyyyyyyyy'
self.client.jobs().cancel.assert_called_with(uuid=job_uuid)
mock_scancel.assert_called_with(['scancel', '--name='+container_uuid])
- @mock.patch("subprocess.check_output")
+ @mock.patch("subprocess32.check_output")
def test_subscribers_get_server_lists(self, mock_squeue):
mock_squeue.return_value = ""
self.subscriber.assert_called_with([testutil.MockSize(1),
testutil.MockSize(2)])
- @mock.patch("subprocess.check_output")
+ @mock.patch("subprocess32.check_output")
def test_squeue_server_list(self, mock_squeue):
mock_squeue.return_value = """1|1024|0|(Resources)|zzzzz-dz642-zzzzzzzzzzzzzzy|(null)|1234567890
2|1024|0|(Resources)|zzzzz-dz642-zzzzzzzzzzzzzzz|(null)|1234567890
self.subscriber.assert_called_with([testutil.MockSize(1),
testutil.MockSize(2)])
- @mock.patch("subprocess.check_output")
+ @mock.patch("subprocess32.check_output")
def test_squeue_server_list_suffix(self, mock_squeue):
mock_squeue.return_value = """1|1024M|0|(ReqNodeNotAvail, UnavailableNodes:compute123)|zzzzz-dz642-zzzzzzzzzzzzzzy|(null)|1234567890
1|2G|0|(ReqNodeNotAvail)|zzzzz-dz642-zzzzzzzzzzzzzzz|(null)|1234567890
self.subscriber.assert_called_with([testutil.MockSize(1),
testutil.MockSize(2)])
- @mock.patch("subprocess.check_output")
+ @mock.patch("subprocess32.check_output")
def test_squeue_server_list_instancetype_constraint(self, mock_squeue):
mock_squeue.return_value = """1|1024|0|(Resources)|zzzzz-dz642-zzzzzzzzzzzzzzy|instancetype=z2.test|1234567890\n"""
super(JobQueueMonitorActorTestCase, self).build_monitor(jobqueue.ServerCalculator(
*args, **kwargs)
self.client.nodes().list().execute.side_effect = side_effect
- @mock.patch("subprocess.check_output")
+ @mock.patch("subprocess32.check_output")
def test_uuid_is_subscription_key(self, sinfo_mock):
sinfo_mock.return_value = ""
node = testutil.arvados_node_mock()
self.subscriber.assert_called_with(node)
self.assertEqual("down", node["crunch_worker_state"])
- @mock.patch("subprocess.check_output")
+ @mock.patch("subprocess32.check_output")
def test_update_from_sinfo(self, sinfo_mock):
sinfo_mock.return_value = """compute1|idle|instancetype=a1.test
compute2|alloc|(null)
self.public_ips = []
self.size = testutil.MockSize(1)
self.state = 0
+ self.extra = {'arvados_node_size': self.size.id}
def build_monitor(self, side_effect, *args, **kwargs):
class MockSize(object):
- def __init__(self, factor):
+ def __init__(self, factor, preemptible=False):
self.id = 'z{}.test'.format(factor)
self.name = 'test size '+self.id
self.ram = 128 * factor
self.bandwidth = 16 * factor
self.price = float(factor)
self.extra = {}
+ self.real = self
+ self.preemptible = preemptible
def __eq__(self, other):
return self.id == other.id
type wsConfig struct {
Client arvados.Client
- Postgres pgConfig
+ Postgres arvados.PostgreSQLConnection
PostgresPool int
Listen string
LogLevel string
Client: arvados.Client{
APIHost: "localhost:443",
},
- Postgres: pgConfig{
+ Postgres: arvados.PostgreSQLConnection{
"dbname": "arvados_production",
"user": "arvados",
"password": "xyzzy",
"context"
"database/sql"
"strconv"
- "strings"
"sync"
"sync/atomic"
"time"
"github.com/lib/pq"
)
-type pgConfig map[string]string
-
-func (c pgConfig) ConnectionString() string {
- s := ""
- for k, v := range c {
- s += k
- s += "='"
- s += strings.Replace(
- strings.Replace(v, `\`, `\\`, -1),
- `'`, `\'`, -1)
- s += "' "
- }
- return s
-}
-
type pgEventSource struct {
DataSource string
MaxOpenConns int
import (
"database/sql"
"fmt"
+ "os"
+ "path/filepath"
"sync"
"time"
- "git.curoverse.com/arvados.git/sdk/go/config"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
check "gopkg.in/check.v1"
)
type eventSourceSuite struct{}
-func testDBConfig() pgConfig {
- var railsDB struct {
- Test struct {
- Database string
- Username string
- Password string
- Host string
- }
- }
- err := config.LoadFile(&railsDB, "../api/config/database.yml")
+func testDBConfig() arvados.PostgreSQLConnection {
+ cfg, err := arvados.GetConfig(filepath.Join(os.Getenv("WORKSPACE"), "tmp", "arvados.yml"))
if err != nil {
panic(err)
}
- cfg := pgConfig{
- "dbname": railsDB.Test.Database,
- "host": railsDB.Test.Host,
- "password": railsDB.Test.Password,
- "user": railsDB.Test.Username,
+ cc, err := cfg.GetCluster("zzzzz")
+ if err != nil {
+ panic(err)
}
- return cfg
+ return cc.PostgreSQL.Connection
}
func testDB() *sql.DB {
- db, err := sql.Open("postgres", testDBConfig().ConnectionString())
+ db, err := sql.Open("postgres", testDBConfig().String())
if err != nil {
panic(err)
}
cfg := testDBConfig()
db := testDB()
pges := &pgEventSource{
- DataSource: cfg.ConnectionString(),
+ DataSource: cfg.String(),
QueueSize: 4,
}
go pges.Run()
srv.listener = ln
srv.eventSource = &pgEventSource{
- DataSource: srv.wsConfig.Postgres.ConnectionString(),
+ DataSource: srv.wsConfig.Postgres.String(),
MaxOpenConns: srv.wsConfig.PostgresPool,
QueueSize: srv.wsConfig.ServerEventQueue,
}
ADD crunch-setup.sh gitolite.rc \
keep-setup.sh common.sh createusers.sh \
logger runsu.sh waitforpostgres.sh \
- application_yml_override.py api-setup.sh \
+ yml_override.py api-setup.sh \
go-setup.sh \
/usr/local/lib/arvbox/
fi
blob_signing_key=$(cat /var/lib/arvados/blob_signing_key)
+if ! test -s /var/lib/arvados/management_token ; then
+ ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/management_token
+fi
+management_token=$(cat /var/lib/arvados/management_token)
+
# self signed key will be created by SSO server script.
test -s /var/lib/arvados/self-signed.key
default_collection_replication: 1
docker_image_formats: ["v2"]
keep_web_service_url: http://$localip:${services[keep-web]}/
+ ManagementToken: $management_token
EOF
-(cd config && /usr/local/lib/arvbox/application_yml_override.py)
+(cd config && /usr/local/lib/arvbox/yml_override.py application.yml)
if ! test -f /var/lib/arvados/api_database_pw ; then
ruby -e 'puts rand(2**128).to_s(36)' > /var/lib/arvados/api_database_pw
declare -A services
services=(
[workbench]=80
- [api]=8000
+ [api]=8004
+ [controller]=8003
+ [controller-ssl]=8000
[sso]=8900
[composer]=4200
[arv-git-httpd]=9001
RAILS_ENV=development
fi
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=$(cat /usr/src/arvados/services/api/superuser_token)
export CRUNCH_JOB_BIN=/usr/src/arvados/sdk/cli/bin/crunch-job
mkdir -p /var/lib/arvados/$1
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
echo $UUID > /var/lib/arvados/$1-uuid
fi
+management_token=$(cat /var/lib/arvados/management_token)
+
set +e
killall -HUP keepproxy
-exec /usr/local/bin/keepstore \
- -listen=:$2 \
- -enforce-permissions=true \
- -blob-signing-key-file=/var/lib/arvados/blob_signing_key \
- -data-manager-token-file=/var/lib/arvados/superuser_token \
- -max-buffers=20 \
- -volume=/var/lib/arvados/$1
+cat >/var/lib/arvados/$1.yml <<EOF
+Listen: ":$2"
+BlobSigningKeyFile: /var/lib/arvados/blob_signing_key
+SystemAuthTokenFile: /var/lib/arvados/superuser_token
+ManagementToken: $management_token
+MaxBuffers: 20
+Volumes:
+ - Type: Directory
+ Root: /var/lib/arvados/$1
+EOF
+
+exec /usr/local/bin/keepstore -config=/var/lib/arvados/$1.yml
exit
fi
-exec bundle exec passenger start --port=${services[api]} \
- --ssl --ssl-certificate=/var/lib/arvados/self-signed.pem \
- --ssl-certificate-key=/var/lib/arvados/self-signed.key
+exec bundle exec passenger start --port=${services[api]}
exit
fi
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
export PATH="$PATH:/var/lib/arvados/git/bin"
cd ~git
exit
fi
-echo "apiEndPoint: https://${localip}:${services[api]}" > /usr/src/composer/src/composer.yml
+echo "apiEndPoint: https://${localip}:${services[controller-ssl]}" > /usr/src/composer/src/composer.yml
exec node_modules/.bin/ng serve --host 0.0.0.0 --port 4200 --env=webdev
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+exec 2>&1
+set -ex -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+. /usr/local/lib/arvbox/go-setup.sh
+
+flock /var/lib/gopath/gopath.lock go get -t "git.curoverse.com/arvados.git/cmd/arvados-server"
+install $GOPATH/bin/arvados-server /usr/local/bin
+(cd /usr/local/bin && ln -sf arvados-server arvados-controller)
+
+if test "$1" = "--only-deps" ; then
+ exit
+fi
+
+uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
+database_pw=$(cat /var/lib/arvados/api_database_pw)
+
+mkdir -p /etc/arvados
+
+cat >/var/lib/arvados/cluster_config.yml <<EOF
+Clusters:
+ ${uuid_prefix}:
+ NodeProfiles:
+ "*":
+ arvados-controller:
+ Listen: ":${services[controller]}" # choose a port
+ arvados-api-server:
+ Listen: ":${services[api]}" # must match Rails server port in your Nginx config
+ PostgreSQL:
+ ConnectionPool: 32 # max concurrent connections per arvados server daemon
+ Connection:
+ # All parameters here are passed to the PG client library in a connection string;
+ # see https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-PARAMKEYWORDS
+ Host: localhost
+ User: arvados
+ Password: ${database_pw}
+ DBName: arvados_development
+ client_encoding: utf8
+EOF
+
+/usr/local/lib/arvbox/yml_override.py /var/lib/arvados/cluster_config.yml
+
+cp /var/lib/arvados/cluster_config.yml /etc/arvados/config.yml
+
+exec /usr/local/lib/arvbox/runsu.sh /usr/local/bin/arvados-controller
EOF
chmod +x /usr/local/bin/crunch-run.sh
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
+++ /dev/null
-/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
--- /dev/null
+#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e
+
+/usr/local/lib/arvbox/runsu.sh $0-service
+sv stop doc
exit
fi
-set -u
-
-cat <<EOF >/var/lib/arvados/doc-nginx.conf
-worker_processes auto;
-pid /var/lib/arvados/doc-nginx.pid;
-
-error_log stderr;
-daemon off;
-
-events {
- worker_connections 64;
-}
-
-http {
- access_log off;
- include /etc/nginx/mime.types;
- default_type application/octet-stream;
- server {
- listen ${services[doc]} default_server;
- listen [::]:${services[doc]} default_server;
- root /usr/src/arvados/doc/.site;
- index index.html;
- server_name _;
- }
-}
-EOF
-
cd /usr/src/arvados/doc
-bundle exec rake generate baseurl=http://$localip:${services[doc]} arvados_api_host=$localip:${services[api]} arvados_workbench_host=http://$localip
-
-exec nginx -c /var/lib/arvados/doc-nginx.conf
+bundle exec rake generate baseurl=http://$localip:${services[doc]} arvados_api_host=$localip:${services[controller-ssl]} arvados_workbench_host=http://$localip
mkdir -p /var/lib/arvados/git
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
$RAILS_ENV:
gitolite_url: /var/lib/arvados/git/repositories/gitolite-admin.git
gitolite_tmp: /var/lib/arvados/git
- arvados_api_host: $localip:${services[api]}
+ arvados_api_host: $localip:${services[controller-ssl]}
arvados_api_token: "$ARVADOS_API_TOKEN"
arvados_api_host_insecure: true
gitolite_arvados_git_user_key: "$git_user_key"
exit
fi
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
exit
fi
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+exec 2>&1
+set -ex -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+
+cat <<EOF >/var/lib/arvados/nginx.conf
+worker_processes auto;
+pid /var/lib/arvados/nginx.pid;
+
+error_log stderr;
+daemon off;
+
+events {
+ worker_connections 64;
+}
+
+http {
+ access_log off;
+ include /etc/nginx/mime.types;
+ default_type application/octet-stream;
+ server {
+ listen ${services[doc]} default_server;
+ listen [::]:${services[doc]} default_server;
+ root /usr/src/arvados/doc/.site;
+ index index.html;
+ server_name _;
+ }
+
+ upstream controller {
+ server localhost:${services[controller]};
+ }
+ server {
+ listen *:${services[controller-ssl]} ssl default_server;
+ server_name controller;
+ ssl_certificate "/var/lib/arvados/self-signed.pem";
+ ssl_certificate_key "/var/lib/arvados/self-signed.key";
+ location / {
+ proxy_pass http://controller;
+ proxy_set_header Host \$http_host;
+ proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
+ }
+ }
+}
+
+EOF
+
+exec nginx -c /var/lib/arvados/nginx.conf
+++ /dev/null
-/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
--- /dev/null
+#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e
+
+/usr/local/lib/arvbox/runsu.sh $0-service
+sv stop ready
waiting="$waiting crunch-dispatch"
fi
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
vm_ok=0
echo "Workbench is running at http://$localip"
rm -r /tmp/arvbox-ready
-
-sv stop ready >/dev/null
allow_account_registration: true
EOF
-(cd config && /usr/local/lib/arvbox/application_yml_override.py)
+(cd config && /usr/local/lib/arvbox/yml_override.py application.yml)
if ! test -f /var/lib/arvados/sso_database_pw ; then
ruby -e 'puts rand(2**128).to_s(36)' > /var/lib/arvados/sso_database_pw
cd /usr/src/arvados/services/login-sync
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
export ARVADOS_VIRTUAL_MACHINE_UUID=$(cat /var/lib/arvados/vm-uuid)
set -u
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
export ARVADOS_VIRTUAL_MACHINE_UUID=$(cat /var/lib/arvados/vm-uuid)
cat >/var/lib/arvados/arvados-ws.yml <<EOF
Client:
- APIHost: $localip:${services[api]}
+ APIHost: $localip:${services[controller-ssl]}
Insecure: true
Postgres:
dbname: arvados_$RAILS_ENV
cat >config/application.yml <<EOF
$RAILS_ENV:
secret_token: $secret_token
- arvados_login_base: https://$localip:${services[api]}/login
- arvados_v1_base: https://$localip:${services[api]}/arvados/v1
+ arvados_login_base: https://$localip:${services[controller-ssl]}/login
+ arvados_v1_base: https://$localip:${services[controller-ssl]}/arvados/v1
arvados_insecure_https: true
keep_web_download_url: http://$localip:${services[keep-web]}/c=%{uuid_or_pdh}
keep_web_url: http://$localip:${services[keep-web]}/c=%{uuid_or_pdh}
bundle exec rake assets:precompile
-(cd config && /usr/local/lib/arvbox/application_yml_override.py)
+(cd config && /usr/local/lib/arvbox/yml_override.py application.yml)
# SPDX-License-Identifier: AGPL-3.0
import yaml
+import sys
+
+fn = sys.argv[1]
try:
- with open("application.yml.override") as f:
+ with open(fn+".override") as f:
b = yaml.load(f)
except IOError:
exit()
-with open("application.yml") as f:
+with open(fn) as f:
a = yaml.load(f)
def recursiveMerge(a, b):
else:
return b
-with open("application.yml", "w") as f:
+with open(fn, "w") as f:
yaml.dump(recursiveMerge(a, b), f)
"revision": "d682213848ed68c0a260ca37d6dd5ace8423f5ba",
"revisionTime": "2017-12-05T20:32:29Z"
},
+ {
+ "checksumSHA1": "st4vb0GmDeoKbsfxdpNZ2MPl76M=",
+ "path": "github.com/StackExchange/wmi",
+ "revision": "cdffdb33acae0e14efff2628f9bae377b597840e",
+ "revisionTime": "2018-04-12T20:51:11Z"
+ },
{
"checksumSHA1": "spyv5/YFBjYyZLZa1U2LBfDR8PM=",
"path": "github.com/beorn7/perks/quantile",
"revision": "0ca9ea5df5451ffdf184b4428c902747c2c11cd7",
"revisionTime": "2017-03-27T23:54:44Z"
},
+ {
+ "checksumSHA1": "Kqv7bA4oJG0nPwQvGWDwGGaKONo=",
+ "path": "github.com/go-ole/go-ole",
+ "revision": "7a0fa49edf48165190530c675167e2f319a05268",
+ "revisionTime": "2018-06-25T08:58:08Z"
+ },
+ {
+ "checksumSHA1": "PArleDBtadu2qO4hJwHR8a3IOTA=",
+ "path": "github.com/go-ole/go-ole/oleutil",
+ "revision": "7a0fa49edf48165190530c675167e2f319a05268",
+ "revisionTime": "2018-06-25T08:58:08Z"
+ },
+ {
+ "checksumSHA1": "8UEp6v0Dczw/SlasE0DivB0mAHA=",
+ "path": "github.com/gogo/protobuf/jsonpb",
+ "revision": "30cf7ac33676b5786e78c746683f0d4cd64fa75b",
+ "revisionTime": "2018-05-09T16:24:41Z"
+ },
{
"checksumSHA1": "wn2shNJMwRZpvuvkf1s7h0wvqHI=",
"path": "github.com/gogo/protobuf/proto",
"revisionTime": "2018-01-04T10:21:28Z"
},
{
- "checksumSHA1": "iVfdaLxIDjfk2KLP8dCMIbsxZZM=",
- "path": "github.com/golang/protobuf/jsonpb",
- "revision": "1e59b77b52bf8e4b449a57e6f79f21226d571845",
- "revisionTime": "2017-11-13T18:07:20Z"
+ "checksumSHA1": "HPVQZu059/Rfw2bAWM538bVTcUc=",
+ "path": "github.com/gogo/protobuf/sortkeys",
+ "revision": "30cf7ac33676b5786e78c746683f0d4cd64fa75b",
+ "revisionTime": "2018-05-09T16:24:41Z"
},
{
- "checksumSHA1": "yqF125xVSkmfLpIVGrLlfE05IUk=",
- "path": "github.com/golang/protobuf/proto",
- "revision": "1e59b77b52bf8e4b449a57e6f79f21226d571845",
- "revisionTime": "2017-11-13T18:07:20Z"
+ "checksumSHA1": "SkxU1+wPGUJyLyQENrZtr2/OUBs=",
+ "path": "github.com/gogo/protobuf/types",
+ "revision": "30cf7ac33676b5786e78c746683f0d4cd64fa75b",
+ "revisionTime": "2018-05-09T16:24:41Z"
},
{
- "checksumSHA1": "Ylq6kq3KWBy6mu68oyEwenhNMdg=",
- "path": "github.com/golang/protobuf/ptypes/struct",
+ "checksumSHA1": "yqF125xVSkmfLpIVGrLlfE05IUk=",
+ "path": "github.com/golang/protobuf/proto",
"revision": "1e59b77b52bf8e4b449a57e6f79f21226d571845",
"revisionTime": "2017-11-13T18:07:20Z"
},
"revision": "1744e2970ca51c86172c8190fadad617561ed6e7",
"revisionTime": "2017-11-10T11:01:46Z"
},
+ {
+ "checksumSHA1": "q14d3C3xvWevU3dSv4P5K0+OSD0=",
+ "path": "github.com/shirou/gopsutil/cpu",
+ "revision": "63728fcf6b24475ecfea044e22242447666c2f52",
+ "revisionTime": "2018-07-05T13:28:12Z"
+ },
+ {
+ "checksumSHA1": "LZ9GloiGLTISmQ4dalK2XspH6Wo=",
+ "path": "github.com/shirou/gopsutil/host",
+ "revision": "63728fcf6b24475ecfea044e22242447666c2f52",
+ "revisionTime": "2018-07-05T13:28:12Z"
+ },
+ {
+ "checksumSHA1": "cyoqI0gryzjxGTkaAfyUqMiuUR0=",
+ "path": "github.com/shirou/gopsutil/internal/common",
+ "revision": "63728fcf6b24475ecfea044e22242447666c2f52",
+ "revisionTime": "2018-07-05T13:28:12Z"
+ },
+ {
+ "checksumSHA1": "vEQLjAO5T5K9zXblEMYdoaBZzj0=",
+ "path": "github.com/shirou/gopsutil/mem",
+ "revision": "63728fcf6b24475ecfea044e22242447666c2f52",
+ "revisionTime": "2018-07-05T13:28:12Z"
+ },
+ {
+ "checksumSHA1": "KMWFRa0DVpabo9d8euB4RYjUBQE=",
+ "path": "github.com/shirou/gopsutil/net",
+ "revision": "63728fcf6b24475ecfea044e22242447666c2f52",
+ "revisionTime": "2018-07-05T13:28:12Z"
+ },
+ {
+ "checksumSHA1": "fbO7c1gv1kSvWKOb/+5HUWFkBaA=",
+ "path": "github.com/shirou/gopsutil/process",
+ "revision": "63728fcf6b24475ecfea044e22242447666c2f52",
+ "revisionTime": "2018-07-05T13:28:12Z"
+ },
+ {
+ "checksumSHA1": "Nve7SpDmjsv6+rhkXAkfg/UQx94=",
+ "path": "github.com/shirou/w32",
+ "revision": "bb4de0191aa41b5507caa14b0650cdbddcd9280b",
+ "revisionTime": "2016-09-30T03:27:40Z"
+ },
{
"checksumSHA1": "8QeSG127zQqbA+YfkO1WkKx/iUI=",
"path": "github.com/src-d/gcfg",
projects / arvados.git / commitdiff
