coffee-script-source (1.12.2)
commonjs (0.2.7)
concurrent-ruby (1.1.5)
- crass (1.0.4)
+ crass (1.0.5)
deep_merge (1.2.1)
docile (1.3.1)
erubis (2.7.0)
railties (>= 4)
request_store (~> 1.0)
logstash-event (1.2.02)
- loofah (2.2.3)
+ loofah (2.3.1)
crass (~> 1.0.2)
nokogiri (>= 1.5.9)
mail (2.7.1)
net-ssh-gateway (2.0.0)
net-ssh (>= 4.0.0)
nio4r (2.3.1)
- nokogiri (1.10.4)
+ nokogiri (1.10.8)
mini_portile2 (~> 2.4.0)
npm-rails (0.2.1)
rails (>= 3.2)
method_source
rake (>= 0.8.7)
thor (>= 0.18.1, < 2.0)
- rake (12.3.2)
+ rake (13.0.1)
raphael-rails (2.1.2)
rb-fsevent (0.10.3)
rb-inotify (0.10.0)
# the browser can't.
f.json { render opts.merge(json: {success: false, errors: @errors}) }
f.html { render({action: 'error'}.merge(opts)) }
+ f.all { render({action: 'error', formats: 'text'}.merge(opts)) }
end
end
# Prefer the attachment-only-host when we want an attachment
# (and when there is no preview link configured)
tmpl = Rails.configuration.Services.WebDAVDownload.ExternalURL.to_s
- elsif not Rails.configuration.Workbench.TrustAllContent
+ elsif not Rails.configuration.Collections.TrustAllContent
check_uri = URI.parse(tmpl.sub("*", munged_id))
if opts[:query_token] and
(check_uri.host.nil? or (
end
def webshell
- return render_not_found if Rails.configuration.Workbench.ShellInABoxURL == URI("")
- webshell_url = URI(Rails.configuration.Workbench.ShellInABoxURL)
+ return render_not_found if Rails.configuration.Services.WebShell.ExternalURL == URI("")
+ webshell_url = URI(Rails.configuration.Services.WebShell.ExternalURL)
if webshell_url.host.index("*") != nil
webshell_url.host = webshell_url.host.sub("*", @object.hostname)
else
--- /dev/null
+<%# Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: AGPL-3.0 %>
+
+Oh... fiddlesticks.
+
+Sorry, I had some trouble handling your request.
+
+<% if @errors.is_a? Array then @errors.each do |error| %>
+<%= error %>
+<% end end %>
<td style="word-break:break-all;">
<% if @my_vm_logins[vm[:uuid]] %>
<% @my_vm_logins[vm[:uuid]].each do |login| %>
- <code>ssh <%= login %>@<%= vm[:hostname] %></code>
+ <code>ssh <%= login %>@<%= vm[:hostname] %><%=Rails.configuration.Workbench.SSHHelpHostSuffix%></code>
<% end %>
<% end %>
</td>
Bundler.require(:default, Rails.env)
+if ENV["ARVADOS_RAILS_LOG_TO_STDOUT"]
+ Rails.logger = ActiveSupport::TaggedLogging.new(Logger.new(STDOUT))
+end
+
module ArvadosWorkbench
class Application < Rails::Application
end
test "Redirect to keep_web_url via #{id_type} when trust_all_content enabled" do
- Rails.configuration.Workbench.TrustAllContent = true
+ Rails.configuration.Collections.TrustAllContent = true
setup_for_keep_web('https://collections.example',
'https://download.example')
tok = api_token('active')
[false, true].each do |trust_all_content|
test "Redirect preview to keep_web_download_url when preview is disabled and trust_all_content is #{trust_all_content}" do
- Rails.configuration.Workbench.TrustAllContent = trust_all_content
+ Rails.configuration.Collections.TrustAllContent = trust_all_content
setup_for_keep_web "", 'https://download.example/'
tok = api_token('active')
id = api_fixture('collections')['w_a_z_file']['uuid']
WORKSPACE=path Path to the Arvados source tree to build packages from
CWLTOOL=path (optional) Path to cwltool git repository.
SALAD=path (optional) Path to schema_salad git repository.
-PYCMD=pythonexec (optional) Specify the python executable to use in the docker image. Defaults to "python".
+PYCMD=pythonexec (optional) Specify the python executable to use in the docker image. Defaults to "python3".
EOF
cd "$WORKSPACE"
-py=python
+py=python3
pipcmd=pip
if [[ -n "$PYCMD" ]] ; then
py="$PYCMD"
- if [[ $py = python3 ]] ; then
- pipcmd=pip3
- fi
+fi
+if [[ $py = python3 ]] ; then
+ pipcmd=pip3
fi
(cd sdk/python && python setup.py sdist)
cwl_runner_version=$(cd sdk/python && nohash_version_from_git 1.0)
fi
-docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py --build-arg pipcmd=$pipcmd -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
+set -x
+docker build --no-cache --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py --build-arg pipcmd=$pipcmd -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
echo arv-keepdocker arvados/jobs $cwl_runner_version
arv-keepdocker arvados/jobs $cwl_runner_version
# The Python SDK - Should be built first because it's needed by others
fpm_build_virtualenv "arvados-python-client" "sdk/python"
-# Arvados cwl runner
-fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl"
+# The Python SDK - Python3 package
+fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3"
-# Arvados cwl runner - Python3 package
+# Arvados cwl runner - Only supports Python3 now
fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "python3"
# The PAM module
# The Arvados crunchstat-summary tool
fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary"
-# The Python SDK - Python3 package
-fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3"
-
# The Docker image cleaner
fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3"
sdk/go/asyncbuf
sdk/go/stats
sdk/go/crunchrunner
-sdk/cwl
+sdk/cwl:py3
sdk/R
sdk/java-v2
tools/sync-groups
checkhealth() {
svc="$1"
- base=$(python -c "import yaml; print list(yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['Services']['$1']['InternalURLs'].keys())[0]")
+ base=$("${VENVDIR}/bin/python" -c "import yaml; print list(yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['Services']['$1']['InternalURLs'].keys())[0]")
url="$base/_health/ping"
if ! curl -Ss -H "Authorization: Bearer e687950a23c3a9bceec28c6223a06c79" "${url}" | tee -a /dev/stderr | grep '"OK"'; then
echo "${url} failed"
dd="https://${1}/discovery/v1/apis/arvados/v1/rest"
if ! (set -o pipefail; curl -fsk "$dd" | grep -q ^{ ); then
echo >&2 "ERROR: could not retrieve discovery doc from RailsAPI at $dd"
- tail -v $WORKSPACE/services/api/log/test.log
+ tail -v $WORKSPACE/tmp/railsapi.log
return 1
fi
echo "${dd} ok"
export HOME=$GEMHOME
("$bundle" version | grep -q 2.0.2) \
|| gem install --user bundler -v 2.0.2
- "$bundle" version | grep 2.0.2
+ "$bundle" version | tee /dev/stderr | grep -q 'version 2'
) || fatal 'install bundler'
fi
}
. "$VENVDIR/bin/activate"
# Needed for run_test_server.py which is used by certain (non-Python) tests.
- pip install --no-cache-dir PyYAML future httplib2 \
- || fatal "`pip install PyYAML future httplib2` failed"
+ (
+ set -e
+ "${VENVDIR}/bin/pip" install PyYAML
+ "${VENV3DIR}/bin/pip" install PyYAML
+ cd "$WORKSPACE/sdk/python"
+ python setup.py install
+ ) || fatal "installing PyYAML and sdk/python failed"
# Preinstall libcloud if using a fork; otherwise nodemanager "pip
# install" won't pick it up by default.
cd "$WORKSPACE/$1" \
&& "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
&& cd "$WORKSPACE" \
- && "${3}pip" install --no-cache-dir --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
- && "${3}pip" install --no-cache-dir --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
+ && "${3}pip" install --no-cache-dir "$WORKSPACE/$1/dist"/*.tar.gz \
+ && "${3}pip" install --no-cache-dir --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
elif [[ "$2" != "" ]]
then
"install_$2"
echo "(Running bundle install again, without --local.)"
"$bundle" install --no-deployment
fi
- "$bundle" package --all
+ "$bundle" package
)
}
install_services/api() {
stop_services
+ check_arvados_config "services/api"
cd "$WORKSPACE/services/api" \
&& RAILS_ENV=test bundle_install_trylocal \
|| return 1
# database, so that we can drop it. This assumes the current user
# is a postgresql superuser.
cd "$WORKSPACE/services/api" \
- && test_database=$(python -c "import yaml; print yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname']") \
+ && test_database=$("${VENVDIR}/bin/python" -c "import yaml; print yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname']") \
&& psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
mkdir -p "$WORKSPACE/services/api/tmp/pids"
&& git --git-dir internal.git init \
|| return 1
-
- (cd "$WORKSPACE/services/api"
- export RAILS_ENV=test
- if "$bundle" exec rails db:environment:set ; then
- "$bundle" exec rake db:drop
- fi
- "$bundle" exec rake db:setup \
- && "$bundle" exec rake db:fixtures:load
- )
+ (
+ set -e
+ cd "$WORKSPACE/services/api"
+ export RAILS_ENV=test
+ if "$bundle" exec rails db:environment:set ; then
+ "$bundle" exec rake db:drop
+ fi
+ "$bundle" exec rake db:setup
+ "$bundle" exec rake db:fixtures:load
+ ) || return 1
}
declare -a pythonstuff
sdk/pam
sdk/python
sdk/python:py3
- sdk/cwl
sdk/cwl:py3
services/dockercleaner:py3
services/fuse
test_apps/workbench_integration() {
local TASK="test:integration"
cd "$WORKSPACE/apps/workbench" \
- && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]}
+ && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]}
}
test_apps/workbench_benchmark() {
do_install sdk/cli
do_install sdk/perl
do_install sdk/python pip
- do_install sdk/python pip3
+ do_install sdk/python pip "${VENV3DIR}/bin/"
do_install sdk/ruby
do_install services/api
do_install services/arv-git-httpd go
${verb}_${target}
;;
*)
- testargs["$target"]="${opts}"
+ argstarget=${target%:py3}
+ testargs["$argstarget"]="${opts}"
tt="${testfuncargs[${target}]}"
tt="${tt:-$target}"
do_$verb $tt
import (
"os"
+ "git.arvados.org/arvados.git/lib/boot"
"git.arvados.org/arvados.git/lib/cloud/cloudtest"
"git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/lib/config"
"-version": cmd.Version,
"--version": cmd.Version,
+ "boot": boot.Command,
"cloudtest": cloudtest.Command,
"config-check": config.CheckCommand,
"config-dump": config.DumpCommand,
- admin/merge-remote-account.html.textile.liquid
- admin/migrating-providers.html.textile.liquid
- user/topics/arvados-sync-groups.html.textile.liquid
+ - admin/scoped-tokens.html.textile.liquid
- Monitoring:
- admin/logging.html.textile.liquid
- admin/metrics.html.textile.liquid
--- /dev/null
+---
+layout: default
+navsection: admin
+title: Securing API access with scoped tokens
+...
+
+By default, Arvados API tokens grant unlimited access to a user account, and admin account tokens have unlimited access to the whole system. If you want to grant restricted access to a user account, you can create a "scoped token" which is an Arvados API token which is limited to accessing specific APIs.
+
+One use of token scopes is to grant access to data, such as a collection, to users who do not have an Arvados accounts on your cluster. This is done by creating scoped token that only allows getting a specific record. An example of this is "creating a collection sharing link.":{{site.baseurl}}/sdk/python/cookbook.html#sharing_link
+
+Another example is situations where admin access is required but there is risk of the token being compromised. Setting a scope prevents the token from being used for any action other than the specific action the token is intended for. For example, "synchronizing user accounts on a shell node.":{{site.baseurl}}/install/install-shell-server.html#scoped-token
+
+h2. Defining scopes
+
+A "scope" consists of a HTTP method and API path. A token can have multiple scopes. Token scopes act as a whitelist, and the API server checks the HTTP method and the API path of every request against the scopes of the request token. Scopes are also described on the "API Authorization":{{site.baseurl}}/api/tokens.html#scopes page of the "API documentation":{{site.baseurl}}/api .
+
+These examples use @/arvados/v1/collections@, but can be applied to any endpoint. Consult the "API documentation":{{site.baseurl}}/api to determine the endpoints for specific methods.
+
+The scope @["GET", "/arvados/v1/collections"]@ will allow only GET or HEAD requests for the list of collections. Any other HTTP method or path (including requests for a specific collection record, eg a request with path @/arvados/v1/collections/zzzzz-4zz18-0123456789abcde@) will return a permission error.
+
+A trailing slash in a scope is signficant. The scope @["GET", "/arvados/v1/collections/"]@ will allow only GET or HEAD requests *starting with* @/arvados/v1/collections/@. A request for an individual record path @/arvados/v1/collections/zzzzz-4zz18-0123456789abcde@) is allowed but a request to list collections (@/arvados/v1/collections@) will be denied because it does not end with @/@ (API requests with a trailing @/@ will have the slash stripped before the scope is checked.)
+
+The scope can include an object uuid. The scope @["GET", "/arvados/v1/collections/zzzzz-4zz18-0123456789abcde"]@ only permits requests to read the record @zzzzz-4zz18-0123456789abcde@.
+
+Since a token can have multiple scopes, use @[["GET", "/arvados/v1/collections"], ["GET", "/arvados/v1/collections/"]]@ to allow both listing collections and fetching individual collection records. This will reject requests to create or change collections, or access any other API method.
+
+Object create calls use the @POST@ method. A scope of @["POST", "/arvados/v1/collections"]@ will allow creating collections, but not reading, listing or updating them (or accessing anything else).
+
+Object update calls use the @PATCH@ method. A scope of @["PATCH", "/arvados/v1/collections/"]@ will allow updating collections, but not listing or creating them. (Note: while GET requests are denied an object can be read indirectly by using an empty PATCH which will return the unmodified object as the result).
+
+Similarly, you can use a scope of @["PATCH", "/arvados/v1/collections/zzzzz-4zz18-0123456789abcde"]@ to restrict updates to a single collection.
+
+h2. Creating a scoped token
+
+A scoped token can be created at the command line:
+
+<pre>
+$ arv api_client_authorization create --api-client-authorization '{"scopes": [["GET", "/arvados/v1/collections"], ["GET", "/arvados/v1/collections/"]]}'
+{
+ "href":"/api_client_authorizations/x1u39-gj3su-bizbsw0mx5pju3w",
+ "kind":"arvados#apiClientAuthorization",
+ "etag":"9yk144t0v6cvyp0342exoh2vq",
+ "uuid":"x1u39-gj3su-bizbsw0mx5pju3w",
+ "owner_uuid":"x1u39-tpzed-fr97h9t4m5jffxs",
+ "created_at":"2020-03-12T20:36:12.517375422Z",
+ "modified_by_client_uuid":null,
+ "modified_by_user_uuid":null,
+ "modified_at":null,
+ "user_id":3,
+ "api_client_id":7,
+ "api_token":"5a74htnoqwkhtfo2upekpfbsg04hv7cy5v4nowf7dtpxer086m",
+ "created_by_ip_address":null,
+ "default_owner_uuid":null,
+ "expires_at":null,
+ "last_used_at":null,
+ "last_used_by_ip_address":null,
+ "scopes":[
+ [
+ "GET",
+ "/arvados/v1/collections"
+ ],
+ [
+ "GET",
+ "/arvados/v1/collections/"
+ ]
+ ]
+}
+</pre>
+
+The response will include @api_token@ field which is the newly issued secret token. It can be passed directly to the API server that issued it, or can be used to construct a @v2@ token. A @v2@ format token is required if the token will be used to access other clusters in an Arvados federation. An Arvados @v2@ format token consists of three fields separate by slashes: the prefix @v2@, followed by the token uuid, followed by the token secret. For example: @v2/x1u39-gj3su-bizbsw0mx5pju3w/5a74htnoqwkhtfo2upekpfbsg04hv7cy5v4nowf7dtpxer086m@.
The API server publishes a machine-readable description of its endpoints and some additional site configuration values via a JSON-formatted discovery document. This is available at @/discovery/v1/apis/arvados/v1/rest@, for example @https://{{ site.arvados_api_host }}/discovery/v1/apis/arvados/v1/rest@. Some Arvados SDKs use the discovery document to generate language bindings.
+h2. Exported configuration
+
+The Controller exposes a subset of the cluster's configuration and makes it available to clients in JSON format. This public config includes valuable information like several service's URLs, timeout settings, etc. and it is available at @/arvados/v1/config@, for example @https://{{ site.arvados_api_host }}/arvados/v1/config@. The new Workbench is one example of a client using this information, as it's a client-side application and doesn't have access to the cluster's config file.
+
h2. Workbench examples
Many Arvados Workbench pages, under the the *Advanced* tab, provide examples of API and SDK use for accessing the current resource .
Scopes can restrict a token so it may only access certain resources. This is in addition to normal permission checks for the user associated with the token.
-Each entry in scopes consists of a @request_method@ and @request_path@, where the @request_method@ is a HTTP method (one of @GET@, @POST@, @PUT@ or @DELETE@) and @request_path@ is the request URI. A given request is permitted if it matches a scopes exactly, or the scope ends with @/@ and the request string is a prefix of the scope.
+Each entry in scopes consists of a @request_method@ and @request_path@. The @request_method@ is a HTTP method (one of @GET@, @POST@, @PATCH@ or @DELETE@) and @request_path@ is the request URI. A given request is permitted if it matches a scopes exactly, or the scope ends with @/@ and the request string is a prefix of the scope.
-As a special case, a scope of ["all"] allows all resources.
+As a special case, a scope of @["all"]@ allows all resources. This is the default if no scope is given.
+
+Using scopes is also described on the "Securing API access with scoped tokens":{{site.baseurl}}/admin/scoped-tokens.html page of the admin documentation.
h3. Scope examples
--- /dev/null
+Clusters:
+ zzzzz:
+ ManagementToken: e687950a23c3a9bceec28c6223a06c79
+ SystemRootToken: systemusertesttoken1234567890aoeuidhtnsqjkxbmwvzpy
+ API:
+ RequestTimeout: 30s
+ TLS:
+ Insecure: true
+ Collections:
+ BlobSigningKey: zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc
+ TrustAllContent: true
+ ForwardSlashNameSubstitution: /
h2(#scoped-token). Create scoped token
-As an admin arvados user (such as the system root user), create a token that is restricted to only reading login information for this VM.
+As an Arvados admin user (such as the system root user), create a "scoped token":{{site.baseurl}}/admin/scoped-tokens.html that is permits only reading login information for this VM. Setting a scope on the token means that even though a user with root access on the shell node can access the token, the token is not usable for admin actions on Arvados.
<notextile>
<pre>
print(collection.open(c).read())
{% endcodeblock %}
-h2. Create a collection sharing link
+h2(#sharing_link). Create a collection sharing link
{% codeblock as python %}
import arvados
github.com/Microsoft/go-winio v0.4.5 // indirect
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 // indirect
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
- github.com/arvados/cgofuse v1.2.0
+ github.com/arvados/cgofuse v1.2.0-arvados1
github.com/aws/aws-sdk-go v1.25.30
github.com/coreos/go-oidc v2.1.0+incompatible
github.com/coreos/go-systemd v0.0.0-20180108085132-cc4f39464dc7
github.com/julienschmidt/httprouter v1.2.0
github.com/karalabe/xgo v0.0.0-20191115072854-c5ccff8648a7 // indirect
github.com/kevinburke/ssh_config v0.0.0-20171013211458-802051befeb5 // indirect
- github.com/lib/pq v0.0.0-20171126050459-83612a56d3dd
+ github.com/lib/pq v1.3.0
github.com/marstr/guid v1.1.1-0.20170427235115-8bdf7d1a087c // indirect
github.com/mitchellh/go-homedir v0.0.0-20161203194507-b8bc1bf76747 // indirect
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
github.com/src-d/gcfg v1.3.0 // indirect
github.com/stretchr/testify v1.4.0 // indirect
github.com/xanzy/ssh-agent v0.1.0 // indirect
- golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
- golang.org/x/net v0.0.0-20190613194153-d28f0bde5980
+ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550
+ golang.org/x/net v0.0.0-20190620200207-3b0461eec859
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd // indirect
google.golang.org/api v0.13.0
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
-github.com/arvados/cgofuse v1.2.0 h1:sWgVxyvSFjH965Uc7ReScn/cBl9Jemc9SeUNlEmjRH4=
-github.com/arvados/cgofuse v1.2.0/go.mod h1:79WFV98hrkRHK9XPhh2IGGOwpFSjocsWubgxAs2KhRc=
+github.com/arvados/cgofuse v1.2.0-arvados1 h1:4Q4vRJ4hbTCcI4gGEaa6hqwj3rqlUuzeFQkfoEA2HqE=
+github.com/arvados/cgofuse v1.2.0-arvados1/go.mod h1:79WFV98hrkRHK9XPhh2IGGOwpFSjocsWubgxAs2KhRc=
github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef h1:cl7DIRbiAYNqaVxg3CZY8qfZoBOKrj06H/x9SPGaxas=
github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef/go.mod h1:rCtgyMmBGEbjTm37fCuBYbNL0IhztiALzo3OB9HyiOM=
github.com/aws/aws-sdk-go v1.25.30 h1:I9qj6zW3mMfsg91e+GMSN/INcaX9tTFvr/l/BAHKaIY=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/lib/pq v0.0.0-20171126050459-83612a56d3dd h1:2RDaVc4/izhWyAvYxNm8c9saSyCDIxefNwOcqaH7pcU=
-github.com/lib/pq v0.0.0-20171126050459-83612a56d3dd/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
+github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/marstr/guid v1.1.1-0.20170427235115-8bdf7d1a087c h1:ouxemItv3B/Zh008HJkEXDYCN3BIRyNHxtUN7ThJ5Js=
github.com/marstr/guid v1.1.1-0.20170427235115-8bdf7d1a087c/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c h1:97SnQk1GYRXJgvwZ8fadnxDOWfKvkNQHH3CtZntPSrM=
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.13.0 h1:Q3Ui3V3/CVinFWFiW39Iw0kMuVrRzYX0wN6OPFp0lTA=
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package boot
+
+import (
+ "context"
+ "io/ioutil"
+ "path/filepath"
+)
+
+// Create a root CA key and use it to make a new server
+// certificate+key pair.
+//
+// In future we'll make one root CA key per host instead of one per
+// cluster, so it only needs to be imported to a browser once for
+// ongoing dev/test usage.
+type createCertificates struct{}
+
+func (createCertificates) String() string {
+ return "certificates"
+}
+
+func (createCertificates) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ // Generate root key
+ err := super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "genrsa", "-out", "rootCA.key", "4096")
+ if err != nil {
+ return err
+ }
+ // Generate a self-signed root certificate
+ err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "req", "-x509", "-new", "-nodes", "-key", "rootCA.key", "-sha256", "-days", "3650", "-out", "rootCA.crt", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost")
+ if err != nil {
+ return err
+ }
+ // Generate server key
+ err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "genrsa", "-out", "server.key", "2048")
+ if err != nil {
+ return err
+ }
+ // Build config file for signing request
+ defaultconf, err := ioutil.ReadFile("/etc/ssl/openssl.cnf")
+ if err != nil {
+ return err
+ }
+ err = ioutil.WriteFile(filepath.Join(super.tempdir, "server.cfg"), append(defaultconf, []byte(`
+[SAN]
+subjectAltName=DNS:localhost,DNS:localhost.localdomain
+`)...), 0644)
+ if err != nil {
+ return err
+ }
+ // Generate signing request
+ err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "req", "-new", "-sha256", "-key", "server.key", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost", "-reqexts", "SAN", "-config", "server.cfg", "-out", "server.csr")
+ if err != nil {
+ return err
+ }
+ // Sign certificate
+ err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "x509", "-req", "-in", "server.csr", "-CA", "rootCA.crt", "-CAkey", "rootCA.key", "-CAcreateserial", "-out", "server.crt", "-days", "3650", "-sha256")
+ if err != nil {
+ return err
+ }
+ return nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package boot
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "io"
+
+ "git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+)
+
+var Command cmd.Handler = bootCommand{}
+
+type supervisedTask interface {
+ // Execute the task. Run should return nil when the task is
+ // done enough to satisfy a dependency relationship (e.g., the
+ // service is running and ready). If the task starts a
+ // goroutine that fails after Run returns (e.g., the service
+ // shuts down), it should call fail().
+ Run(ctx context.Context, fail func(error), super *Supervisor) error
+ String() string
+}
+
+type bootCommand struct{}
+
+func (bootCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ super := &Supervisor{
+ Stderr: stderr,
+ logger: ctxlog.New(stderr, "json", "info"),
+ }
+
+ ctx := ctxlog.Context(context.Background(), super.logger)
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
+ var err error
+ defer func() {
+ if err != nil {
+ super.logger.WithError(err).Info("exiting")
+ }
+ }()
+
+ flags := flag.NewFlagSet(prog, flag.ContinueOnError)
+ flags.SetOutput(stderr)
+ loader := config.NewLoader(stdin, super.logger)
+ loader.SetupFlags(flags)
+ versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
+ flags.StringVar(&super.SourcePath, "source", ".", "arvados source tree `directory`")
+ flags.StringVar(&super.ClusterType, "type", "production", "cluster `type`: development, test, or production")
+ flags.StringVar(&super.ListenHost, "listen-host", "localhost", "host name or interface address for service listeners")
+ flags.StringVar(&super.ControllerAddr, "controller-address", ":0", "desired controller address, `host:port` or `:port`")
+ flags.BoolVar(&super.OwnTemporaryDatabase, "own-temporary-database", false, "bring up a postgres server and create a temporary database")
+ err = flags.Parse(args)
+ if err == flag.ErrHelp {
+ err = nil
+ return 0
+ } else if err != nil {
+ return 2
+ } else if *versionFlag {
+ return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+ } else if super.ClusterType != "development" && super.ClusterType != "test" && super.ClusterType != "production" {
+ err = fmt.Errorf("cluster type must be 'development', 'test', or 'production'")
+ return 2
+ }
+
+ loader.SkipAPICalls = true
+ cfg, err := loader.Load()
+ if err != nil {
+ return 1
+ }
+
+ super.Start(ctx, cfg)
+ defer super.Stop()
+ url, ok := super.WaitReady()
+ if !ok {
+ return 1
+ }
+ // Write controller URL to stdout. Nothing else goes to
+ // stdout, so this provides an easy way for a calling script
+ // to discover the controller URL when everything is ready.
+ fmt.Fprintln(stdout, url)
+ // Wait for signal/crash + orderly shutdown
+ <-super.done
+ return 0
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package boot
+
+import (
+ "context"
+ "fmt"
+ "io/ioutil"
+ "net"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+// Run an Nginx process that proxies the supervisor's configured
+// ExternalURLs to the appropriate InternalURLs.
+type runNginx struct{}
+
+func (runNginx) String() string {
+ return "nginx"
+}
+
+func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ vars := map[string]string{
+ "LISTENHOST": super.ListenHost,
+ "SSLCERT": filepath.Join(super.SourcePath, "services", "api", "tmp", "self-signed.pem"), // TODO: root ca
+ "SSLKEY": filepath.Join(super.SourcePath, "services", "api", "tmp", "self-signed.key"), // TODO: root ca
+ "ACCESSLOG": filepath.Join(super.tempdir, "nginx_access.log"),
+ "ERRORLOG": filepath.Join(super.tempdir, "nginx_error.log"),
+ "TMPDIR": super.tempdir,
+ }
+ var err error
+ for _, cmpt := range []struct {
+ varname string
+ svc arvados.Service
+ }{
+ {"CONTROLLER", super.cluster.Services.Controller},
+ {"KEEPWEB", super.cluster.Services.WebDAV},
+ {"KEEPWEBDL", super.cluster.Services.WebDAVDownload},
+ {"KEEPPROXY", super.cluster.Services.Keepproxy},
+ {"GIT", super.cluster.Services.GitHTTP},
+ {"WORKBENCH1", super.cluster.Services.Workbench1},
+ {"WS", super.cluster.Services.Websocket},
+ } {
+ port, err := internalPort(cmpt.svc)
+ if err != nil {
+ return fmt.Errorf("%s internal port: %s (%v)", cmpt.varname, err, cmpt.svc)
+ }
+ if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil {
+ return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err)
+ }
+ vars[cmpt.varname+"PORT"] = port
+
+ port, err = externalPort(cmpt.svc)
+ if err != nil {
+ return fmt.Errorf("%s external port: %s (%v)", cmpt.varname, err, cmpt.svc)
+ }
+ if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil {
+ return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err)
+ }
+ vars[cmpt.varname+"SSLPORT"] = port
+ }
+ tmpl, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "sdk", "python", "tests", "nginx.conf"))
+ if err != nil {
+ return err
+ }
+ conf := regexp.MustCompile(`{{.*?}}`).ReplaceAllStringFunc(string(tmpl), func(src string) string {
+ if len(src) < 4 {
+ return src
+ }
+ return vars[src[2:len(src)-2]]
+ })
+ conffile := filepath.Join(super.tempdir, "nginx.conf")
+ err = ioutil.WriteFile(conffile, []byte(conf), 0755)
+ if err != nil {
+ return err
+ }
+ nginx := "nginx"
+ if _, err := exec.LookPath(nginx); err != nil {
+ for _, dir := range []string{"/sbin", "/usr/sbin", "/usr/local/sbin"} {
+ if _, err = os.Stat(dir + "/nginx"); err == nil {
+ nginx = dir + "/nginx"
+ break
+ }
+ }
+ }
+ super.waitShutdown.Add(1)
+ go func() {
+ defer super.waitShutdown.Done()
+ fail(super.RunProgram(ctx, ".", nil, nil, nginx,
+ "-g", "error_log stderr info;",
+ "-g", "pid "+filepath.Join(super.tempdir, "nginx.pid")+";",
+ "-c", conffile))
+ }()
+ return waitForConnect(ctx, super.cluster.Services.Controller.ExternalURL.Host)
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package boot
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+// Don't trust "passenger-config" (or "bundle install") to handle
+// concurrent installs.
+var passengerInstallMutex sync.Mutex
+
+var railsEnv = []string{
+ "ARVADOS_RAILS_LOG_TO_STDOUT=1",
+ "ARVADOS_CONFIG_NOLEGACY=1", // don't load database.yml from source tree
+}
+
+// Install a Rails application's dependencies, including phusion
+// passenger.
+type installPassenger struct {
+ src string
+ depends []supervisedTask
+}
+
+func (runner installPassenger) String() string {
+ return "installPassenger:" + runner.src
+}
+
+func (runner installPassenger) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ err := super.wait(ctx, runner.depends...)
+ if err != nil {
+ return err
+ }
+
+ passengerInstallMutex.Lock()
+ defer passengerInstallMutex.Unlock()
+
+ var buf bytes.Buffer
+ err = super.RunProgram(ctx, runner.src, &buf, nil, "gem", "list", "--details", "bundler")
+ if err != nil {
+ return err
+ }
+ for _, version := range []string{"1.11.0", "1.17.3", "2.0.2"} {
+ if !strings.Contains(buf.String(), "("+version+")") {
+ err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "bundler:1.11", "bundler:1.17.3", "bundler:2.0.2")
+ if err != nil {
+ return err
+ }
+ break
+ }
+ }
+ err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "install", "--jobs", "4", "--path", filepath.Join(os.Getenv("HOME"), ".gem"))
+ if err != nil {
+ return err
+ }
+ err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger-config", "build-native-support")
+ if err != nil {
+ return err
+ }
+ err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger-config", "install-standalone-runtime")
+ if err != nil {
+ return err
+ }
+ err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger-config", "validate-install")
+ if err != nil && !strings.Contains(err.Error(), "exit status 2") {
+ // Exit code 2 indicates there were warnings (like
+ // "other passenger installations have been detected",
+ // which we can't expect to avoid) but no errors.
+ // Other non-zero exit codes (1, 9) indicate errors.
+ return err
+ }
+ return nil
+}
+
+type runPassenger struct {
+ src string
+ svc arvados.Service
+ depends []supervisedTask
+}
+
+func (runner runPassenger) String() string {
+ return "runPassenger:" + runner.src
+}
+
+func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ err := super.wait(ctx, runner.depends...)
+ if err != nil {
+ return err
+ }
+ port, err := internalPort(runner.svc)
+ if err != nil {
+ return fmt.Errorf("bug: no internalPort for %q: %v (%#v)", runner, err, runner.svc)
+ }
+ loglevel := "4"
+ if lvl, ok := map[string]string{
+ "debug": "5",
+ "info": "4",
+ "warn": "2",
+ "warning": "2",
+ "error": "1",
+ "fatal": "0",
+ "panic": "0",
+ }[super.cluster.SystemLogs.LogLevel]; ok {
+ loglevel = lvl
+ }
+ super.waitShutdown.Add(1)
+ go func() {
+ defer super.waitShutdown.Done()
+ err = super.RunProgram(ctx, runner.src, nil, railsEnv, "bundle", "exec",
+ "passenger", "start",
+ "-p", port,
+ "--log-file", "/dev/stderr",
+ "--log-level", loglevel,
+ "--no-friendly-error-pages",
+ "--pid-file", filepath.Join(super.tempdir, "passenger."+strings.Replace(runner.src, "/", "_", -1)+".pid"))
+ fail(err)
+ }()
+ return nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package boot
+
+import (
+ "bytes"
+ "context"
+ "database/sql"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "github.com/lib/pq"
+)
+
+// Run a postgresql server in a private data directory. Set up a db
+// user, database, and TCP listener that match the supervisor's
+// configured database connection info.
+type runPostgreSQL struct{}
+
+func (runPostgreSQL) String() string {
+ return "postgresql"
+}
+
+func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ err := super.wait(ctx, createCertificates{})
+ if err != nil {
+ return err
+ }
+
+ buf := bytes.NewBuffer(nil)
+ err = super.RunProgram(ctx, super.tempdir, buf, nil, "pg_config", "--bindir")
+ if err != nil {
+ return err
+ }
+ bindir := strings.TrimSpace(buf.String())
+
+ datadir := filepath.Join(super.tempdir, "pgdata")
+ err = os.Mkdir(datadir, 0755)
+ if err != nil {
+ return err
+ }
+ err = super.RunProgram(ctx, super.tempdir, nil, nil, filepath.Join(bindir, "initdb"), "-D", datadir)
+ if err != nil {
+ return err
+ }
+
+ err = super.RunProgram(ctx, super.tempdir, nil, nil, "cp", "server.crt", "server.key", datadir)
+ if err != nil {
+ return err
+ }
+
+ port := super.cluster.PostgreSQL.Connection["port"]
+
+ super.waitShutdown.Add(1)
+ go func() {
+ defer super.waitShutdown.Done()
+ fail(super.RunProgram(ctx, super.tempdir, nil, nil, filepath.Join(bindir, "postgres"),
+ "-l", // enable ssl
+ "-D", datadir, // data dir
+ "-k", datadir, // socket dir
+ "-p", super.cluster.PostgreSQL.Connection["port"],
+ ))
+ }()
+
+ for {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ if exec.CommandContext(ctx, "pg_isready", "--timeout=10", "--host="+super.cluster.PostgreSQL.Connection["host"], "--port="+port).Run() == nil {
+ break
+ }
+ time.Sleep(time.Second / 2)
+ }
+ db, err := sql.Open("postgres", arvados.PostgreSQLConnection{
+ "host": datadir,
+ "port": port,
+ "dbname": "postgres",
+ }.String())
+ if err != nil {
+ return fmt.Errorf("db open failed: %s", err)
+ }
+ defer db.Close()
+ conn, err := db.Conn(ctx)
+ if err != nil {
+ return fmt.Errorf("db conn failed: %s", err)
+ }
+ defer conn.Close()
+ _, err = conn.ExecContext(ctx, `CREATE USER `+pq.QuoteIdentifier(super.cluster.PostgreSQL.Connection["user"])+` WITH SUPERUSER ENCRYPTED PASSWORD `+pq.QuoteLiteral(super.cluster.PostgreSQL.Connection["password"]))
+ if err != nil {
+ return fmt.Errorf("createuser failed: %s", err)
+ }
+ _, err = conn.ExecContext(ctx, `CREATE DATABASE `+pq.QuoteIdentifier(super.cluster.PostgreSQL.Connection["dbname"]))
+ if err != nil {
+ return fmt.Errorf("createdb failed: %s", err)
+ }
+ return nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package boot
+
+import (
+ "context"
+)
+
+// Populate a blank database with arvados tables and seed rows.
+type seedDatabase struct{}
+
+func (seedDatabase) String() string {
+ return "seedDatabase"
+}
+
+func (seedDatabase) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ err := super.wait(ctx, runPostgreSQL{}, installPassenger{src: "services/api"})
+ if err != nil {
+ return err
+ }
+ err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "rake", "db:setup")
+ if err != nil {
+ return err
+ }
+ return nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package boot
+
+import (
+ "context"
+ "errors"
+ "path/filepath"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+// Run a service using the arvados-server binary.
+//
+// In future this will bring up the service in the current process,
+// but for now (at least until the subcommand handlers get a shutdown
+// mechanism) it starts a child process using the arvados-server
+// binary, which the supervisor is assumed to have installed in
+// {super.tempdir}/bin/.
+type runServiceCommand struct {
+ name string // arvados-server subcommand, e.g., "controller"
+ svc arvados.Service // cluster.Services.* entry with the desired InternalURLs
+ depends []supervisedTask // wait for these tasks before starting
+}
+
+func (runner runServiceCommand) String() string {
+ return runner.name
+}
+
+func (runner runServiceCommand) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ binfile := filepath.Join(super.tempdir, "bin", "arvados-server")
+ err := super.RunProgram(ctx, super.tempdir, nil, nil, binfile, "-version")
+ if err != nil {
+ return err
+ }
+ super.wait(ctx, runner.depends...)
+ for u := range runner.svc.InternalURLs {
+ u := u
+ if islocal, err := addrIsLocal(u.Host); err != nil {
+ return err
+ } else if !islocal {
+ continue
+ }
+ super.waitShutdown.Add(1)
+ go func() {
+ defer super.waitShutdown.Done()
+ fail(super.RunProgram(ctx, super.tempdir, nil, []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}, binfile, runner.name, "-config", super.configfile))
+ }()
+ }
+ return nil
+}
+
+// Run a Go service that isn't bundled in arvados-server.
+type runGoProgram struct {
+ src string // source dir, e.g., "services/keepproxy"
+ svc arvados.Service // cluster.Services.* entry with the desired InternalURLs
+ depends []supervisedTask // wait for these tasks before starting
+}
+
+func (runner runGoProgram) String() string {
+ _, basename := filepath.Split(runner.src)
+ return basename
+}
+
+func (runner runGoProgram) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ if len(runner.svc.InternalURLs) == 0 {
+ return errors.New("bug: runGoProgram needs non-empty svc.InternalURLs")
+ }
+
+ binfile, err := super.installGoProgram(ctx, runner.src)
+ if err != nil {
+ return err
+ }
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+
+ err = super.RunProgram(ctx, super.tempdir, nil, nil, binfile, "-version")
+ if err != nil {
+ return err
+ }
+
+ super.wait(ctx, runner.depends...)
+ for u := range runner.svc.InternalURLs {
+ u := u
+ if islocal, err := addrIsLocal(u.Host); err != nil {
+ return err
+ } else if !islocal {
+ continue
+ }
+ super.waitShutdown.Add(1)
+ go func() {
+ defer super.waitShutdown.Done()
+ fail(super.RunProgram(ctx, super.tempdir, nil, []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}, binfile))
+ }()
+ }
+ return nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package boot
+
+import (
+ "bytes"
+ "context"
+ "crypto/rand"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net"
+ "os"
+ "os/exec"
+ "os/signal"
+ "os/user"
+ "path/filepath"
+ "strings"
+ "sync"
+ "syscall"
+ "time"
+
+ "git.arvados.org/arvados.git/lib/service"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/health"
+ "github.com/sirupsen/logrus"
+)
+
+type Supervisor struct {
+ SourcePath string // e.g., /home/username/src/arvados
+ SourceVersion string // e.g., acbd1324...
+ ClusterType string // e.g., production
+ ListenHost string // e.g., localhost
+ ControllerAddr string // e.g., 127.0.0.1:8000
+ OwnTemporaryDatabase bool
+ Stderr io.Writer
+
+ logger logrus.FieldLogger
+ cluster *arvados.Cluster
+
+ ctx context.Context
+ cancel context.CancelFunc
+ done chan struct{}
+ healthChecker *health.Aggregator
+ tasksReady map[string]chan bool
+ waitShutdown sync.WaitGroup
+
+ tempdir string
+ configfile string
+ environ []string // for child processes
+}
+
+func (super *Supervisor) Start(ctx context.Context, cfg *arvados.Config) {
+ super.ctx, super.cancel = context.WithCancel(ctx)
+ super.done = make(chan struct{})
+
+ go func() {
+ sigch := make(chan os.Signal)
+ signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM)
+ defer signal.Stop(sigch)
+ go func() {
+ for sig := range sigch {
+ super.logger.WithField("signal", sig).Info("caught signal")
+ super.cancel()
+ }
+ }()
+
+ err := super.run(cfg)
+ if err != nil {
+ super.logger.WithError(err).Warn("supervisor shut down")
+ }
+ close(super.done)
+ }()
+}
+
+func (super *Supervisor) run(cfg *arvados.Config) error {
+ cwd, err := os.Getwd()
+ if err != nil {
+ return err
+ }
+ if !strings.HasPrefix(super.SourcePath, "/") {
+ super.SourcePath = filepath.Join(cwd, super.SourcePath)
+ }
+ super.SourcePath, err = filepath.EvalSymlinks(super.SourcePath)
+ if err != nil {
+ return err
+ }
+
+ super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(super.tempdir)
+ if err := os.Mkdir(filepath.Join(super.tempdir, "bin"), 0755); err != nil {
+ return err
+ }
+
+ // Fill in any missing config keys, and write the resulting
+ // config in the temp dir for child services to use.
+ err = super.autofillConfig(cfg)
+ if err != nil {
+ return err
+ }
+ conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
+ if err != nil {
+ return err
+ }
+ defer conffile.Close()
+ err = json.NewEncoder(conffile).Encode(cfg)
+ if err != nil {
+ return err
+ }
+ err = conffile.Close()
+ if err != nil {
+ return err
+ }
+ super.configfile = conffile.Name()
+
+ super.environ = os.Environ()
+ super.cleanEnv([]string{"ARVADOS_"})
+ super.setEnv("ARVADOS_CONFIG", super.configfile)
+ super.setEnv("RAILS_ENV", super.ClusterType)
+ super.setEnv("TMPDIR", super.tempdir)
+ super.prependEnv("PATH", filepath.Join(super.tempdir, "bin")+":")
+
+ super.cluster, err = cfg.GetCluster("")
+ if err != nil {
+ return err
+ }
+ // Now that we have the config, replace the bootstrap logger
+ // with a new one according to the logging config.
+ loglevel := super.cluster.SystemLogs.LogLevel
+ if s := os.Getenv("ARVADOS_DEBUG"); s != "" && s != "0" {
+ loglevel = "debug"
+ }
+ super.logger = ctxlog.New(super.Stderr, super.cluster.SystemLogs.Format, loglevel).WithFields(logrus.Fields{
+ "PID": os.Getpid(),
+ })
+
+ if super.SourceVersion == "" {
+ // Find current source tree version.
+ var buf bytes.Buffer
+ err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat")
+ if err != nil {
+ return err
+ }
+ dirty := buf.Len() > 0
+ buf.Reset()
+ err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "log", "-n1", "--format=%H")
+ if err != nil {
+ return err
+ }
+ super.SourceVersion = strings.TrimSpace(buf.String())
+ if dirty {
+ super.SourceVersion += "+uncommitted"
+ }
+ } else {
+ return errors.New("specifying a version to run is not yet supported")
+ }
+
+ _, err = super.installGoProgram(super.ctx, "cmd/arvados-server")
+ if err != nil {
+ return err
+ }
+ err = super.setupRubyEnv()
+ if err != nil {
+ return err
+ }
+
+ tasks := []supervisedTask{
+ createCertificates{},
+ runPostgreSQL{},
+ runNginx{},
+ runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{runPostgreSQL{}}},
+ runGoProgram{src: "services/arv-git-httpd", svc: super.cluster.Services.GitHTTP},
+ runGoProgram{src: "services/health", svc: super.cluster.Services.Health},
+ runGoProgram{src: "services/keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
+ runGoProgram{src: "services/keepstore", svc: super.cluster.Services.Keepstore},
+ runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
+ runGoProgram{src: "services/ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}},
+ installPassenger{src: "services/api"},
+ runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
+ installPassenger{src: "apps/workbench", depends: []supervisedTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup
+ runPassenger{src: "apps/workbench", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
+ seedDatabase{},
+ }
+ if super.ClusterType != "test" {
+ tasks = append(tasks,
+ runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller},
+ runGoProgram{src: "services/keep-balance"},
+ )
+ }
+ super.tasksReady = map[string]chan bool{}
+ for _, task := range tasks {
+ super.tasksReady[task.String()] = make(chan bool)
+ }
+ for _, task := range tasks {
+ task := task
+ fail := func(err error) {
+ if super.ctx.Err() != nil {
+ return
+ }
+ super.cancel()
+ super.logger.WithField("task", task.String()).WithError(err).Error("task failed")
+ }
+ go func() {
+ super.logger.WithField("task", task.String()).Info("starting")
+ err := task.Run(super.ctx, fail, super)
+ if err != nil {
+ fail(err)
+ return
+ }
+ close(super.tasksReady[task.String()])
+ }()
+ }
+ err = super.wait(super.ctx, tasks...)
+ if err != nil {
+ return err
+ }
+ super.logger.Info("all startup tasks are complete; starting health checks")
+ super.healthChecker = &health.Aggregator{Cluster: super.cluster}
+ <-super.ctx.Done()
+ super.logger.Info("shutting down")
+ super.waitShutdown.Wait()
+ return super.ctx.Err()
+}
+
+func (super *Supervisor) wait(ctx context.Context, tasks ...supervisedTask) error {
+ for _, task := range tasks {
+ ch, ok := super.tasksReady[task.String()]
+ if !ok {
+ return fmt.Errorf("no such task: %s", task)
+ }
+ super.logger.WithField("task", task.String()).Info("waiting")
+ select {
+ case <-ch:
+ super.logger.WithField("task", task.String()).Info("ready")
+ case <-ctx.Done():
+ super.logger.WithField("task", task.String()).Info("task was never ready")
+ return ctx.Err()
+ }
+ }
+ return nil
+}
+
+func (super *Supervisor) Stop() {
+ super.cancel()
+ <-super.done
+}
+
+func (super *Supervisor) WaitReady() (*arvados.URL, bool) {
+ ticker := time.NewTicker(time.Second)
+ defer ticker.Stop()
+ for waiting := "all"; waiting != ""; {
+ select {
+ case <-ticker.C:
+ case <-super.ctx.Done():
+ return nil, false
+ }
+ if super.healthChecker == nil {
+ // not set up yet
+ continue
+ }
+ resp := super.healthChecker.ClusterHealth()
+ // The overall health check (resp.Health=="OK") might
+ // never pass due to missing components (like
+ // arvados-dispatch-cloud in a test cluster), so
+ // instead we wait for all configured components to
+ // pass.
+ waiting = ""
+ for target, check := range resp.Checks {
+ if check.Health != "OK" {
+ waiting += " " + target
+ }
+ }
+ if waiting != "" {
+ super.logger.WithField("targets", waiting[1:]).Info("waiting")
+ }
+ }
+ u := super.cluster.Services.Controller.ExternalURL
+ return &u, true
+}
+
+func (super *Supervisor) prependEnv(key, prepend string) {
+ for i, s := range super.environ {
+ if strings.HasPrefix(s, key+"=") {
+ super.environ[i] = key + "=" + prepend + s[len(key)+1:]
+ return
+ }
+ }
+ super.environ = append(super.environ, key+"="+prepend)
+}
+
+func (super *Supervisor) cleanEnv(prefixes []string) {
+ var cleaned []string
+ for _, s := range super.environ {
+ drop := false
+ for _, p := range prefixes {
+ if strings.HasPrefix(s, p) {
+ drop = true
+ break
+ }
+ }
+ if !drop {
+ cleaned = append(cleaned, s)
+ }
+ }
+ super.environ = cleaned
+}
+
+func (super *Supervisor) setEnv(key, val string) {
+ for i, s := range super.environ {
+ if strings.HasPrefix(s, key+"=") {
+ super.environ[i] = key + "=" + val
+ return
+ }
+ }
+ super.environ = append(super.environ, key+"="+val)
+}
+
+// Remove all but the first occurrence of each env var.
+func dedupEnv(in []string) []string {
+ saw := map[string]bool{}
+ var out []string
+ for _, kv := range in {
+ if split := strings.Index(kv, "="); split < 1 {
+ panic("invalid environment var: " + kv)
+ } else if saw[kv[:split]] {
+ continue
+ } else {
+ saw[kv[:split]] = true
+ out = append(out, kv)
+ }
+ }
+ return out
+}
+
+func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) (string, error) {
+ _, basename := filepath.Split(srcpath)
+ bindir := filepath.Join(super.tempdir, "bin")
+ binfile := filepath.Join(bindir, basename)
+ err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
+ return binfile, err
+}
+
+func (super *Supervisor) usingRVM() bool {
+ return os.Getenv("rvm_path") != ""
+}
+
+func (super *Supervisor) setupRubyEnv() error {
+ if !super.usingRVM() {
+ // (If rvm is in use, assume the caller has everything
+ // set up as desired)
+ super.cleanEnv([]string{
+ "GEM_HOME=",
+ "GEM_PATH=",
+ })
+ cmd := exec.Command("gem", "env", "gempath")
+ cmd.Env = super.environ
+ buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:...
+ if err != nil || len(buf) == 0 {
+ return fmt.Errorf("gem env gempath: %v", err)
+ }
+ gempath := string(bytes.Split(buf, []byte{':'})[0])
+ super.prependEnv("PATH", gempath+"/bin:")
+ super.setEnv("GEM_HOME", gempath)
+ super.setEnv("GEM_PATH", gempath)
+ }
+ // Passenger install doesn't work unless $HOME is ~user
+ u, err := user.Current()
+ if err != nil {
+ return err
+ }
+ super.setEnv("HOME", u.HomeDir)
+ return nil
+}
+
+func (super *Supervisor) lookPath(prog string) string {
+ for _, val := range super.environ {
+ if strings.HasPrefix(val, "PATH=") {
+ for _, dir := range filepath.SplitList(val[5:]) {
+ path := filepath.Join(dir, prog)
+ if fi, err := os.Stat(path); err == nil && fi.Mode()&0111 != 0 {
+ return path
+ }
+ }
+ }
+ }
+ return prog
+}
+
+// Run prog with args, using dir as working directory. If ctx is
+// cancelled while the child is running, RunProgram terminates the
+// child, waits for it to exit, then returns.
+//
+// Child's environment will have our env vars, plus any given in env.
+//
+// Child's stdout will be written to output if non-nil, otherwise the
+// boot command's stderr.
+func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.Writer, env []string, prog string, args ...string) error {
+ cmdline := fmt.Sprintf("%s", append([]string{prog}, args...))
+ super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
+
+ logprefix := strings.TrimPrefix(prog, super.tempdir+"/bin/")
+ if logprefix == "bundle" && len(args) > 2 && args[0] == "exec" {
+ logprefix = args[1]
+ } else if logprefix == "arvados-server" && len(args) > 1 {
+ logprefix = args[0]
+ }
+ if !strings.HasPrefix(dir, "/") {
+ logprefix = dir + ": " + logprefix
+ }
+
+ cmd := exec.Command(super.lookPath(prog), args...)
+ stdout, err := cmd.StdoutPipe()
+ if err != nil {
+ return err
+ }
+ stderr, err := cmd.StderrPipe()
+ if err != nil {
+ return err
+ }
+ logwriter := &service.LogPrefixer{Writer: super.Stderr, Prefix: []byte("[" + logprefix + "] ")}
+ var copiers sync.WaitGroup
+ copiers.Add(1)
+ go func() {
+ io.Copy(logwriter, stderr)
+ copiers.Done()
+ }()
+ copiers.Add(1)
+ go func() {
+ if output == nil {
+ io.Copy(logwriter, stdout)
+ } else {
+ io.Copy(output, stdout)
+ }
+ copiers.Done()
+ }()
+
+ if strings.HasPrefix(dir, "/") {
+ cmd.Dir = dir
+ } else {
+ cmd.Dir = filepath.Join(super.SourcePath, dir)
+ }
+ env = append([]string(nil), env...)
+ env = append(env, super.environ...)
+ cmd.Env = dedupEnv(env)
+
+ exited := false
+ defer func() { exited = true }()
+ go func() {
+ <-ctx.Done()
+ log := ctxlog.FromContext(ctx).WithFields(logrus.Fields{"dir": dir, "cmdline": cmdline})
+ for !exited {
+ if cmd.Process == nil {
+ log.Debug("waiting for child process to start")
+ time.Sleep(time.Second / 2)
+ } else {
+ log.WithField("PID", cmd.Process.Pid).Debug("sending SIGTERM")
+ cmd.Process.Signal(syscall.SIGTERM)
+ time.Sleep(5 * time.Second)
+ if !exited {
+ stdout.Close()
+ stderr.Close()
+ log.WithField("PID", cmd.Process.Pid).Warn("still waiting for child process to exit 5s after SIGTERM")
+ }
+ }
+ }
+ }()
+
+ err = cmd.Start()
+ if err != nil {
+ return err
+ }
+ copiers.Wait()
+ err = cmd.Wait()
+ if ctx.Err() != nil {
+ // Return "context canceled", instead of the "killed"
+ // error that was probably caused by the context being
+ // canceled.
+ return ctx.Err()
+ } else if err != nil {
+ return fmt.Errorf("%s: error: %v", cmdline, err)
+ }
+ return nil
+}
+
+func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return err
+ }
+ usedPort := map[string]bool{}
+ nextPort := func(host string) string {
+ for {
+ port, err := availablePort(host)
+ if err != nil {
+ panic(err)
+ }
+ if usedPort[port] {
+ continue
+ }
+ usedPort[port] = true
+ return port
+ }
+ }
+ if cluster.Services.Controller.ExternalURL.Host == "" {
+ h, p, err := net.SplitHostPort(super.ControllerAddr)
+ if err != nil {
+ return err
+ }
+ if h == "" {
+ h = super.ListenHost
+ }
+ if p == "0" {
+ p = nextPort(h)
+ }
+ cluster.Services.Controller.ExternalURL = arvados.URL{Scheme: "https", Host: net.JoinHostPort(h, p)}
+ }
+ for _, svc := range []*arvados.Service{
+ &cluster.Services.Controller,
+ &cluster.Services.DispatchCloud,
+ &cluster.Services.GitHTTP,
+ &cluster.Services.Health,
+ &cluster.Services.Keepproxy,
+ &cluster.Services.Keepstore,
+ &cluster.Services.RailsAPI,
+ &cluster.Services.WebDAV,
+ &cluster.Services.WebDAVDownload,
+ &cluster.Services.Websocket,
+ &cluster.Services.Workbench1,
+ } {
+ if svc == &cluster.Services.DispatchCloud && super.ClusterType == "test" {
+ continue
+ }
+ if svc.ExternalURL.Host == "" {
+ if svc == &cluster.Services.Controller ||
+ svc == &cluster.Services.GitHTTP ||
+ svc == &cluster.Services.Keepproxy ||
+ svc == &cluster.Services.WebDAV ||
+ svc == &cluster.Services.WebDAVDownload ||
+ svc == &cluster.Services.Workbench1 {
+ svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}
+ } else if svc == &cluster.Services.Websocket {
+ svc.ExternalURL = arvados.URL{Scheme: "wss", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}
+ }
+ }
+ if len(svc.InternalURLs) == 0 {
+ svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{
+ arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}: arvados.ServiceInstance{},
+ }
+ }
+ }
+ if cluster.SystemRootToken == "" {
+ cluster.SystemRootToken = randomHexString(64)
+ }
+ if cluster.ManagementToken == "" {
+ cluster.ManagementToken = randomHexString(64)
+ }
+ if cluster.API.RailsSessionSecretToken == "" {
+ cluster.API.RailsSessionSecretToken = randomHexString(64)
+ }
+ if cluster.Collections.BlobSigningKey == "" {
+ cluster.Collections.BlobSigningKey = randomHexString(64)
+ }
+ if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" {
+ buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
+ if err != nil {
+ return err
+ }
+ cluster.Containers.DispatchPrivateKey = string(buf)
+ }
+ if super.ClusterType != "production" {
+ cluster.TLS.Insecure = true
+ }
+ if super.ClusterType == "test" {
+ // Add a second keepstore process.
+ cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}] = arvados.ServiceInstance{}
+
+ // Create a directory-backed volume for each keepstore
+ // process.
+ cluster.Volumes = map[string]arvados.Volume{}
+ for url := range cluster.Services.Keepstore.InternalURLs {
+ volnum := len(cluster.Volumes)
+ datadir := fmt.Sprintf("%s/keep%d.data", super.tempdir, volnum)
+ if _, err = os.Stat(datadir + "/."); err == nil {
+ } else if !os.IsNotExist(err) {
+ return err
+ } else if err = os.Mkdir(datadir, 0755); err != nil {
+ return err
+ }
+ cluster.Volumes[fmt.Sprintf(cluster.ClusterID+"-nyw5e-%015d", volnum)] = arvados.Volume{
+ Driver: "Directory",
+ DriverParameters: json.RawMessage(fmt.Sprintf(`{"Root":%q}`, datadir)),
+ AccessViaHosts: map[arvados.URL]arvados.VolumeAccess{
+ url: {},
+ },
+ }
+ }
+ }
+ if super.OwnTemporaryDatabase {
+ cluster.PostgreSQL.Connection = arvados.PostgreSQLConnection{
+ "client_encoding": "utf8",
+ "host": "localhost",
+ "port": nextPort(super.ListenHost),
+ "dbname": "arvados_test",
+ "user": "arvados",
+ "password": "insecure_arvados_test",
+ }
+ }
+
+ cfg.Clusters[cluster.ClusterID] = *cluster
+ return nil
+}
+
+func addrIsLocal(addr string) (bool, error) {
+ return true, nil
+ listener, err := net.Listen("tcp", addr)
+ if err == nil {
+ listener.Close()
+ return true, nil
+ } else if strings.Contains(err.Error(), "cannot assign requested address") {
+ return false, nil
+ } else {
+ return false, err
+ }
+}
+
+func randomHexString(chars int) string {
+ b := make([]byte, chars/2)
+ _, err := rand.Read(b)
+ if err != nil {
+ panic(err)
+ }
+ return fmt.Sprintf("%x", b)
+}
+
+func internalPort(svc arvados.Service) (string, error) {
+ if len(svc.InternalURLs) > 1 {
+ return "", errors.New("internalPort() doesn't work with multiple InternalURLs")
+ }
+ for u := range svc.InternalURLs {
+ if _, p, err := net.SplitHostPort(u.Host); err != nil {
+ return "", err
+ } else if p != "" {
+ return p, nil
+ } else if u.Scheme == "https" {
+ return "443", nil
+ } else {
+ return "80", nil
+ }
+ }
+ return "", fmt.Errorf("service has no InternalURLs")
+}
+
+func externalPort(svc arvados.Service) (string, error) {
+ if _, p, err := net.SplitHostPort(svc.ExternalURL.Host); err != nil {
+ return "", err
+ } else if p != "" {
+ return p, nil
+ } else if svc.ExternalURL.Scheme == "https" {
+ return "443", nil
+ } else {
+ return "80", nil
+ }
+}
+
+func availablePort(host string) (string, error) {
+ ln, err := net.Listen("tcp", net.JoinHostPort(host, "0"))
+ if err != nil {
+ return "", err
+ }
+ defer ln.Close()
+ _, port, err := net.SplitHostPort(ln.Addr().String())
+ if err != nil {
+ return "", err
+ }
+ return port, nil
+}
+
+// Try to connect to addr until it works, then close ch. Give up if
+// ctx cancels.
+func waitForConnect(ctx context.Context, addr string) error {
+ dialer := net.Dialer{Timeout: time.Second}
+ for ctx.Err() == nil {
+ conn, err := dialer.DialContext(ctx, "tcp", addr)
+ if err != nil {
+ time.Sleep(time.Second / 10)
+ continue
+ }
+ conn.Close()
+ return nil
+ }
+ return ctx.Err()
+}
# (experimental) cloud dispatcher for executing containers on
# worker VMs. Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
# and ends with "\n-----END RSA PRIVATE KEY-----\n".
- DispatchPrivateKey: none
+ DispatchPrivateKey: ""
# Maximum time to wait for workers to come up before abandoning
# stale locks from a previous dispatch process.
<a href="https://doc.arvados.org/user/getting_started/ssh-access-unix.html">Accessing an Arvados VM with SSH</a> (generic instructions).
Site configurations vary. Contact your local cluster administrator if you have difficulty accessing an Arvados shell node.
+ # Sample text if you are using a "switchyard" ssh proxy.
+ # Replace "zzzzz" with your Cluster ID.
+ #SSHHelpPageHTML: |
+ # <p>Add a section like this to your SSH configuration file ( <i>~/.ssh/config</i>):</p>
+ # <pre>Host *.zzzzz
+ # TCPKeepAlive yes
+ # ServerAliveInterval 60
+ # ProxyCommand ssh -p2222 turnout@switchyard.zzzzz.arvadosapi.com -x -a $SSH_PROXY_FLAGS %h
+ # </pre>
+
+ # If you are using a switchyard ssh proxy, shell node hostnames
+ # may require a special hostname suffix. In the sample ssh
+ # configuration above, this would be ".zzzzz"
+ # This is added to the hostname in the "command line" column
+ # the Workbench "shell VMs" page.
+ #
+ # If your shell nodes are directly accessible by users without a
+ # proxy and have fully qualified host names, you should leave
+ # this blank.
+ SSHHelpHostSuffix: ""
+
# Bypass new (Arvados 1.5) API implementations, and hand off
# requests directly to Rails instead. This can provide a temporary
# workaround for clients that are incompatible with the new API
"Login.ProviderAppSecret": false,
"Login.LoginCluster": true,
"Login.RemoteTokenRefresh": true,
- "Mail": false,
+ "Mail": true,
+ "Mail.MailchimpAPIKey": false,
+ "Mail.MailchimpListID": false,
+ "Mail.SendUserSetupNotificationEmail": false,
+ "Mail.IssueReporterEmailFrom": false,
+ "Mail.IssueReporterEmailTo": false,
+ "Mail.SupportEmailAddress": true,
+ "Mail.EmailFrom": false,
"ManagementToken": false,
"PostgreSQL": false,
"RemoteClusters": true,
"Workbench.WelcomePageHTML": true,
"Workbench.InactivePageHTML": true,
"Workbench.SSHHelpPageHTML": true,
+ "Workbench.SSHHelpHostSuffix": true,
}
func redactUnsafe(m map[string]interface{}, mPrefix, lookupPrefix string) error {
# (experimental) cloud dispatcher for executing containers on
# worker VMs. Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
# and ends with "\n-----END RSA PRIVATE KEY-----\n".
- DispatchPrivateKey: none
+ DispatchPrivateKey: ""
# Maximum time to wait for workers to come up before abandoning
# stale locks from a previous dispatch process.
<a href="https://doc.arvados.org/user/getting_started/ssh-access-unix.html">Accessing an Arvados VM with SSH</a> (generic instructions).
Site configurations vary. Contact your local cluster administrator if you have difficulty accessing an Arvados shell node.
+ # Sample text if you are using a "switchyard" ssh proxy.
+ # Replace "zzzzz" with your Cluster ID.
+ #SSHHelpPageHTML: |
+ # <p>Add a section like this to your SSH configuration file ( <i>~/.ssh/config</i>):</p>
+ # <pre>Host *.zzzzz
+ # TCPKeepAlive yes
+ # ServerAliveInterval 60
+ # ProxyCommand ssh -p2222 turnout@switchyard.zzzzz.arvadosapi.com -x -a $SSH_PROXY_FLAGS %h
+ # </pre>
+
+ # If you are using a switchyard ssh proxy, shell node hostnames
+ # may require a special hostname suffix. In the sample ssh
+ # configuration above, this would be ".zzzzz"
+ # This is added to the hostname in the "command line" column
+ # the Workbench "shell VMs" page.
+ #
+ # If your shell nodes are directly accessible by users without a
+ # proxy and have fully qualified host names, you should leave
+ # this blank.
+ SSHHelpHostSuffix: ""
+
# Bypass new (Arvados 1.5) API implementations, and hand off
# requests directly to Rails instead. This can provide a temporary
# workaround for clients that are incompatible with the new API
import (
"bytes"
"context"
- "crypto/md5"
"encoding/json"
"errors"
"fmt"
local := localdb.NewConn(cluster)
remotes := map[string]backend{}
for id, remote := range cluster.RemoteClusters {
- if !remote.Proxy {
+ if !remote.Proxy || id == cluster.ClusterID {
continue
}
conn := rpc.NewConn(id, &url.URL{Scheme: remote.Scheme, Host: remote.Host}, remote.Insecure, saltedTokenProvider(local, id))
})
}
-// this could be in sdk/go/arvados
-func portableDataHash(mt string) string {
- h := md5.New()
- blkRe := regexp.MustCompile(`^ [0-9a-f]{32}\+\d+`)
- size := 0
- _ = regexp.MustCompile(` ?[^ ]*`).ReplaceAllFunc([]byte(mt), func(tok []byte) []byte {
- if m := blkRe.Find(tok); m != nil {
- // write hash+size, ignore remaining block hints
- tok = m
- }
- n, err := h.Write(tok)
- if err != nil {
- panic(err)
- }
- size += n
- return nil
- })
- return fmt.Sprintf("%x+%d", h.Sum(nil), size)
-}
-
func (conn *Conn) ConfigGet(ctx context.Context) (json.RawMessage, error) {
var buf bytes.Buffer
err := config.ExportJSON(&buf, conn.cluster)
// options.UUID is either hash+size or
// hash+size+hints; only hash+size need to
// match the computed PDH.
- if pdh := portableDataHash(c.ManifestText); pdh != options.UUID && !strings.HasPrefix(options.UUID, pdh+"+") {
+ if pdh := arvados.PortableDataHash(c.ManifestText); pdh != options.UUID && !strings.HasPrefix(options.UUID, pdh+"+") {
err = httpErrorf(http.StatusBadGateway, "bad portable data hash %q received from remote %q (expected %q)", pdh, remoteID, options.UUID)
ctxlog.FromContext(ctx).Warn(err)
return err
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"github.com/prometheus/client_golang/prometheus"
c.Check(user.Authorization.TokenV2(), check.Equals, arvadostest.ActiveTokenV2)
}
+func (s *HandlerSuite) TestValidateRemoteToken(c *check.C) {
+ saltedToken, err := auth.SaltToken(arvadostest.ActiveTokenV2, "abcde")
+ c.Assert(err, check.IsNil)
+ for _, trial := range []struct {
+ code int
+ token string
+ }{
+ {http.StatusOK, saltedToken},
+ {http.StatusUnauthorized, "bogus"},
+ } {
+ req := httptest.NewRequest("GET", "https://0.0.0.0:1/arvados/v1/users/current?remote=abcde", nil)
+ req.Header.Set("Authorization", "Bearer "+trial.token)
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ if !c.Check(resp.Code, check.Equals, trial.code) {
+ c.Logf("HTTP %d: %s", resp.Code, resp.Body.String())
+ }
+ }
+}
+
func (s *HandlerSuite) TestCreateAPIToken(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
auth, err := s.handler.(*Handler).createAPItoken(req, arvadostest.ActiveUserUUID, nil)
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package controller
+
+import (
+ "bytes"
+ "context"
+ "io"
+ "net"
+ "net/url"
+ "os"
+ "path/filepath"
+
+ "git.arvados.org/arvados.git/lib/boot"
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/lib/service"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/auth"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&IntegrationSuite{})
+
+type testCluster struct {
+ super boot.Supervisor
+ config arvados.Config
+ controllerURL *url.URL
+}
+
+type IntegrationSuite struct {
+ testClusters map[string]*testCluster
+}
+
+func (s *IntegrationSuite) SetUpSuite(c *check.C) {
+ if forceLegacyAPI14 {
+ c.Skip("heavy integration tests don't run with forceLegacyAPI14")
+ return
+ }
+
+ cwd, _ := os.Getwd()
+ s.testClusters = map[string]*testCluster{
+ "z1111": nil,
+ "z2222": nil,
+ "z3333": nil,
+ }
+ hostport := map[string]string{}
+ for id := range s.testClusters {
+ hostport[id] = func() string {
+ // TODO: Instead of expecting random ports on
+ // 127.0.0.11, 22, 33 to be race-safe, try
+ // different 127.x.y.z until finding one that
+ // isn't in use.
+ ln, err := net.Listen("tcp", ":0")
+ c.Assert(err, check.IsNil)
+ ln.Close()
+ _, port, err := net.SplitHostPort(ln.Addr().String())
+ c.Assert(err, check.IsNil)
+ return "127.0.0." + id[3:] + ":" + port
+ }()
+ }
+ for id := range s.testClusters {
+ yaml := `Clusters:
+ ` + id + `:
+ Services:
+ Controller:
+ ExternalURL: https://` + hostport[id] + `
+ TLS:
+ Insecure: true
+ Login:
+ # LoginCluster: z1111
+ SystemLogs:
+ Format: text
+ RemoteClusters:
+ z1111:
+ Host: ` + hostport["z1111"] + `
+ Scheme: https
+ Insecure: true
+ Proxy: true
+ ActivateUsers: true
+ z2222:
+ Host: ` + hostport["z2222"] + `
+ Scheme: https
+ Insecure: true
+ Proxy: true
+ ActivateUsers: true
+ z3333:
+ Host: ` + hostport["z3333"] + `
+ Scheme: https
+ Insecure: true
+ Proxy: true
+ ActivateUsers: true
+`
+ loader := config.NewLoader(bytes.NewBufferString(yaml), ctxlog.TestLogger(c))
+ loader.Path = "-"
+ loader.SkipLegacy = true
+ loader.SkipAPICalls = true
+ cfg, err := loader.Load()
+ c.Assert(err, check.IsNil)
+ s.testClusters[id] = &testCluster{
+ super: boot.Supervisor{
+ SourcePath: filepath.Join(cwd, "..", ".."),
+ ClusterType: "test",
+ ListenHost: "127.0.0." + id[3:],
+ ControllerAddr: ":0",
+ OwnTemporaryDatabase: true,
+ Stderr: &service.LogPrefixer{Writer: ctxlog.LogWriter(c.Log), Prefix: []byte("[" + id + "] ")},
+ },
+ config: *cfg,
+ }
+ s.testClusters[id].super.Start(context.Background(), &s.testClusters[id].config)
+ }
+ for _, tc := range s.testClusters {
+ au, ok := tc.super.WaitReady()
+ c.Assert(ok, check.Equals, true)
+ u := url.URL(*au)
+ tc.controllerURL = &u
+ }
+}
+
+func (s *IntegrationSuite) TearDownSuite(c *check.C) {
+ for _, c := range s.testClusters {
+ c.super.Stop()
+ }
+}
+
+func (s *IntegrationSuite) conn(clusterID string) *rpc.Conn {
+ return rpc.NewConn(clusterID, s.testClusters[clusterID].controllerURL, true, rpc.PassthroughTokenProvider)
+}
+
+func (s *IntegrationSuite) clientsWithToken(clusterID string, token string) (context.Context, *arvados.Client, *keepclient.KeepClient) {
+ cl := s.testClusters[clusterID].config.Clusters[clusterID]
+ ctx := auth.NewContext(context.Background(), auth.NewCredentials(token))
+ ac, err := arvados.NewClientFromConfig(&cl)
+ if err != nil {
+ panic(err)
+ }
+ ac.AuthToken = token
+ arv, err := arvadosclient.New(ac)
+ if err != nil {
+ panic(err)
+ }
+ kc := keepclient.New(arv)
+ return ctx, ac, kc
+}
+
+func (s *IntegrationSuite) userClients(c *check.C, conn *rpc.Conn, rootctx context.Context, clusterID string, activate bool) (context.Context, *arvados.Client, *keepclient.KeepClient) {
+ login, err := conn.UserSessionCreate(rootctx, rpc.UserSessionCreateOptions{
+ ReturnTo: ",https://example.com",
+ AuthInfo: rpc.UserSessionAuthInfo{
+ Email: "user@example.com",
+ FirstName: "Example",
+ LastName: "User",
+ Username: "example",
+ },
+ })
+ c.Assert(err, check.IsNil)
+ redirURL, err := url.Parse(login.RedirectLocation)
+ c.Assert(err, check.IsNil)
+ userToken := redirURL.Query().Get("api_token")
+ c.Logf("user token: %q", userToken)
+ ctx, ac, kc := s.clientsWithToken(clusterID, userToken)
+ user, err := conn.UserGetCurrent(ctx, arvados.GetOptions{})
+ c.Assert(err, check.IsNil)
+ _, err = conn.UserSetup(rootctx, arvados.UserSetupOptions{UUID: user.UUID})
+ c.Assert(err, check.IsNil)
+ if activate {
+ _, err = conn.UserActivate(rootctx, arvados.UserActivateOptions{UUID: user.UUID})
+ c.Assert(err, check.IsNil)
+ user, err = conn.UserGetCurrent(ctx, arvados.GetOptions{})
+ c.Assert(err, check.IsNil)
+ c.Logf("user UUID: %q", user.UUID)
+ if !user.IsActive {
+ c.Fatalf("failed to activate user -- %#v", user)
+ }
+ }
+ return ctx, ac, kc
+}
+
+func (s *IntegrationSuite) rootClients(clusterID string) (context.Context, *arvados.Client, *keepclient.KeepClient) {
+ return s.clientsWithToken(clusterID, s.testClusters[clusterID].config.Clusters[clusterID].SystemRootToken)
+}
+
+func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) {
+ conn1 := s.conn("z1111")
+ rootctx1, _, _ := s.rootClients("z1111")
+ conn3 := s.conn("z3333")
+ userctx1, ac1, kc1 := s.userClients(c, conn1, rootctx1, "z1111", true)
+
+ // Create the collection to find its PDH (but don't save it
+ // anywhere yet)
+ var coll1 arvados.Collection
+ fs1, err := coll1.FileSystem(ac1, kc1)
+ c.Assert(err, check.IsNil)
+ f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = io.WriteString(f, "IntegrationSuite.TestGetCollectionByPDH")
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+ mtxt, err := fs1.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ pdh := arvados.PortableDataHash(mtxt)
+
+ // Looking up the PDH before saving returns 404 if cycle
+ // detection is working.
+ _, err = conn1.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh})
+ c.Assert(err, check.ErrorMatches, `.*404 Not Found.*`)
+
+ // Save the collection on cluster z1111.
+ coll1, err = conn1.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
+ "manifest_text": mtxt,
+ }})
+ c.Assert(err, check.IsNil)
+
+ // Retrieve the collection from cluster z3333.
+ coll, err := conn3.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh})
+ c.Check(err, check.IsNil)
+ c.Check(coll.PortableDataHash, check.Equals, pdh)
+}
cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
s.cluster, err = cfg.GetCluster("")
+ s.cluster.Login.ProviderAppID = ""
+ s.cluster.Login.ProviderAppSecret = ""
s.cluster.Login.GoogleClientID = "test%client$id"
s.cluster.Login.GoogleClientSecret = "test#client/secret"
s.cluster.Users.PreferDomainForUsername = "PreferDomainForUsername.example.com"
// A QueueEnt is an entry in the queue, consisting of a container
// record and the instance type that should be used to run it.
type QueueEnt struct {
- // The container to run. Only the UUID, State, Priority, and
- // RuntimeConstraints fields are populated.
+ // The container to run. Only the UUID, State, Priority,
+ // RuntimeConstraints, Mounts, and ContainerImage fields are
+ // populated.
Container arvados.Container `json:"container"`
InstanceType arvados.InstanceType `json:"instance_type"`
}
*next[upd.UUID] = upd
}
}
- selectParam := []string{"uuid", "state", "priority", "runtime_constraints"}
+ selectParam := []string{"uuid", "state", "priority", "runtime_constraints", "container_image", "mounts"}
limitParam := 1000
mine, err := cq.fetchAll(arvados.ResourceListParams{
func (suite *IntegrationSuite) TestCancelIfNoInstanceType(c *check.C) {
errorTypeChooser := func(ctr *arvados.Container) (arvados.InstanceType, error) {
+ // Make sure the relevant container fields are
+ // actually populated.
+ c.Check(ctr.ContainerImage, check.Equals, "test")
+ c.Check(ctr.RuntimeConstraints.VCPUs, check.Equals, 4)
+ c.Check(ctr.RuntimeConstraints.RAM, check.Equals, int64(12000000000))
+ c.Check(ctr.Mounts["/tmp"].Capacity, check.Equals, int64(24000000000))
+ c.Check(ctr.Mounts["/var/spool/cwl"].Capacity, check.Equals, int64(24000000000))
return arvados.InstanceType{}, errors.New("no suitable instance type")
}
// time (Idle) or the earliest create time (Booting)
for _, wkr := range wp.workers {
if wkr.idleBehavior != IdleBehaviorHold && wkr.state == tryState && wkr.instType == it {
- logger.WithField("Instance", wkr.instance).Info("shutting down")
+ logger.WithField("Instance", wkr.instance.ID()).Info("shutting down")
wkr.shutdown()
return true
}
itTag := inst.Tags()[wp.tagKeyPrefix+tagKeyInstanceType]
it, ok := wp.instanceTypes[itTag]
if !ok {
- wp.logger.WithField("Instance", inst).Errorf("unknown InstanceType tag %q --- ignoring", itTag)
+ wp.logger.WithField("Instance", inst.ID()).Errorf("unknown InstanceType tag %q --- ignoring", itTag)
continue
}
if wkr, isNew := wp.updateWorker(inst, it); isNew {
notify = true
} else if wkr.state == StateShutdown && time.Since(wkr.destroyed) > wp.timeoutShutdown {
- wp.logger.WithField("Instance", inst).Info("worker still listed after shutdown; retrying")
+ wp.logger.WithField("Instance", inst.ID()).Info("worker still listed after shutdown; retrying")
wkr.shutdown()
}
}
"io"
"net"
"net/http"
+ "net/url"
"os"
"strings"
var err error
defer func() {
if err != nil {
- log.WithError(err).Info("exiting")
+ log.WithError(err).Error("exiting")
}
}()
if !ok {
return arvados.URL{}, fmt.Errorf("unknown service name %q", prog)
}
+
+ if want := os.Getenv("ARVADOS_SERVICE_INTERNAL_URL"); want == "" {
+ } else if url, err := url.Parse(want); err != nil {
+ return arvados.URL{}, fmt.Errorf("$ARVADOS_SERVICE_INTERNAL_URL (%q): %s", want, err)
+ } else {
+ return arvados.URL(*url), nil
+ }
+
errors := []string{}
for url := range svc.InternalURLs {
listener, err := net.Listen("tcp", url.Host)
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package service
+
+import (
+ "bytes"
+ "io"
+)
+
+type LogPrefixer struct {
+ io.Writer
+ Prefix []byte
+ did bool
+}
+
+func (lp *LogPrefixer) Write(p []byte) (int, error) {
+ if len(p) == 0 {
+ return 0, nil
+ }
+ var out []byte
+ if !lp.did {
+ out = append(out, lp.Prefix...)
+ }
+ lp.did = p[len(p)-1] != '\n'
+ out = append(out, bytes.Replace(p[:len(p)-1], []byte("\n"), append([]byte("\n"), lp.Prefix...), -1)...)
+ out = append(out, p[len(p)-1])
+ _, err := lp.Writer.Write(out)
+ if err != nil {
+ return 0, err
+ }
+ return len(p), nil
+}
ENV["GIT_DIR"] = File.expand_path "#{__dir__}/../../.git"
ENV["GIT_WORK_TREE"] = File.expand_path "#{__dir__}/../.."
git_timestamp, git_hash = `git log -n1 --first-parent --format=%ct:%H #{__dir__}`.chomp.split(":")
- version = `#{__dir__}/../../build/version-at-commit.sh #{git_hash}`.encode('utf-8').strip
+ if ENV["ARVADOS_BUILDING_VERSION"]
+ version = ENV["ARVADOS_BUILDING_VERSION"]
+ else
+ version = `#{__dir__}/../../build/version-at-commit.sh #{git_hash}`.encode('utf-8').strip
+ end
git_timestamp = Time.at(git_timestamp.to_i).utc
ensure
ENV["GIT_DIR"] = git_dir
s.add_runtime_dependency 'andand', '~> 1.3', '>= 1.3.3'
s.add_runtime_dependency 'oj', '~> 3.0'
s.add_runtime_dependency 'curb', '~> 0.8'
+ s.add_runtime_dependency 'launchy', '< 2.5'
# arvados-google-api-client 0.8.7.2 is incompatible with faraday 0.16.2
s.add_dependency('faraday', '< 0.16')
s.homepage =
import logging
from schema_salad.sourceline import SourceLine, cmap
+import schema_salad.ref_resolver
from cwltool.pack import pack
-from cwltool.load_tool import fetch_document
+from cwltool.load_tool import fetch_document, resolve_and_validate_document
from cwltool.process import shortname
from cwltool.workflow import Workflow, WorkflowException, WorkflowStep
from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
with SourceLine(self.tool, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
if "id" not in self.tool:
raise WorkflowException("%s object must have 'id'" % (self.tool["class"]))
- document_loader, workflowobj, uri = (self.doc_loader, self.doc_loader.fetch(self.tool["id"]), self.tool["id"])
discover_secondary_files(self.arvrunner.fs_access, builder,
self.tool["inputs"], joborder)
with Perf(metrics, "subworkflow upload_deps"):
upload_dependencies(self.arvrunner,
os.path.basename(joborder.get("id", "#")),
- document_loader,
+ self.doc_loader,
joborder,
joborder.get("id", "#"),
False)
if self.wf_pdh is None:
- workflowobj["requirements"] = dedup_reqs(self.requirements)
- workflowobj["hints"] = dedup_reqs(self.hints)
+ packed = pack(self.loadingContext, self.tool["id"], loader=self.doc_loader)
- packed = pack(document_loader, workflowobj, uri, self.metadata)
+ for p in packed["$graph"]:
+ if p["id"] == "#main":
+ p["requirements"] = dedup_reqs(self.requirements)
+ p["hints"] = dedup_reqs(self.hints)
def visit(item):
+ if "requirements" in item:
+ item["requirements"] = [i for i in item["requirements"] if i["class"] != "DockerRequirement"]
for t in ("hints", "requirements"):
if t not in item:
continue
raise WorkflowException("Non-top-level ResourceRequirement in single container cannot have expressions")
if not dyn:
self.static_resource_req.append(req)
- if req["class"] == "DockerRequirement":
- if "http://arvados.org/cwl#dockerCollectionPDH" in req:
- del req["http://arvados.org/cwl#dockerCollectionPDH"]
visit_class(packed["$graph"], ("Workflow", "CommandLineTool"), visit)
upload_dependencies(self.arvrunner,
runtimeContext.name,
- document_loader,
+ self.doc_loader,
packed,
- uri,
+ self.tool["id"],
False)
# Discover files/directories referenced by the
if job_res_reqs[0].get("ramMin", 1024) < 128:
job_res_reqs[0]["ramMin"] = 128
+ arguments = ["--no-container", "--move-outputs", "--preserve-entire-environment", "workflow.cwl", "cwl.input.yml"]
+ if runtimeContext.debug:
+ arguments.insert(0, '--debug')
+
wf_runner = cmap({
"class": "CommandLineTool",
"baseCommand": "cwltool",
}]
}],
"hints": self.hints,
- "arguments": ["--no-container", "--move-outputs", "--preserve-entire-environment", "workflow.cwl#main", "cwl.input.yml"],
+ "arguments": arguments,
"id": "#"
})
return ArvadosCommandTool(self.arvrunner, wf_runner, self.loadingContext).job(joborder_resolved, output_callback, runtimeContext)
A "packed" workflow is one where all the components have been combined into a single document."""
rewrites = {}
- packed = pack(tool.doc_loader, tool.doc_loader.fetch(tool.tool["id"]),
- tool.tool["id"], tool.metadata, rewrite_out=rewrites)
+ packed = pack(arvrunner.loadingContext, tool.tool["id"],
+ rewrite_out=rewrites,
+ loader=tool.doc_loader)
rewrite_to_orig = {v: k for k,v in viewitems(rewrites)}
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==1.0.20190831161204',
- 'schema-salad==4.5.20190815125611',
- 'typing >= 3.6.4',
- 'ruamel.yaml >=0.15.54, <=0.15.77',
+ 'cwltool==3.0.20200317203547',
+ 'schema-salad==5.0.20200302192450',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
- 'ciso8601 >= 2.0.0',
- 'networkx < 2.3'
+ 'ciso8601 >= 2.0.0'
],
extras_require={
':os.name=="posix" and python_version<"3"': ['subprocess32 >= 3.5.1'],
data_files=[
('share/doc/arvados-cwl-runner', ['LICENSE-2.0.txt', 'README.rst']),
],
+ python_requires=">=3.5, <4",
classifiers=[
- 'Programming Language :: Python :: 2',
'Programming Language :: Python :: 3',
],
test_suite='tests',
reset_container=1
leave_running=0
config=dev
+devcwl=0
tag="latest"
-pythoncmd=python
+pythoncmd=python3
suite=conformance
runapi=containers
build=1
shift
;;
+ --devcwl)
+ devcwl=1
+ shift
+ ;;
--pythoncmd)
pythoncmd=$2
shift ; shift
shift ; shift
;;
-h|--help)
- echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag] [--build] [--pythoncmd python(2|3)] [--suite (integration|conformance-v1.0|conformance-v1.1)]"
+ echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag] [--build] [--pythoncmd python(2|3)] [--suite (integration|conformance-v1.0|conformance-*)]"
exit
;;
*)
git clone https://github.com/common-workflow-language/common-workflow-language.git
fi
cd common-workflow-language
-elif [[ "$suite" = "conformance-v1.1" ]] ; then
- if ! test -d cwl-v1.1 ; then
- git clone https://github.com/common-workflow-language/cwl-v1.1.git
+elif [[ "$suite" =~ conformance-(.*) ]] ; then
+ version=\${BASH_REMATCH[1]}
+ if ! test -d cwl-\${version} ; then
+ git clone https://github.com/common-workflow-language/cwl-\${version}.git
fi
- cd cwl-v1.1
+ cd cwl-\${version}
fi
if [[ "$suite" != "integration" ]] ; then
EOF2
chmod +x /tmp/cwltest/arv-cwl-containers
+EXTRA=--compute-checksum
+
+if [[ $devcwl == 1 ]] ; then
+ EXTRA="\$EXTRA --enable-dev"
+fi
+
env
if [[ "$suite" = "integration" ]] ; then
cd /usr/src/arvados/sdk/cwl/tests
exec ./arvados-tests.sh $@
else
- exec ./run_test.sh RUNNER=/tmp/cwltest/arv-cwl-${runapi} EXTRA=--compute-checksum $@
+ exec ./run_test.sh RUNNER=/tmp/cwltest/arv-cwl-${runapi} EXTRA="\$EXTRA" $@
fi
EOF
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+requirements:
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+inputs:
+ d: Directory
+steps:
+ step1:
+ in:
+ d: d
+ out: [out]
+ run: wf/16169-step.cwl
+outputs:
+ out:
+ type: File
+ outputSource: step1/out
should_fail: true
tool: 15295-bad-keep-ref.cwl
doc: Test checking for invalid keepref
+
+- job: listing-job.yml
+ output: {
+ "out": {
+ "class": "File",
+ "location": "output.txt",
+ "size": 5,
+ "checksum": "sha1$724ba28f4a9a1b472057ff99511ed393a45552e1"
+ }
+ }
+ tool: 16169-no-listing-hint.cwl
+ doc: "Test cwltool:LoadListingRequirement propagation"
import functools
import cwltool.process
import cwltool.secrets
+from cwltool.update import INTERNAL_VERSION
from schema_salad.ref_resolver import Loader
from schema_salad.sourceline import cmap
cwltool.process._names = set()
def helper(self, runner, enable_reuse=True):
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
"basedir": "",
"make_fs_access": make_fs_access,
"loader": Loader({}),
- "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
+ "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
runtimeContext = arvados_cwl.context.ArvRuntimeContext(
{"work_api": "containers",
"basedir": "",
runner.api.collections().get().execute.return_value = {
"portable_data_hash": "99999999999999999999999999999993+99"}
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
tool = cmap({
"inputs": [],
cwltool.process._names = set()
def helper(self, runner, enable_reuse=True):
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
"basedir": "",
"make_fs_access": make_fs_access,
"loader": document_loader,
- "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
+ "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
"construct_tool_object": runner.arv_make_tool})
runtimeContext = arvados_cwl.context.ArvRuntimeContext(
{"work_api": "containers",
"--no-container",
"--move-outputs",
"--preserve-entire-environment",
- "workflow.cwl#main",
+ "workflow.cwl",
"cwl.input.yml"
],
"container_image": "99999999999999999999999999999993+99",
u'--no-container',
u'--move-outputs',
u'--preserve-entire-environment',
- u'workflow.cwl#main',
+ u'workflow.cwl',
u'cwl.input.yml'
],
'use_existing': True,
import sys
import unittest
import cwltool.process
+import re
from io import BytesIO
self.assertEqual(exited, 1)
self.assertRegexpMatches(
- capture_stderr.getvalue(),
+ re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
finally:
cwltool_logger.removeHandler(stderr_logger)
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+class: CommandLineTool
+cwlVersion: v1.0
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: debian:stretch-slim
+inputs:
+ d: Directory
+outputs:
+ out: stdout
+stdout: output.txt
+arguments:
+ [echo, "${if(inputs.d.listing === undefined) {return 'true';} else {return 'false';}}"]
{
"$graph": [
{
+ "$namespaces": {
+ "arv": "http://arvados.org/cwl#"
+ },
"class": "Workflow",
- "cwlVersion": "v1.1",
+ "cwlVersion": "v1.0",
"hints": [],
"id": "#main",
"inputs": [
"run": {
"baseCommand": "sleep",
"class": "CommandLineTool",
- "id": "#main/sleep1/run/subtool",
+ "id": "#main/sleep1/subtool",
"inputs": [
{
- "id": "#main/sleep1/run/subtool/sleeptime",
+ "id": "#main/sleep1/subtool/sleeptime",
"inputBinding": {
"position": 1
},
],
"outputs": [
{
- "id": "#main/sleep1/run/subtool/out",
+ "id": "#main/sleep1/subtool/out",
"outputBinding": {
"outputEval": "out"
},
]
}
],
- "cwlVersion": "v1.1"
-}
+ "cwlVersion": "v1.0"
+}
\ No newline at end of file
Select []string `json:"select"`
IncludeTrash bool `json:"include_trash"`
ForwardedFor string `json:"forwarded_for"`
+ Remote string `json:"remote"`
}
type UntrashOptions struct {
import (
"bufio"
+ "crypto/md5"
"fmt"
+ "regexp"
"strings"
"time"
Offset int `json:"offset"`
Limit int `json:"limit"`
}
+
+var (
+ blkRe = regexp.MustCompile(`^ [0-9a-f]{32}\+\d+`)
+ tokRe = regexp.MustCompile(` ?[^ ]*`)
+)
+
+// PortableDataHash computes the portable data hash of the given
+// manifest.
+func PortableDataHash(mt string) string {
+ h := md5.New()
+ size := 0
+ _ = tokRe.ReplaceAllFunc([]byte(mt), func(tok []byte) []byte {
+ if m := blkRe.Find(tok); m != nil {
+ // write hash+size, ignore remaining block hints
+ tok = m
+ }
+ n, err := h.Write(tok)
+ if err != nil {
+ panic(err)
+ }
+ size += n
+ return nil
+ })
+ return fmt.Sprintf("%x+%d", h.Sum(nil), size)
+}
WelcomePageHTML string
InactivePageHTML string
SSHHelpPageHTML string
+ SSHHelpHostSuffix string
}
ForceLegacyAPI14 bool
type ContainerRequestState string
const (
- ContainerRequestStateUncomitted = ContainerState("Uncommitted")
- ContainerRequestStateCommitted = ContainerState("Committed")
- ContainerRequestStateFinal = ContainerState("Final")
+ ContainerRequestStateUncomitted = ContainerRequestState("Uncommitted")
+ ContainerRequestStateCommitted = ContainerRequestState("Committed")
+ ContainerRequestStateFinal = ContainerRequestState("Final")
)
Tokens []string
}
-func NewCredentials() *Credentials {
- return &Credentials{Tokens: []string{}}
+func NewCredentials(tokens ...string) *Credentials {
+ return &Credentials{Tokens: tokens}
}
func NewContext(ctx context.Context, c *Credentials) context.Context {
return logger
}
+// LogWriter returns an io.Writer that writes to the given log func,
+// which is typically (*check.C).Log().
+func LogWriter(log func(...interface{})) io.Writer {
+ return &logWriter{log}
+}
+
// SetLevel sets the current logging level. See logrus for level
// names.
func SetLevel(level string) {
sendErr(http.StatusUnauthorized, errUnauthorized)
return
}
- if req.URL.Path != "/_health/all" {
+ if req.URL.Path == "/_health/all" {
+ json.NewEncoder(resp).Encode(agg.ClusterHealth())
+ } else if req.URL.Path == "/_health/ping" {
+ resp.Write(healthyBody)
+ } else {
sendErr(http.StatusNotFound, errNotFound)
return
}
- json.NewEncoder(resp).Encode(agg.ClusterHealth())
if agg.Log != nil {
agg.Log(req, nil)
}
}
func (agg *Aggregator) ClusterHealth() ClusterHealthResponse {
+ agg.setupOnce.Do(agg.setup)
resp := ClusterHealthResponse{
Health: "OK",
Checks: make(map[string]CheckResult),
--- /dev/null
+/*
+ * Copyright (C) The Arvados Authors. All rights reserved.
+ *
+ * SPDX-License-Identifier: AGPL-3.0 OR Apache-2.0
+ *
+ */
+
+package org.arvados.client.api.client;
+
+import org.arvados.client.api.model.Link;
+import org.arvados.client.api.model.LinkList;
+import org.arvados.client.config.ConfigProvider;
+
+public class LinksApiClient extends BaseStandardApiClient<Link, LinkList> {
+
+ private static final String RESOURCE = "links";
+
+ public LinksApiClient(ConfigProvider config) {
+ super(config);
+ }
+
+ @Override
+ String getResource() {
+ return RESOURCE;
+ }
+
+ @Override
+ Class<Link> getType() {
+ return Link.class;
+ }
+
+ @Override
+ Class<LinkList> getListType() {
+ return LinkList.class;
+ }
+}
--- /dev/null
+/*
+ * Copyright (C) The Arvados Authors. All rights reserved.
+ *
+ * SPDX-License-Identifier: AGPL-3.0 OR Apache-2.0
+ *
+ */
+
+package org.arvados.client.api.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonPropertyOrder({ "name", "head_kind", "head_uuid", "link_class" })
+public class Link extends Item {
+
+ @JsonProperty("name")
+ private String name;
+ @JsonProperty("head_kind")
+ private String headKind;
+ @JsonProperty("head_uuid")
+ private String headUuid;
+ @JsonProperty("link_class")
+ private String linkClass;
+
+ public String getName() {
+ return name;
+ }
+
+ public String getHeadKind() {
+ return headKind;
+ }
+
+ public String getHeadUuid() {
+ return headUuid;
+ }
+
+ public String getLinkClass() {
+ return linkClass;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public void setHeadKind(String headKind) {
+ this.headKind = headKind;
+ }
+
+ public void setHeadUuid(String headUuid) {
+ this.headUuid = headUuid;
+ }
+
+ public void setLinkClass(String linkClass) {
+ this.linkClass = linkClass;
+ }
+
+}
--- /dev/null
+/*
+ * Copyright (C) The Arvados Authors. All rights reserved.
+ *
+ * SPDX-License-Identifier: AGPL-3.0 OR Apache-2.0
+ *
+ */
+
+package org.arvados.client.api.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+import java.util.List;
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonPropertyOrder({ "items" })
+public class LinkList extends ItemList {
+
+ @JsonProperty("items")
+ private List<Link> items;
+
+ public List<Link> getItems() {
+ return this.items;
+ }
+
+ public void setItems(List<Link> items) {
+ this.items = items;
+ }
+}
#
# By default, arv-copy recursively copies any dependent objects
# necessary to make the object functional in the new instance
-# (e.g. for a pipeline instance, arv-copy copies the pipeline
-# template, input collection, docker images, git repositories). If
+# (e.g. for a workflow, arv-copy copies the workflow,
+# input collections, and docker images). If
# --no-recursive is given, arv-copy copies only the single record
# identified by object-uuid.
#
copy_opts.add_argument(
'-f', '--force', dest='force', action='store_true',
help='Perform copy even if the object appears to exist at the remote destination.')
- copy_opts.add_argument(
- '--force-filters', action='store_true', default=False,
- help="Copy pipeline template filters verbatim, even if they act differently on the destination cluster.")
copy_opts.add_argument(
'--src', dest='source_arvados', required=True,
help='The name of the source Arvados instance (required) - points at an Arvados config file. May be either a pathname to a config file, or (for example) "foo" as shorthand for $HOME/.config/arvados/foo.conf.')
copy_opts.add_argument(
'--no-recursive', dest='recursive', action='store_false',
help='Do not copy any dependencies. NOTE: if this option is given, the copied object will need to be updated manually in order to be functional.')
- copy_opts.add_argument(
- '--dst-git-repo', dest='dst_git_repo',
- help='The name of the destination git repository. Required when copying a pipeline recursively.')
copy_opts.add_argument(
'--project-uuid', dest='project_uuid',
- help='The UUID of the project at the destination to which the pipeline should be copied.')
- copy_opts.add_argument(
- '--allow-git-http-src', action="store_true",
- help='Allow cloning git repositories over insecure http')
- copy_opts.add_argument(
- '--allow-git-http-dst', action="store_true",
- help='Allow pushing git repositories over insecure http')
+ help='The UUID of the project at the destination to which the collection or workflow should be copied.')
copy_opts.add_argument(
'object_uuid',
copy_opts.set_defaults(recursive=True)
parser = argparse.ArgumentParser(
- description='Copy a pipeline instance, template, workflow, or collection from one Arvados instance to another.',
+ description='Copy a workflow or collection from one Arvados instance to another.',
parents=[copy_opts, arv_cmd.retry_opt])
args = parser.parse_args()
result = copy_collection(args.object_uuid,
src_arv, dst_arv,
args)
- elif t == 'PipelineInstance':
- set_src_owner_uuid(src_arv.pipeline_instances(), args.object_uuid, args)
- result = copy_pipeline_instance(args.object_uuid,
- src_arv, dst_arv,
- args)
- elif t == 'PipelineTemplate':
- set_src_owner_uuid(src_arv.pipeline_templates(), args.object_uuid, args)
- result = copy_pipeline_template(args.object_uuid,
- src_arv, dst_arv, args)
elif t == 'Workflow':
set_src_owner_uuid(src_arv.workflows(), args.object_uuid, args)
result = copy_workflow(args.object_uuid, src_arv, dst_arv, args)
except Exception:
abort('git command is not available. Please ensure git is installed.')
-# copy_pipeline_instance(pi_uuid, src, dst, args)
-#
-# Copies a pipeline instance identified by pi_uuid from src to dst.
-#
-# If the args.recursive option is set:
-# 1. Copies all input collections
-# * For each component in the pipeline, include all collections
-# listed as job dependencies for that component)
-# 2. Copy docker images
-# 3. Copy git repositories
-# 4. Copy the pipeline template
-#
-# The only changes made to the copied pipeline instance are:
-# 1. The original pipeline instance UUID is preserved in
-# the 'properties' hash as 'copied_from_pipeline_instance_uuid'.
-# 2. The pipeline_template_uuid is changed to the new template uuid.
-# 3. The owner_uuid of the instance is changed to the user who
-# copied it.
-#
-def copy_pipeline_instance(pi_uuid, src, dst, args):
- # Fetch the pipeline instance record.
- pi = src.pipeline_instances().get(uuid=pi_uuid).execute(num_retries=args.retries)
-
- if args.recursive:
- check_git_availability()
-
- if not args.dst_git_repo:
- abort('--dst-git-repo is required when copying a pipeline recursively.')
- # Copy the pipeline template and save the copied template.
- if pi.get('pipeline_template_uuid', None):
- pt = copy_pipeline_template(pi['pipeline_template_uuid'],
- src, dst, args)
-
- # Copy input collections, docker images and git repos.
- pi = copy_collections(pi, src, dst, args)
- copy_git_repos(pi, src, dst, args.dst_git_repo, args)
- copy_docker_images(pi, src, dst, args)
-
- # Update the fields of the pipeline instance with the copied
- # pipeline template.
- if pi.get('pipeline_template_uuid', None):
- pi['pipeline_template_uuid'] = pt['uuid']
-
- else:
- # not recursive
- logger.info("Copying only pipeline instance %s.", pi_uuid)
- logger.info("You are responsible for making sure all pipeline dependencies have been updated.")
-
- # Update the pipeline instance properties, and create the new
- # instance at dst.
- pi['properties']['copied_from_pipeline_instance_uuid'] = pi_uuid
- pi['description'] = "Pipeline copied from {}\n\n{}".format(
- pi_uuid,
- pi['description'] if pi.get('description', None) else '')
-
- pi['owner_uuid'] = args.project_uuid
-
- del pi['uuid']
-
- new_pi = dst.pipeline_instances().create(body=pi, ensure_unique_name=True).execute(num_retries=args.retries)
- return new_pi
def filter_iter(arg):
"""Iterate a filter string-or-list.
except exc_types as error:
handler(error)
-def migrate_components_filters(template_components, dst_git_repo):
- """Update template component filters in-place for the destination.
-
- template_components is a dictionary of components in a pipeline template.
- This method walks over each component's filters, and updates them to have
- identical semantics on the destination cluster. It returns a list of
- error strings that describe what filters could not be updated safely.
-
- dst_git_repo is the name of the destination Git repository, which can
- be None if that is not known.
- """
- errors = []
- for cname, cspec in template_components.items():
- def add_error(errmsg):
- errors.append("{}: {}".format(cname, errmsg))
- if not isinstance(cspec, dict):
- add_error("value is not a component definition")
- continue
- src_repository = cspec.get('repository')
- filters = cspec.get('filters', [])
- if not isinstance(filters, list):
- add_error("filters are not a list")
- continue
- for cfilter in filters:
- if not (isinstance(cfilter, list) and (len(cfilter) == 3)):
- add_error("malformed filter {!r}".format(cfilter))
- continue
- if attr_filtered(cfilter, 'repository'):
- with exception_handler(add_error, ValueError):
- migrate_repository_filter(cfilter, src_repository, dst_git_repo)
- if attr_filtered(cfilter, 'script_version'):
- with exception_handler(add_error, ValueError):
- migrate_script_version_filter(cfilter)
- return errors
-
-# copy_pipeline_template(pt_uuid, src, dst, args)
-#
-# Copies a pipeline template identified by pt_uuid from src to dst.
-#
-# If args.recursive is True, also copy any collections, docker
-# images and git repositories that this template references.
-#
-# The owner_uuid of the new template is changed to that of the user
-# who copied the template.
-#
-# Returns the copied pipeline template object.
-#
-def copy_pipeline_template(pt_uuid, src, dst, args):
- # fetch the pipeline template from the source instance
- pt = src.pipeline_templates().get(uuid=pt_uuid).execute(num_retries=args.retries)
-
- if not args.force_filters:
- filter_errors = migrate_components_filters(pt['components'], args.dst_git_repo)
- if filter_errors:
- abort("Template filters cannot be copied safely. Use --force-filters to copy anyway.\n" +
- "\n".join(filter_errors))
-
- if args.recursive:
- check_git_availability()
-
- if not args.dst_git_repo:
- abort('--dst-git-repo is required when copying a pipeline recursively.')
- # Copy input collections, docker images and git repos.
- pt = copy_collections(pt, src, dst, args)
- copy_git_repos(pt, src, dst, args.dst_git_repo, args)
- copy_docker_images(pt, src, dst, args)
-
- pt['description'] = "Pipeline template copied from {}\n\n{}".format(
- pt_uuid,
- pt['description'] if pt.get('description', None) else '')
- pt['name'] = "{} copied from {}".format(pt.get('name', ''), pt_uuid)
- del pt['uuid']
-
- pt['owner_uuid'] = args.project_uuid
-
- return dst.pipeline_templates().create(body=pt, ensure_unique_name=True).execute(num_retries=args.retries)
# copy_workflow(wf_uuid, src, dst, args)
#
return type(obj)(copy_collections(v, src, dst, args) for v in obj)
return obj
-def migrate_jobspec(jobspec, src, dst, dst_repo, args):
- """Copy a job's script to the destination repository, and update its record.
-
- Given a jobspec dictionary, this function finds the referenced script from
- src and copies it to dst and dst_repo. It also updates jobspec in place to
- refer to names on the destination.
- """
- repo = jobspec.get('repository')
- if repo is None:
- return
- # script_version is the "script_version" parameter from the source
- # component or job. If no script_version was supplied in the
- # component or job, it is a mistake in the pipeline, but for the
- # purposes of copying the repository, default to "master".
- script_version = jobspec.get('script_version') or 'master'
- script_key = (repo, script_version)
- if script_key not in scripts_copied:
- copy_git_repo(repo, src, dst, dst_repo, script_version, args)
- scripts_copied.add(script_key)
- jobspec['repository'] = dst_repo
- repo_dir = local_repo_dir[repo]
- for version_key in ['script_version', 'supplied_script_version']:
- if version_key in jobspec:
- jobspec[version_key] = git_rev_parse(jobspec[version_key], repo_dir)
-
-# copy_git_repos(p, src, dst, dst_repo, args)
-#
-# Copies all git repositories referenced by pipeline instance or
-# template 'p' from src to dst.
-#
-# For each component c in the pipeline:
-# * Copy git repositories named in c['repository'] and c['job']['repository'] if present
-# * Rename script versions:
-# * c['script_version']
-# * c['job']['script_version']
-# * c['job']['supplied_script_version']
-# to the commit hashes they resolve to, since any symbolic
-# names (tags, branches) are not preserved in the destination repo.
-#
-# The pipeline object is updated in place with the new repository
-# names. The return value is undefined.
-#
-def copy_git_repos(p, src, dst, dst_repo, args):
- for component in p['components'].values():
- migrate_jobspec(component, src, dst, dst_repo, args)
- if 'job' in component:
- migrate_jobspec(component['job'], src, dst, dst_repo, args)
def total_collection_size(manifest_text):
"""Return the total number of bytes in this collection (excluding
available."""
collection_uuid = c['uuid']
- del c['uuid']
-
- if not c["name"]:
- c['name'] = "copied from " + collection_uuid
+ body = {}
+ for d in ('description', 'manifest_text', 'name', 'portable_data_hash', 'properties'):
+ body[d] = c[d]
- if 'properties' in c:
- del c['properties']
+ if not body["name"]:
+ body['name'] = "copied from " + collection_uuid
- c['owner_uuid'] = args.project_uuid
+ body['owner_uuid'] = args.project_uuid
- dst_collection = dst.collections().create(body=c, ensure_unique_name=True).execute(num_retries=args.retries)
+ dst_collection = dst.collections().create(body=body, ensure_unique_name=True).execute(num_retries=args.retries)
# Create docker_image_repo+tag and docker_image_hash links
# at the destination.
c = items[0]
if not c:
# See if there is a collection that's in the same project
- # as the root item (usually a pipeline) being copied.
+ # as the root item (usually a workflow) being copied.
for i in items:
if i.get("owner_uuid") == src_owner_uuid and i.get("name"):
c = i
return (git_url, git_config)
-# copy_git_repo(src_git_repo, src, dst, dst_git_repo, script_version, args)
-#
-# Copies commits from git repository 'src_git_repo' on Arvados
-# instance 'src' to 'dst_git_repo' on 'dst'. Both src_git_repo
-# and dst_git_repo are repository names, not UUIDs (i.e. "arvados"
-# or "jsmith")
-#
-# All commits will be copied to a destination branch named for the
-# source repository URL.
-#
-# The destination repository must already exist.
-#
-# The user running this command must be authenticated
-# to both repositories.
-#
-def copy_git_repo(src_git_repo, src, dst, dst_git_repo, script_version, args):
- # Identify the fetch and push URLs for the git repositories.
-
- (src_git_url, src_git_config) = select_git_url(src, src_git_repo, args.retries, args.allow_git_http_src, "--allow-git-http-src")
- (dst_git_url, dst_git_config) = select_git_url(dst, dst_git_repo, args.retries, args.allow_git_http_dst, "--allow-git-http-dst")
-
- logger.debug('src_git_url: {}'.format(src_git_url))
- logger.debug('dst_git_url: {}'.format(dst_git_url))
-
- dst_branch = re.sub(r'\W+', '_', "{}_{}".format(src_git_url, script_version))
-
- # Copy git commits from src repo to dst repo.
- if src_git_repo not in local_repo_dir:
- local_repo_dir[src_git_repo] = tempfile.mkdtemp()
- arvados.util.run_command(
- ["git"] + src_git_config + ["clone", "--bare", src_git_url,
- local_repo_dir[src_git_repo]],
- cwd=os.path.dirname(local_repo_dir[src_git_repo]),
- env={"HOME": os.environ["HOME"],
- "ARVADOS_API_TOKEN": src.api_token,
- "GIT_ASKPASS": "/bin/false"})
- arvados.util.run_command(
- ["git", "remote", "add", "dst", dst_git_url],
- cwd=local_repo_dir[src_git_repo])
- arvados.util.run_command(
- ["git", "branch", dst_branch, script_version],
- cwd=local_repo_dir[src_git_repo])
- arvados.util.run_command(["git"] + dst_git_config + ["push", "dst", dst_branch],
- cwd=local_repo_dir[src_git_repo],
- env={"HOME": os.environ["HOME"],
- "ARVADOS_API_TOKEN": dst.api_token,
- "GIT_ASKPASS": "/bin/false"})
-
-def copy_docker_images(pipeline, src, dst, args):
- """Copy any docker images named in the pipeline components'
- runtime_constraints field from src to dst."""
-
- logger.debug('copy_docker_images: {}'.format(pipeline['uuid']))
- for c_name, c_info in pipeline['components'].items():
- if ('runtime_constraints' in c_info and
- 'docker_image' in c_info['runtime_constraints']):
- copy_docker_image(
- c_info['runtime_constraints']['docker_image'],
- c_info['runtime_constraints'].get('docker_image_tag', 'latest'),
- src, dst, args)
-
-
def copy_docker_image(docker_image, docker_image_tag, src, dst, args):
"""Copy the docker image identified by docker_image and
docker_image_tag from src to dst. Create appropriate
# the second field of the uuid. This function consults the api's
# schema to identify the object class.
#
-# It returns a string such as 'Collection', 'PipelineInstance', etc.
+# It returns a string such as 'Collection', 'Workflow', etc.
#
# Special case: if handed a Keep locator hash, return 'Collection'.
#
try:
image_hash = find_one_image_hash(args.image, args.tag)
except DockerError as error:
- logger.error(error.message)
+ logger.error(str(error))
sys.exit(1)
if not docker_image_compatible(api, image_hash):
return rid
def get_config_once(svc):
- if not svc._rootDesc.get('resources')['configs']:
+ if not svc._rootDesc.get('resources').get('configs', False):
# Old API server version, no config export endpoint
return {}
if not hasattr(svc, '_cached_config'):
'google-api-python-client >=1.6.2, <1.7',
'httplib2 >=0.9.2',
'pycurl >=7.19.5.1',
- 'ruamel.yaml >=0.15.54, <=0.15.77',
+ 'ruamel.yaml >=0.15.54, <=0.16.5',
'setuptools',
'ws4py >=0.4.2',
],
uwsgi_temp_path "{{TMPDIR}}";
scgi_temp_path "{{TMPDIR}}";
upstream arv-git-http {
- server localhost:{{GITPORT}};
+ server {{LISTENHOST}}:{{GITPORT}};
}
server {
- listen *:{{GITSSLPORT}} ssl default_server;
+ listen {{LISTENHOST}}:{{GITSSLPORT}} ssl default_server;
server_name arv-git-http;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
}
}
upstream keepproxy {
- server localhost:{{KEEPPROXYPORT}};
+ server {{LISTENHOST}}:{{KEEPPROXYPORT}};
}
server {
- listen *:{{KEEPPROXYSSLPORT}} ssl default_server;
+ listen {{LISTENHOST}}:{{KEEPPROXYSSLPORT}} ssl default_server;
server_name keepproxy;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
}
}
upstream keep-web {
- server localhost:{{KEEPWEBPORT}};
+ server {{LISTENHOST}}:{{KEEPWEBPORT}};
}
server {
- listen *:{{KEEPWEBSSLPORT}} ssl default_server;
+ listen {{LISTENHOST}}:{{KEEPWEBSSLPORT}} ssl default_server;
server_name keep-web;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
}
}
server {
- listen *:{{KEEPWEBDLSSLPORT}} ssl default_server;
+ listen {{LISTENHOST}}:{{KEEPWEBDLSSLPORT}} ssl default_server;
server_name keep-web-dl ~.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
}
}
upstream ws {
- server localhost:{{WSPORT}};
+ server {{LISTENHOST}}:{{WSPORT}};
}
server {
- listen *:{{WSSPORT}} ssl default_server;
+ listen {{LISTENHOST}}:{{WSSSLPORT}} ssl default_server;
server_name websocket;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
proxy_redirect off;
}
}
+ upstream workbench1 {
+ server {{LISTENHOST}}:{{WORKBENCH1PORT}};
+ }
+ server {
+ listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl default_server;
+ server_name workbench1;
+ ssl_certificate "{{SSLCERT}}";
+ ssl_certificate_key "{{SSLKEY}}";
+ location / {
+ proxy_pass http://workbench1;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
+ }
+ }
upstream controller {
- server localhost:{{CONTROLLERPORT}};
+ server {{LISTENHOST}}:{{CONTROLLERPORT}};
}
server {
- listen *:{{CONTROLLERSSLPORT}} ssl default_server;
+ listen {{LISTENHOST}}:{{CONTROLLERSSLPORT}} ssl default_server;
server_name controller;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
port = internal_port_from_config("RailsAPI")
env = os.environ.copy()
env['RAILS_ENV'] = 'test'
+ env['ARVADOS_RAILS_LOG_TO_STDOUT'] = '1'
env.pop('ARVADOS_WEBSOCKETS', None)
env.pop('ARVADOS_TEST_API_HOST', None)
env.pop('ARVADOS_API_HOST', None)
env.pop('ARVADOS_API_HOST_INSECURE', None)
env.pop('ARVADOS_API_TOKEN', None)
- start_msg = subprocess.check_output(
+ logf = open(_logfilename('railsapi'), 'a')
+ railsapi = subprocess.Popen(
['bundle', 'exec',
- 'passenger', 'start', '-d', '-p{}'.format(port),
+ 'passenger', 'start', '-p{}'.format(port),
'--pid-file', pid_file,
- '--log-file', os.path.join(os.getcwd(), 'log/test.log'),
+ '--log-file', '/dev/stdout',
'--ssl',
'--ssl-certificate', 'tmp/self-signed.pem',
'--ssl-certificate-key', 'tmp/self-signed.key'],
- env=env)
+ env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
if not leave_running_atexit:
atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
- match = re.search(r'Accessible via: https://(.*?)/', start_msg)
- if not match:
- raise Exception(
- "Passenger did not report endpoint: {}".format(start_msg))
- my_api_host = match.group(1)
+ my_api_host = "127.0.0.1:"+str(port)
os.environ['ARVADOS_API_HOST'] = my_api_host
# Make sure the server has written its pid file and started
# listening on its TCP port
- find_server_pid(pid_file)
_wait_until_port_listens(port)
+ find_server_pid(pid_file)
reset()
os.chdir(restore_cwd)
return
stop_nginx()
nginxconf = {}
+ nginxconf['LISTENHOST'] = 'localhost'
nginxconf['CONTROLLERPORT'] = internal_port_from_config("Controller")
nginxconf['CONTROLLERSSLPORT'] = external_port_from_config("Controller")
nginxconf['KEEPWEBPORT'] = internal_port_from_config("WebDAV")
nginxconf['GITPORT'] = internal_port_from_config("GitHTTP")
nginxconf['GITSSLPORT'] = external_port_from_config("GitHTTP")
nginxconf['WSPORT'] = internal_port_from_config("Websocket")
- nginxconf['WSSPORT'] = external_port_from_config("Websocket")
+ nginxconf['WSSSLPORT'] = external_port_from_config("Websocket")
+ nginxconf['WORKBENCH1PORT'] = internal_port_from_config("Workbench1")
+ nginxconf['WORKBENCH1SSLPORT'] = external_port_from_config("Workbench1")
nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
with open(conffile, 'w') as f:
f.write(re.sub(
- r'{{([A-Z]+)}}',
+ r'{{([A-Z]+[A-Z0-9]+)}}',
lambda match: str(nginxconf.get(match.group(1))),
open(conftemplatefile).read()))
controller_external_port = find_available_port()
websocket_port = find_available_port()
websocket_external_port = find_available_port()
+ workbench1_port = find_available_port()
+ workbench1_external_port = find_available_port()
git_httpd_port = find_available_port()
git_httpd_external_port = find_available_port()
keepproxy_port = find_available_port()
"http://%s:%s"%(localhost, websocket_port): {},
},
},
+ "Workbench1": {
+ "ExternalURL": "https://%s:%s/" % (localhost, workbench1_external_port),
+ "InternalURLs": {
+ "http://%s:%s"%(localhost, workbench1_port): {},
+ },
+ },
"GitHTTP": {
"ExternalURL": "https://%s:%s" % (localhost, git_httpd_external_port),
"InternalURLs": {
"http://%s:%s"%(localhost, keep_web_dl_port): {},
},
},
+ "SSO": {
+ "ExternalURL": "http://localhost:3002",
+ },
}
config = {
"SystemRootToken": auth_token('system_user'),
"API": {
"RequestTimeout": "30s",
+ "RailsSessionSecretToken": "e24205c490ac07e028fd5f8a692dcb398bcd654eff1aef5f9fe6891994b18483",
+ },
+ "Login": {
+ "ProviderAppID": "arvados-server",
+ "ProviderAppSecret": "608dbf356a327e2d0d4932b60161e212c2d8d8f5e25690d7b622f850a990cd33",
},
"SystemLogs": {
"LogLevel": ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
"Services": services,
"Users": {
"AnonymousUserToken": auth_token('anonymous'),
+ "UserProfileNotificationAddress": "arvados@example.com",
},
"Collections": {
"BlobSigningKey": "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc",
- "TrustAllContent": True,
+ "TrustAllContent": False,
"ForwardSlashNameSubstitution": "/",
+ "TrashSweepInterval": "-1s",
},
"Git": {
- "Repositories": "%s/test" % os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git'),
+ "Repositories": os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git', 'test'),
+ },
+ "Containers": {
+ "JobsAPI": {
+ "GitInternalDir": os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'internal.git'),
+ },
+ "SupportedDockerImageFormats": {"v1": {}},
},
"Volumes": {
"zzzzz-nyw5e-%015d"%n: {
ENV["GIT_DIR"] = File.expand_path "#{__dir__}/../../.git"
ENV["GIT_WORK_TREE"] = File.expand_path "#{__dir__}/../.."
git_timestamp, git_hash = `git log -n1 --first-parent --format=%ct:%H #{__dir__}`.chomp.split(":")
- version = `#{__dir__}/../../build/version-at-commit.sh #{git_hash}`.encode('utf-8').strip
+ if ENV["ARVADOS_BUILDING_VERSION"]
+ version = ENV["ARVADOS_BUILDING_VERSION"]
+ else
+ version = `#{__dir__}/../../build/version-at-commit.sh #{git_hash}`.encode('utf-8').strip
+ end
git_timestamp = Time.at(git_timestamp.to_i).utc
ensure
ENV["GIT_DIR"] = git_dir
method_source
rake (>= 0.8.7)
thor (>= 0.18.1, < 2.0)
- rake (12.3.2)
+ rake (13.0.1)
rb-fsevent (0.10.3)
rb-inotify (0.9.10)
ffi (>= 0.5.0, < 2)
end
def activate
+ if params[:id] and params[:id].match(/\D/)
+ params[:uuid] = params.delete :id
+ end
if current_user.andand.is_admin && params[:uuid]
- @object = User.find params[:uuid]
+ @object = User.find_by_uuid params[:uuid]
else
@object = current_user
end
begin
user = User.register(authinfo)
rescue => e
- Rails.logger.warn e
+ Rails.logger.warn "User.register error #{e}"
+ Rails.logger.warn "authinfo was #{authinfo.inspect}"
return redirect_to login_failure_url
end
def self.check_system_root_token token
if token == Rails.configuration.SystemRootToken
return ApiClientAuthorization.new(user: User.find_by_uuid(system_user_uuid),
- uuid: uuid_prefix+"-gj3su-000000000000000",
+ uuid: Rails.configuration.ClusterID+"-gj3su-000000000000000",
api_token: token,
api_client: ApiClient.new(is_trusted: true, url_prefix: ""))
else
user = self
redirects = 0
while (uuid = user.redirect_to_user_uuid)
- user = User.unscoped.find_by_uuid(uuid)
- if !user
- raise Exception.new("user uuid #{user.uuid} redirects to nonexistent uuid #{uuid}")
+ break if uuid.empty?
+ nextuser = User.unscoped.find_by_uuid(uuid)
+ if !nextuser
+ raise Exception.new("user uuid #{user.uuid} redirects to nonexistent uuid '#{uuid}'")
end
+ user = nextuser
redirects += 1
if redirects > 15
raise "Starting from #{self.uuid} redirect_to_user_uuid exceeded maximum number of redirects"
action_controller.allow_forgery_protection: false
action_mailer.delivery_method: :test
active_support.deprecation: :stderr
- uuid_prefix: zzzzz
- sso_app_id: arvados-server
- sso_app_secret: <%= rand(2**512).to_s(36) %>
- sso_provider_url: http://localhost:3002
- secret_token: <%= rand(2**512).to_s(36) %>
- blob_signing_key: zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc
- user_profile_notification_address: arvados@example.com
- workbench_address: https://localhost:3001/
- git_repositories_dir: <%= Rails.root.join 'tmp', 'git', 'test' %>
- git_internal_dir: <%= Rails.root.join 'tmp', 'internal.git' %>
- trash_sweep_interval: -1
- docker_image_formats: ["v1"]
end
end
+if ENV["ARVADOS_RAILS_LOG_TO_STDOUT"]
+ Rails.logger = ActiveSupport::TaggedLogging.new(Logger.new(STDOUT))
+end
+
module Server
class Application < Rails::Application
# The following is to avoid SafeYAML's warning message
db_config = {}
path = "#{::Rails.root.to_s}/config/database.yml"
-if File.exist? path
+if !ENV['ARVADOS_CONFIG_NOLEGACY'] && File.exist?(path)
db_config = ConfigLoader.load(path, erb: true)
end
end
end
- def self.parse_duration durstr, cfgkey:
- duration_re = /-?(\d+(\.\d+)?)(s|m|h)/
+ def self.parse_duration(durstr, cfgkey:)
+ sign = 1
+ if durstr[0] == '-'
+ durstr = durstr[1..-1]
+ sign = -1
+ end
+ duration_re = /(\d+(\.\d+)?)(s|m|h)/
dursec = 0
while durstr != ""
mt = duration_re.match durstr
raise "#{cfgkey} not a valid duration: '#{durstr}', accepted suffixes are s, m, h"
end
multiplier = {s: 1, m: 60, h: 3600}
- dursec += (Float(mt[1]) * multiplier[mt[3].to_sym])
+ dursec += (Float(mt[1]) * multiplier[mt[3].to_sym] * sign)
durstr = durstr[mt[0].length..-1]
end
return dursec.seconds
end
- test "cannot set is_activate to false directly" do
+ test "cannot set is_active to false directly" do
post('/arvados/v1/users',
params: {
user: {
user = json_response
assert_equal false, user['is_active']
+ token = act_as_system_user do
+ ApiClientAuthorization.create!(user: User.find_by_uuid(user['uuid']), api_client: ApiClient.all.first).api_token
+ end
+ post("/arvados/v1/user_agreements/sign",
+ params: {uuid: 'zzzzz-4zz18-t68oksiu9m80s4y'},
+ headers: {"HTTP_AUTHORIZATION" => "Bearer #{token}"})
+ assert_response :success
+
post("/arvados/v1/users/#{user['uuid']}/activate",
params: {},
headers: auth(:admin))
signal.Notify(term, syscall.SIGINT)
// Start serving requests.
- router = MakeRESTRouter(kc, time.Duration(cluster.API.KeepServiceRequestTimeout), cluster.SystemRootToken)
+ router = MakeRESTRouter(kc, time.Duration(cluster.API.KeepServiceRequestTimeout), cluster.ManagementToken)
return http.Serve(listener, httpserver.AddRequestIDs(httpserver.LogRequests(router)))
}
*.gem
+Gemfile.lock
\ No newline at end of file
+++ /dev/null
-PATH
- remote: .
- specs:
- arvados-login-sync (1.5.0.dev20200114213539)
- arvados (~> 1.3.0, >= 1.3.0)
- faraday (< 0.16)
- signet (< 0.12)
-
-GEM
- remote: https://rubygems.org/
- specs:
- activesupport (5.0.7.2)
- concurrent-ruby (~> 1.0, >= 1.0.2)
- i18n (>= 0.7, < 2)
- minitest (~> 5.1)
- tzinfo (~> 1.1)
- addressable (2.7.0)
- public_suffix (>= 2.0.2, < 5.0)
- andand (1.3.3)
- arvados (1.3.3.20190320201707)
- activesupport (>= 3)
- andand (~> 1.3, >= 1.3.3)
- arvados-google-api-client (>= 0.7, < 0.8.9)
- i18n (~> 0)
- json (>= 1.7.7, < 3)
- jwt (>= 0.1.5, < 2)
- arvados-google-api-client (0.8.7.3)
- activesupport (>= 3.2, < 5.1)
- addressable (~> 2.3)
- autoparse (~> 0.3)
- extlib (~> 0.9)
- faraday (~> 0.9)
- googleauth (~> 0.3)
- launchy (~> 2.4)
- multi_json (~> 1.10)
- retriable (~> 1.4)
- signet (~> 0.6)
- autoparse (0.3.3)
- addressable (>= 2.3.1)
- extlib (>= 0.9.15)
- multi_json (>= 1.0.0)
- concurrent-ruby (1.1.5)
- extlib (0.9.16)
- faraday (0.15.4)
- multipart-post (>= 1.2, < 3)
- googleauth (0.9.0)
- faraday (~> 0.12)
- jwt (>= 1.4, < 3.0)
- memoist (~> 0.16)
- multi_json (~> 1.11)
- os (>= 0.9, < 2.0)
- signet (~> 0.7)
- i18n (0.9.5)
- concurrent-ruby (~> 1.0)
- json (2.3.0)
- jwt (1.5.6)
- launchy (2.4.3)
- addressable (~> 2.3)
- memoist (0.16.2)
- metaclass (0.0.4)
- minitest (5.11.3)
- mocha (1.8.0)
- metaclass (~> 0.0.1)
- multi_json (1.14.1)
- multipart-post (2.1.1)
- os (1.0.1)
- public_suffix (4.0.3)
- rake (12.3.2)
- retriable (1.4.1)
- signet (0.11.0)
- addressable (~> 2.3)
- faraday (~> 0.9)
- jwt (>= 1.5, < 3.0)
- multi_json (~> 1.10)
- thread_safe (0.3.6)
- tzinfo (1.2.6)
- thread_safe (~> 0.1)
-
-PLATFORMS
- ruby
-
-DEPENDENCIES
- arvados-login-sync!
- minitest (>= 5.0.0)
- mocha (>= 1.5.0)
- rake
-
-BUNDLED WITH
- 1.11
ENV["GIT_DIR"] = File.expand_path "#{__dir__}/../../.git"
ENV["GIT_WORK_TREE"] = File.expand_path "#{__dir__}/../.."
git_timestamp, git_hash = `git log -n1 --first-parent --format=%ct:%H #{__dir__}`.chomp.split(":")
- version = `#{__dir__}/../../build/version-at-commit.sh #{git_hash}`.encode('utf-8').strip
+ if ENV["ARVADOS_BUILDING_VERSION"]
+ version = ENV["ARVADOS_BUILDING_VERSION"]
+ else
+ version = `#{__dir__}/../../build/version-at-commit.sh #{git_hash}`.encode('utf-8').strip
+ end
git_timestamp = Time.at(git_timestamp.to_i).utc
ensure
ENV["GIT_DIR"] = git_dir
s.executables << "arvados-login-sync"
s.required_ruby_version = '>= 2.1.0'
s.add_runtime_dependency 'arvados', '~> 1.3.0', '>= 1.3.0'
+ s.add_runtime_dependency 'launchy', '< 2.5'
# arvados-google-api-client 0.8.7.2 is incompatible with faraday 0.16.2
s.add_dependency('faraday', '< 0.16')
# arvados-google-api-client (and thus arvados) gems
import os
import re
+SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+
+def choose_version_from():
+ sdk_ts = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', os.path.join(SETUP_DIR, "../../sdk/python")]).strip()
+ cwl_ts = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', SETUP_DIR]).strip()
+ if int(sdk_ts) > int(cwl_ts):
+ getver = os.path.join(SETUP_DIR, "../../sdk/python")
+ else:
+ getver = SETUP_DIR
+ return getver
+
def git_version_at_commit():
- curdir = os.path.dirname(os.path.abspath(__file__))
+ curdir = choose_version_from()
myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
'--format=%H', curdir]).strip()
myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
import (
"context"
"database/sql"
+ "errors"
+ "fmt"
"strconv"
"sync"
"sync/atomic"
case <-ticker.C:
logger(nil).Debug("listener ping")
- ps.pqListener.Ping()
+ err := ps.pqListener.Ping()
+ if err != nil {
+ ps.listenerProblem(-1, fmt.Errorf("pqListener ping failed: %s", err))
+ continue
+ }
case pqEvent, ok := <-ps.pqListener.Notify:
if !ok {
- logger(nil).Debug("pqListener Notify chan closed")
+ logger(nil).Error("pqListener Notify chan closed")
return
}
if pqEvent == nil {
// itself in addition to sending us a
// nil event, so this might be
// superfluous:
- ps.listenerProblem(-1, nil)
+ ps.listenerProblem(-1, errors.New("pqListener Notify chan received nil event"))
continue
}
if pqEvent.Channel != "logs" {
func (srv *server) Close() {
srv.WaitReady()
srv.eventSource.Close()
+ srv.httpServer.Close()
srv.listener.Close()
}
popd
if [ "$PYCMD" = "python3" ]; then
- if ! pip3 install --no-index --find-links /var/lib/pip $1 ; then
- pip3 install $1
+ if ! pip3 install --prefix /usr/local --no-index --find-links /var/lib/pip $1 ; then
+ pip3 install --prefix /usr/local $1
fi
else
if ! pip install --no-index --find-links /var/lib/pip $1 ; then
fi
fi
+geo_dockerip=
+if [[ -f /var/run/localip_override ]] ; then
+ geo_dockerip="$dockerip/32 0;"
+fi
+
openssl verify -CAfile $root_cert $server_cert
cat <<EOF >/var/lib/arvados/nginx.conf
default 1;
127.0.0.0/8 0;
$containerip/32 0;
- $dockerip/32 0;
+ $geo_dockerip
}
server {
run_bundler --binstubs=$PWD/binstubs
ln -sf /usr/src/arvados/sdk/cli/binstubs/arv /usr/local/bin/arv
+export PYCMD=python3
+
# Need to install the upstream version of pip because the python-pip package
# shipped with Debian 9 is patched to change behavior in a way that breaks our
# use case.
# multiple packages, because it will blindly install the latest version of each
# dependency requested by each package, even if a compatible package version is
# already installed.
-pip_install pip==9.0.3
+if ! pip3 install --no-index --find-links /var/lib/pip pip==9.0.3 ; then
+ pip3 install pip==9.0.3
+fi
pip_install wheel
import os
import re
+SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+
+def choose_version_from():
+ sdk_ts = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', os.path.join(SETUP_DIR, "../../sdk/python")]).strip()
+ cwl_ts = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', SETUP_DIR]).strip()
+ if int(sdk_ts) > int(cwl_ts):
+ getver = os.path.join(SETUP_DIR, "../../sdk/python")
+ else:
+ getver = SETUP_DIR
+ return getver
+
def git_version_at_commit():
- curdir = os.path.dirname(os.path.abspath(__file__))
+ curdir = choose_version_from()
myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
'--format=%H', curdir]).strip()
myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
projects / arvados.git / commitdiff