source 'https://rubygems.org'
gem 'rails', '~> 5.2.0'
-gem 'arvados', git: 'https://github.com/arvados/arvados.git', glob: 'sdk/ruby/arvados.gemspec'
+gem 'arvados', '~> 2.1.5'
gem 'activerecord-nulldb-adapter', git: 'https://github.com/arvados/nulldb'
gem 'multi_json'
-GIT
- remote: https://github.com/arvados/arvados.git
- revision: c210114aa8c77ba0bb8e4d487fc1507b40f9560f
- glob: sdk/ruby/arvados.gemspec
- specs:
- arvados (1.5.0.pre20200114202620)
- activesupport (>= 3)
- andand (~> 1.3, >= 1.3.3)
- arvados-google-api-client (>= 0.7, < 0.8.9)
- faraday (< 0.16)
- i18n (~> 0)
- json (>= 1.7.7, < 3)
- jwt (>= 0.1.5, < 2)
-
GIT
remote: https://github.com/arvados/nulldb
revision: d8e0073b665acdd2537c5eb15178a60f02f4b413
remote: https://rubygems.org/
specs:
RedCloth (4.3.2)
- actioncable (5.2.4.5)
- actionpack (= 5.2.4.5)
+ actioncable (5.2.6)
+ actionpack (= 5.2.6)
nio4r (~> 2.0)
websocket-driver (>= 0.6.1)
- actionmailer (5.2.4.5)
- actionpack (= 5.2.4.5)
- actionview (= 5.2.4.5)
- activejob (= 5.2.4.5)
+ actionmailer (5.2.6)
+ actionpack (= 5.2.6)
+ actionview (= 5.2.6)
+ activejob (= 5.2.6)
mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 2.0)
- actionpack (5.2.4.5)
- actionview (= 5.2.4.5)
- activesupport (= 5.2.4.5)
+ actionpack (5.2.6)
+ actionview (= 5.2.6)
+ activesupport (= 5.2.6)
rack (~> 2.0, >= 2.0.8)
rack-test (>= 0.6.3)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.2)
- actionview (5.2.4.5)
- activesupport (= 5.2.4.5)
+ actionview (5.2.6)
+ activesupport (= 5.2.6)
builder (~> 3.1)
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.3)
- activejob (5.2.4.5)
- activesupport (= 5.2.4.5)
+ activejob (5.2.6)
+ activesupport (= 5.2.6)
globalid (>= 0.3.6)
- activemodel (5.2.4.5)
- activesupport (= 5.2.4.5)
- activerecord (5.2.4.5)
- activemodel (= 5.2.4.5)
- activesupport (= 5.2.4.5)
+ activemodel (5.2.6)
+ activesupport (= 5.2.6)
+ activerecord (5.2.6)
+ activemodel (= 5.2.6)
+ activesupport (= 5.2.6)
arel (>= 9.0)
- activestorage (5.2.4.5)
- actionpack (= 5.2.4.5)
- activerecord (= 5.2.4.5)
- marcel (~> 0.3.1)
- activesupport (5.2.4.5)
+ activestorage (5.2.6)
+ actionpack (= 5.2.6)
+ activerecord (= 5.2.6)
+ marcel (~> 1.0.0)
+ activesupport (5.2.6)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 0.7, < 2)
minitest (~> 5.1)
andand (1.3.3)
angularjs-rails (1.3.15)
arel (9.0.0)
+ arvados (2.1.5)
+ activesupport (>= 3)
+ andand (~> 1.3, >= 1.3.3)
+ arvados-google-api-client (>= 0.7, < 0.8.9)
+ faraday (< 0.16)
+ i18n (~> 0)
+ json (>= 1.7.7, < 3)
+ jwt (>= 0.1.5, < 2)
arvados-google-api-client (0.8.7.4)
activesupport (>= 3.2, < 5.3)
addressable (~> 2.3)
execjs
coffee-script-source (1.12.2)
commonjs (0.2.7)
- concurrent-ruby (1.1.8)
+ concurrent-ruby (1.1.9)
crass (1.0.6)
deep_merge (1.2.1)
docile (1.3.1)
rails-dom-testing (>= 1, < 3)
railties (>= 4.2.0)
thor (>= 0.14, < 2.0)
- json (2.3.0)
+ json (2.5.1)
jwt (1.5.6)
launchy (2.4.3)
addressable (~> 2.3)
railties (>= 4)
request_store (~> 1.0)
logstash-event (1.2.02)
- loofah (2.9.0)
+ loofah (2.10.0)
crass (~> 1.0.2)
nokogiri (>= 1.5.9)
mail (2.7.1)
mini_mime (>= 0.1.1)
- marcel (0.3.3)
- mimemagic (~> 0.3.2)
+ marcel (1.0.1)
memoist (0.16.2)
metaclass (0.0.4)
method_source (1.0.0)
mime-types (3.2.2)
mime-types-data (~> 3.2015)
mime-types-data (3.2019.0331)
- mimemagic (0.3.8)
- nokogiri (~> 1)
- mini_mime (1.0.2)
- mini_portile2 (2.5.0)
+ mini_mime (1.1.0)
+ mini_portile2 (2.5.3)
minitest (5.10.3)
mocha (1.8.0)
metaclass (~> 0.0.1)
net-ssh-gateway (2.0.0)
net-ssh (>= 4.0.0)
nio4r (2.5.7)
- nokogiri (1.11.2)
+ nokogiri (1.11.7)
mini_portile2 (~> 2.5.0)
racc (~> 1.4)
npm-rails (0.2.1)
cliver (~> 0.3.1)
multi_json (~> 1.0)
websocket-driver (>= 0.2.0)
- public_suffix (4.0.5)
+ public_suffix (4.0.6)
racc (1.5.2)
rack (2.2.3)
rack-mini-profiler (1.0.2)
rack (>= 1.2.0)
rack-test (1.1.0)
rack (>= 1.0, < 3)
- rails (5.2.4.5)
- actioncable (= 5.2.4.5)
- actionmailer (= 5.2.4.5)
- actionpack (= 5.2.4.5)
- actionview (= 5.2.4.5)
- activejob (= 5.2.4.5)
- activemodel (= 5.2.4.5)
- activerecord (= 5.2.4.5)
- activestorage (= 5.2.4.5)
- activesupport (= 5.2.4.5)
+ rails (5.2.6)
+ actioncable (= 5.2.6)
+ actionmailer (= 5.2.6)
+ actionpack (= 5.2.6)
+ actionview (= 5.2.6)
+ activejob (= 5.2.6)
+ activemodel (= 5.2.6)
+ activerecord (= 5.2.6)
+ activestorage (= 5.2.6)
+ activesupport (= 5.2.6)
bundler (>= 1.3.0)
- railties (= 5.2.4.5)
+ railties (= 5.2.6)
sprockets-rails (>= 2.0.0)
rails-controller-testing (1.0.4)
actionpack (>= 5.0.1.x)
rails-html-sanitizer (1.3.0)
loofah (~> 2.3)
rails-perftest (0.0.7)
- railties (5.2.4.5)
- actionpack (= 5.2.4.5)
- activesupport (= 5.2.4.5)
+ railties (5.2.6)
+ actionpack (= 5.2.6)
+ activesupport (= 5.2.6)
method_source
rake (>= 0.8.7)
thor (>= 0.19.0, < 2.0)
uglifier (2.7.2)
execjs (>= 0.3.0)
json (>= 1.8.0)
- websocket-driver (0.7.3)
+ websocket-driver (0.7.4)
websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5)
xpath (2.1.0)
activerecord-nulldb-adapter!
andand
angularjs-rails (~> 1.3.8)
- arvados!
+ arvados (~> 2.1.5)
bootsnap
bootstrap-sass (~> 3.4.1)
bootstrap-tab-history-rails
api_params[:filters] = @filters if @filters
api_params[:distinct] = @distinct if @distinct
api_params[:include_trash] = @include_trash if @include_trash
+ api_params[:cluster_id] = Rails.configuration.ClusterID
if @fetch_multiple_pages
# Default limit to (effectively) api server's MAX_LIMIT
api_params[:limit] = 2**(0.size*8 - 1) - 1
<% content_for :breadcrumbs do raw '<!-- -->' end %>
-<div class="row">
- <div class="col-sm-8 col-sm-push-4" style="margin-top: 1em">
- <div class="well clearfix">
- <%= image_tag "dax.png", style: "width: 112px; height: 150px; margin-right: 2em", class: 'pull-left' %>
-
- <h3 style="margin-top:0">Please log in.</h3>
-
- <p>
+<%= javascript_tag do %>
+ function controller_password_authenticate(event) {
+ event.preventDefault()
+ document.getElementById('login-authenticate-error').innerHTML = '';
+ fetch('<%= "#{Rails.configuration.Services.Controller.ExternalURL}" %>arvados/v1/users/authenticate', {
+ method: 'POST',
- The "Log in" button below will show you a Google sign-in page.
- After you assure Google that you want to log in here with your
- Google account, you will be redirected back here to
- <%= Rails.configuration.Workbench.SiteName %>.
+ headers: {'Content-Type': 'application/json'},
+ body: JSON.stringify({
+ username: document.getElementById('login-username').value,
+ password: document.getElementById('login-password').value,
+ }),
+ }).then(function(resp) {
+ if (!resp.ok) {
+ resp.json().then(function(respj) {
+ document.getElementById('login-authenticate-error').innerHTML = "<p>"+respj.errors[0]+"</p>";
+ });
+ return;
+ }
- </p><p>
+ var redir = document.getElementById('login-return-to').value
+ if (redir.indexOf('?') > 0) {
+ redir += '&'
+ } else {
+ redir += '?'
+ }
+ resp.json().then(function(respj) {
+ document.location = redir + "api_token=v2/" + respj.uuid + "/" + respj.api_token;
+ });
+ });
+ }
+ function clear_authenticate_error() {
+ document.getElementById('login-authenticate-error').innerHTML = "";
+ }
+<% end %>
- If you have never used <%= Rails.configuration.Workbench.SiteName %>
- before, logging in for the first time will automatically
- create a new account.
-
- </p><p>
+<div class="row">
+ <div class="col-sm-8 col-sm-push-4" style="margin-top: 1em">
+ <div class="well clearfix">
- <i><%= Rails.configuration.Workbench.SiteName %> uses your name and
- email address only for identification, and does not retrieve
- any other personal information from Google.</i>
+ <%= raw(Rails.configuration.Workbench.WelcomePageHTML) %>
- </p>
- <%# Todo: add list of external authentications providers to
- discovery document, then generate the option list here. Right
- now, don't provide 'auth_provider' to get the default one. %>
+ <% case %>
+ <% when Rails.configuration.Login.PAM.Enable,
+ Rails.configuration.Login.LDAP.Enable,
+ Rails.configuration.Login.Test.Enable %>
+ <form id="login-form-tag" onsubmit="controller_password_authenticate(event)">
+ <p>username <input type="text" class="form-control" name="login-username"
+ value="" id="login-username" style="width: 50%"
+ oninput="clear_authenticate_error()"></input></p>
+ <p>password <input type="password" class="form-control" name="login-password" value=""
+ id="login-password" style="width: 50%"
+ oninput="clear_authenticate_error()"></input></p>
+ <input type="hidden" name="return_to" value="<%= "#{Rails.configuration.Services.Workbench1.ExternalURL}" %>" id="login-return-to">
+ <span style="color: red"><p id="login-authenticate-error"></p></span>
+ <button type="submit" class="btn btn-primary">Log in</button>
+ </form>
+ <% else %>
<div class="pull-right">
<%= link_to arvados_api_client.arvados_login_url(return_to: request.url), class: "btn btn-primary" do %>
Log in to <%= Rails.configuration.Workbench.SiteName %>
<i class="fa fa-fw fa-arrow-circle-right"></i>
<% end %>
</div>
+ <% end %>
+
</div>
</div>
</div>
if !user
assert page.has_text?('Please log in'), 'Not found text - Please log in'
- assert page.has_text?('The "Log in" button below will show you a Google sign-in page'), 'Not found text - google sign in page'
+ assert page.has_text?('If you have never used Arvados Workbench before'), 'Not found text - If you have never'
assert page.has_no_text?('My projects'), 'Found text - My projects'
- assert page.has_link?("Log in to #{Rails.configuration.Workbench.SiteName}"), 'Not found text - log in to'
+ assert page.has_link?("Log in"), 'Not found text - Log in'
elsif user['is_active']
if profile_config && !has_profile
assert page.has_text?('Save profile'), 'No text - Save profile'
ENV DEBIAN_FRONTEND noninteractive
# Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3-venv python3-dev libpam-dev
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3-venv python3-dev libpam-dev equivs
# Install virtualenv
RUN /usr/bin/pip3 install 'virtualenv<20'
ENV DEBIAN_FRONTEND noninteractive
# Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3-venv python3-dev libpam-dev shared-mime-info
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3-venv python3-dev libpam-dev shared-mime-info equivs
# Install virtualenv
RUN /usr/bin/pip3 install 'virtualenv<20'
cd "$WORKSPACE"
if [[ -z "$ARVADOS_BUILDING_VERSION" ]] && ! [[ -z "$version_tag" ]]; then
- ARVADOS_BUILDING_VERSION="$version_tag"
- ARVADOS_BUILDING_ITERATION="1"
+ export ARVADOS_BUILDING_VERSION="$version_tag"
+ export ARVADOS_BUILDING_ITERATION="1"
fi
# This defines python_sdk_version and cwl_runner_version with python-style
return 0
fi
+ if [[ "$ARVADOS_BUILDING_ITERATION" == "" ]]; then
+ ARVADOS_BUILDING_ITERATION=1
+ fi
+
if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
cd $WORKSPACE/$PKG_DIR
pwd
rm -rf dist/*
# Get the latest setuptools
- if ! $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then
- echo "Error, unable to upgrade setuptools with"
- echo " $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'"
+ if ! pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then
+ echo "Error, unable to upgrade setuptools with XY"
+ echo " pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'"
exit 1
fi
# filter a useless warning (when building the cwltest package) from the stderr output
merge_base=$(git merge-base origin/master "$commit")
if git merge-base --is-ancestor "$nearest_tag" "$merge_base" ; then
- # x.(y+1).0.devTIMESTAMP, where x.y.z is the newest version that does not contain $commit
+ # x.(y+1).0~devTIMESTAMP, where x.y.z is the newest version that does not contain $commit
# grep reads the list of tags (-f) that contain $commit and filters them out (-v)
# this prevents a newer tag from retroactively changing the versions of everything before it
- v=$(git tag | grep -vFf <(git tag --contains "$commit") | sort -Vr | head -n1 | perl -pe 's/\.(\d+)\.\d+/".".($1+1).".0"/e')
+ v=$(git tag | grep -vFf <(git tag --contains "$commit") | sort -Vr | head -n1 | perl -pe 's/(\d+)\.(\d+)\.\d+.*/"$1.".($2+1).".0"/e')
else
- # x.y.(z+1).devTIMESTAMP, where x.y.z is the latest released ancestor of $commit
+ # x.y.(z+1)~devTIMESTAMP, where x.y.z is the latest released ancestor of $commit
v=$(echo $nearest_tag | perl -pe 's/(\d+)$/$1+1/e')
fi
isodate=$(TZ=UTC git log -n1 --format=%cd --date=iso "$commit")
// kex_exchange_identification: Connection closed by remote host
// Connection closed by UNKNOWN port 65535
// exit status 255
+ //
+ // In case our target is a container request, the probe also
+ // resolves it to a container, so we don't connect to two
+ // different containers in a race.
+ var probetarget bytes.Buffer
exitcode := connectSSHCommand{}.RunCommand(
"arvados-client connect-ssh",
[]string{"-detach-keys=" + *detachKeys, "-probe-only=true", target},
- &bytes.Buffer{}, &bytes.Buffer{}, stderr)
+ &bytes.Buffer{}, &probetarget, stderr)
if exitcode != 0 {
return exitcode
}
+ target = strings.Trim(probetarget.String(), "\n")
selfbin, err := os.Readlink("/proc/self/exe")
if err != nil {
`)
f.PrintDefaults()
}
- probeOnly := f.Bool("probe-only", false, "do not transfer IO, just exit 0 immediately if tunnel setup succeeds")
+ probeOnly := f.Bool("probe-only", false, "do not transfer IO, just setup tunnel, print target UUID, and exit")
detachKeys := f.String("detach-keys", "", "set detach key sequence, as in docker-attach(1)")
if err := f.Parse(args); err != nil {
fmt.Fprintln(stderr, err)
defer sshconn.Conn.Close()
if *probeOnly {
+ fmt.Fprintln(stdout, targetUUID)
return 0
}
- user/cwl/federated-workflows.html.textile.liquid
- user/cwl/cwl-versions.html.textile.liquid
- user/cwl/crunchstat-summary.html.textile.liquid
+ - user/cwl/costanalyzer.html.textile.liquid
+ - user/debugging/container-shell-access.html.textile.liquid
- Working with git repositories:
- user/tutorials/add-new-repository.html.textile.liquid
- user/tutorials/git-arvados-guide.html.textile.liquid
- install/crunch2-slurm/configure-slurm.html.textile.liquid
- install/crunch2-slurm/install-compute-node.html.textile.liquid
- install/crunch2-slurm/install-test.html.textile.liquid
+ - Additional configuration:
+ - install/container-shell-access.html.textile.liquid
- External dependencies:
- install/install-postgresql.html.textile.liquid
- install/ruby.html.textile.liquid
# After applying changes, re-run @arvados-server config-check@ again to check for additional warnings and recommendations.
# When you are satisfied, delete the legacy config file, restart the service, and check its startup logs.
# Copy the updated @config.yml@ file to your next node, and repeat the process there.
+# When you have a @config.yml@ file that includes all volumes on all keepstores, it is important to add a 'Rendezvous' parameter to the InternalURLs entries to make sure the old volume identifiers line up with the new config. If you don't do this, @keep-balance@ will want to shuffle all the existing data around to match the new volume order. The 'Rendezvous' value should be the last 15 characters of the keepstore's UUID in the old configuration. Here's an example:
+
+<notextile>
+<pre><code>Clusters:
+ xxxxx:
+ Services:
+ Keepstore:
+ InternalURLs:
+ "http://keep1.xxxxx.arvadosapi.com:25107": {Rendezvous: "eim6eefaibesh3i"}
+ "http://keep2.xxxxx.arvadosapi.com:25107": {Rendezvous: "yequoodalai7ahg"}
+ "http://keep3.xxxxx.arvadosapi.com:25107": {Rendezvous: "eipheho6re1shou"}
+ "http://keep4.xxxxx.arvadosapi.com:25107": {Rendezvous: "ahk7chahthae3oo"}
+</code></pre>
+</notextile>
+
+In this example, the keepstore with the name `keep1` had the uuid `xxxxx-bi6l4-eim6eefaibesh3i` in the old configuration.
After migrating and removing all legacy config files, make sure the @/etc/arvados/config.yml@ file is identical across all system nodes -- API server, keepstore, etc. -- and restart all services to make sure they are using the latest configuration.
This is independent of @Workbench.IdleTimeout@. Even if Workbench auto-logout is disabled, this option will ensure that the user is always required to log in again after the configured amount of time.
-When this configuration is active (has a nonzero value), the Workbench client will also be "untrusted" by default. This means tokens issued to Workbench cannot be used to list other tokens issued to the user, and cannot be used to grant new tokens. This stops an attacker from leveraging a leaked token to aquire other tokens, but also interferes with some Workbench features that create new tokens on behalf of the user.
+h2. Untrusted login tokens
+
+<pre>
+Clusters:
+ zzzzz:
+ ...
+ Login:
+ TrustLoginTokens: false
+ ...
+</pre>
+
+When `TrustLoginTokens` is `false`, tokens issued through login will be "untrusted" by default. Untrusted tokens cannot be used to list other tokens issued to the user, and cannot be used to grant new tokens. This stops an attacker from leveraging a leaked token to aquire other tokens, but also interferes with some Workbench features that create new tokens on behalf of the user.
The default value @Login.TokenLifetime@ is zero, meaning login tokens do not expire (unless @API.MaxTokenLifetime@ is set).
...
</pre>
-Tokens created without an explicit expiration time, or that exceed maximum lifetime, will be clamped to @API.MaxTokenLifetime@.
+Tokens created without an explicit expiration time, or that exceed maximum lifetime, will be set to @API.MaxTokenLifetime@.
Similar to @Login.TokenLifetime@, this option ensures that the user is always required to log in again after the configured amount of time.
-Unlike @Login.TokenLifetime@, this applies to all API operations that manipulate tokens, regardless of whether the token was created by logging in, or by using the API. Also unlike @Login.TokenLifetime@, this setting does not imply any additional restrictions on token capabilities (it does not interfere with Workbench features that create new tokens on behalf of the user). If @Login.TokenLifetime@ is greater than @API.MaxTokenLifetime@, MaxTokenLifetime takes precedence.
+Unlike @Login.TokenLifetime@, this applies to all API operations that manipulate tokens, regardless of whether the token was created by logging in, or by using the API. If @Login.TokenLifetime@ is greater than @API.MaxTokenLifetime@, MaxTokenLifetime takes precedence.
-Admin users are permitted to create tokens with expiration times further in the future than MaxTokenLifetime, or with no expiration time at all.
+Admin users are permitted to create tokens with expiration times further in the future than @MaxTokenLifetime@.
The default value @MaxTokenLifetime@ is zero, which means there is no maximum token lifetime.
h2. Choosing a policy
-@Workbench.IdleTimeout@ only affects browser behavior. It is strongly recommended that automatic browser logout be used together with one or both token lifetime options, which are enforced on API side.
+@Workbench.IdleTimeout@ only affects browser behavior. It is strongly recommended that automatic browser logout be used together with @Login.TokenLifetime@, which is enforced on API side.
-@Login.TokenLifetime@ is more restrictive. A token obtained by logging into Workbench cannot be "refreshed" to gain access for an indefinite period. However, it interferes with some Workbench features, as well as ease of use in other contexts, such as the Arvados command line. This option is recommended only if most users will only ever interact with the system through Workbench or WebShell. For users or service accounts that need to tokens with fewer restrictions, the admin can "create a token at the command line":user-management-cli.html#create-token .
+@TrustLoginTokens: true@ (default value) is less restrictive. Be aware that an unrestricted token can be "refreshed" to gain access for an indefinite period. This means, during the window that the token is valid, the user is permitted to create a new token, which will have a new expiration further in the future (of course, once the token has expired, this is no longer possible). Unrestricted tokens are required for some Workbench features, as well as ease of use in other contexts, such as the Arvados command line. This option is recommended if many users will interact with the system through the command line.
-@API.MaxTokenLifetime@ is less restrictive. Be aware that an unrestricted token can be "refreshed" to gain access for an indefinite period. This means, during the window that the token is valid, the user is permitted to create a new token, which will have a new expiration further in the future (of course, once the token has expired, this is no longer possible). Unrestricted tokens are required for some Workbench features, as well as ease of use in other contexts, such as the Arvados command line. This option is recommended if many users will interact with the system through the command line.
+@TrustLoginTokens: false@ is more restrictive. A token obtained by logging into Workbench cannot be "refreshed" to gain access for an indefinite period. However, it interferes with some Workbench features, as well as ease of use in other contexts, such as the Arvados command line. This option is recommended only if most users will only ever interact with the system through Workbench or WebShell. For users or service accounts that need to tokens with fewer restrictions, the admin can "create a token at the command line":user-management-cli.html#create-token using the @SystemRootToken@.
-In every case, admin users may always create tokens with no expiration date.
+In every case, admin users may always create tokens with expiration dates far in the future.
These policies do not apply to tokens created by the API server for the purposes of authorizing a container to run, as those tokens are automatically expired when the container is finished.
<div class="releasenotes">
</notextile>
-h2(#main). development main (as of 2020-12-10)
+h2(#main). development main (as of 2021-06-03)
+
+"Upgrading from 2.2.0":#v2_2_0
+
+h2(#v2_2_0). v2.2.0 (2021-06-03)
"Upgrading from 2.1.0":#v2_1_0
+h3. Multi-file docker image collections
+
+Typically a docker image collection contains a single @.tar@ file at the top level. Handling of atypical cases has changed. If a docker image collection contains files with extensions other than @.tar@, they will be ignored (previously they could cause errors). If a docker image collection contains multiple @.tar@ files, it will cause an error at runtime, "cannot choose from multiple tar files in image collection" (previously one of the @.tar@ files was selected). Subdirectories are ignored. The @arv keep docker@ command always creates a collection with a single @.tar@ file, and never uses subdirectories, so this change will not affect most users.
+
h3. New spelling of S3 credential configs
If you use the S3 driver for Keep volumes and specify credentials in your configuration file (as opposed to using an IAM role), you should change the spelling of the @AccessKey@ and @SecretKey@ config keys to @AccessKeyID@ and @SecretAccessKey@. If you don't update them, the previous spellings will still be accepted, but warnings will be logged at server startup.
|old_user_uuid|uuid|The uuid of the "old" account|query||
|new_owner_uuid|uuid|The uuid of a project to which objects owned by the "old" user will be reassigned.|query||
|redirect_to_new_user|boolean|If true, also redirect login and reassign authorization credentials from "old" user to the "new" user|query||
+
+h3. authenticate
+
+Create a new API token based on username/password credentials. Returns an "API client authorization":api_client_authorizations.html object containing the API token, or an "error object.":../requests.html#errors
+
+Valid credentials are determined by the choice of "configured login backend.":{{site.baseurl}}/install/setup-login.html
+
+Note: this endpoint cannot be used with login backends that use web-based third party authentication, such as Google or OpenID Connect.
+
+Arguments:
+
+table(table table-bordered table-condensed).
+|_. Argument |_. Type |_. Description |_. Location |_. Example |
+{background:#ccffcc}.|username|string|The username.|body||
+{background:#ccffcc}.|password|string|The password.|body||
API requests must provide the API token using the @Authorization@ header in the following format:
<pre>
-$ curl -v -H "Authorization: OAuth2 xxxxapitokenxxxx" https://192.168.5.2:8000/arvados/v1/collections
+$ curl -v -H "Authorization: Bearer xxxxapitokenxxxx" https://192.168.5.2:8000/arvados/v1/collections
> GET /arvados/v1/collections HTTP/1.1
> ...
-> Authorization: OAuth2 xxxxapitokenxxxx
+> Authorization: Bearer xxxxapitokenxxxx
> ...
</pre>
+On a cluster configured to use an OpenID Connect provider (other than Google) as a login backend, Arvados can be configured to accept an OpenID Connect access token in place of an Arvados API token. OIDC access tokens are also accepted by a cluster that delegates login to another cluster (LoginCluster) which in turn has this feature configured. See @Login.OpenIDConnect.AcceptAccessTokenScope@ in the "default config.yml file":{{site.baseurl}}/admin/config.html for details.
+
+<pre>
+$ curl -v -H "Authorization: Bearer xxxx-openid-connect-access-token-xxxx" https://192.168.5.2:8000/arvados/v1/collections
+</pre>
+
h3. Parameters
Request parameters may be provided in one of two ways. They may be provided in the "query" section of request URI, or they may be provided in the body of the request with application/x-www-form-urlencoded encoding. If parameters are provided in both places, their values will be merged. Parameter names must be unique. If a parameter appears multiple times, the behavior is undefined.
Results are returned JSON-encoded in the response body.
-h3. Errors
+h3(#errors). Errors
If a request cannot be fulfilled, the API will return 4xx or 5xx HTTP status code. Be aware that the API server may return a 404 (Not Found) status for resources that exist but for which the client does not have read access. The API will also return an error record:
h3. Create a new record
<pre>
-$ curl -v -X POST --data-urlencode 'collection={"name":"empty collection"}' -H "Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections | jq .
+$ curl -v -X POST --data-urlencode 'collection={"name":"empty collection"}' -H "Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections | jq .
> POST /arvados/v1/collections HTTP/1.1
> User-Agent: curl/7.38.0
> Host: 192.168.5.2:8000
> Accept: */*
-> Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
+> Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
> Content-Length: 54
> Content-Type: application/x-www-form-urlencoded
>
h3. Delete a record
<pre>
-$ curl -X DELETE -v -H "Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections/962eh-4zz18-m1ma0mxxfg3mbcc | jq .
+$ curl -X DELETE -v -H "Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections/962eh-4zz18-m1ma0mxxfg3mbcc | jq .
> DELETE /arvados/v1/collections/962eh-4zz18-m1ma0mxxfg3mbcc HTTP/1.1
> User-Agent: curl/7.38.0
> Host: 192.168.5.2:8000
> Accept: */*
-> Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
+> Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
>
< HTTP/1.1 200 OK
< Content-Type: application/json; charset=utf-8
h3. Get a specific record
<pre>
-$ curl -v -H "Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km | jq .
+$ curl -v -H "Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km | jq .
> GET /arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km HTTP/1.1
> User-Agent: curl/7.38.0
> Host: 192.168.5.2:8000
> Accept: */*
-> Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
+> Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
>
< HTTP/1.1 200 OK
< Content-Type: application/json; charset=utf-8
(Note, return result is truncated).
<pre>
-$ curl -v -G --data-urlencode 'filters=[["created_at",">","2016-11-08T21:38:24.124834000Z"]]' -H "Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections | jq .
+$ curl -v -G --data-urlencode 'filters=[["created_at",">","2016-11-08T21:38:24.124834000Z"]]' -H "Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections | jq .
> GET /arvados/v1/collections?filters=%5B%5B%22uuid%22%2C%20%22%3D%22%2C%20%22962eh-4zz18-xi32mpz2621o8km%22%5D%5D HTTP/1.1
> User-Agent: curl/7.38.0
> Host: 192.168.5.2:8000
> Accept: */*
-> Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
+> Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
>
< HTTP/1.1 200 OK
< Content-Type: application/json; charset=utf-8
h3. Update a field
<pre>
-$ curl -v -X PUT --data-urlencode 'collection={"name":"rna.SRR948778.bam"}' -H "Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km | jq .
+$ curl -v -X PUT --data-urlencode 'collection={"name":"rna.SRR948778.bam"}' -H "Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km | jq .
> PUT /arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km HTTP/1.1
> User-Agent: curl/7.38.0
> Host: 192.168.5.2:8000
> Accept: */*
-> Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
+> Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
> Content-Length: 53
> Content-Type: application/x-www-form-urlencoded
>
All requests to the API server must have an API token. API tokens can be issued by going though the login flow, or created via the API. At this time, only browser based applications can perform login from email/password. Command line applications and services must use an API token provided via the @ARVADOS_API_TOKEN@ environment variable or configuration file.
-h2. Browser login
+h2. Login
-Browser based applications can perform log in via the following highlevel flow:
+Browser based applications can log in using one of the two possible flows:
-# The web application presents a "login" link to @/login@ on the API server with a @return_to@ parameter provided in the query portion of the URL. For example @https://{{ site.arvados_api_host }}/login?return_to=XXX@ , where @return_to=XXX@ is the URL of the login page for the web application.
-# The "login" link takes the browser to the login page (this may involve several redirects)
-# The user logs in. API server authenticates the user and issues a new API token.
-# The browser is redirected to the login page URL provided in @return_to=XXX@ with the addition of @?api_token=xxxxapitokenxxxx@.
-# The web application gets the login request with the included authorization token.
+h3. Authenticate via a third party
-!{{site.baseurl}}/images/Session_Establishment.svg!
+# The web application instructs the user to click on a link to the @/login@ endpoint on the API server. This link should include the @return_to@ parameter in the query portion of the URL. For example @https://{{ site.arvados_api_host }}/login?return_to=XXX@ , where @return_to=XXX@ is a page in the web application.
+# The @/login@ endpoint redirects the user to the configured third party authentication provider (e.g. Google or other OpenID Connect provider).
+# The user logs in to the third party provider, then they are redirected back to the API server.
+# The API server authenticates the user, issues a new API token, and redirects the browser to the URL provided in @return_to=XXX@ with the addition of @?api_token=xxxxapitokenxxxx@.
+# The web application gets the authorization token from the query and uses it to access the API server on the user's behalf.
+
+h3. Direct username/password authentication
+
+# The web application presents username and password fields.
+# When the submit button is pressed, using Javascript, the browser sends a POST request to @/arvados/v1/users/authenticate@
+** The request payload type is @application/javascript@
+** The request body is a JSON object with @username@ and @password@ fields.
+# The API server receives the username and password, authenticates them with the upstream provider (such as LDAP or PAM), and responds with the @api_client_authorization@ object for the new API token.
+# The web application receives the authorization token in the response and uses it to access the API server on the user's behalf.
+
+h3. Using an OpenID Connect access token
-The "browser authentication process is documented in detail on the Arvados wiki.":https://dev.arvados.org/projects/arvados/wiki/Workbench_authentication_process
+A cluster that uses OpenID Connect as a login provider can be configured to accept OIDC access tokens as well as Arvados API tokens (this is disabled by default; see @Login.OpenIDConnect.AcceptAccessToken@ in the "default config.yml file":{{site.baseurl}}/admin/config.html).
+# The client obtains an access token from the OpenID Connect provider via some method outside of Arvados.
+# The client presents the access token with an Arvados API request (e.g., request header @Authorization: Bearer xxxxaccesstokenxxxx@).
+# Depending on configuration, the API server decodes the access token (which must be a signed JWT) and confirms that it includes the required scope (see @Login.OpenIDConnect.AcceptAccessTokenScope@ in the "default config.yml file":{{site.baseurl}}/admin/config.html).
+# The API server uses the provider's UserInfo endpoint to validate the presented token.
+# If the token is valid, it is cached in the Arvados database and accepted in subsequent API calls for the next 10 minutes.
+
+h3. Diagram
+
+!{{site.baseurl}}/images/Session_Establishment.svg!
h2. User activation
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Configure container shell access
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+Arvados can be configured to permit shell access to running containers. This can be handy for debugging, but it could affect reproducability of workflows. This feature can be enabled for admin users, or for all users. By default, it is entirely disabled.
+
+The relevant configuration section is
+
+<notextile>
+<pre><code> Containers:
+ ShellAccess:
+ # An admin user can use "arvados-client shell" to start an
+ # interactive shell (with any user ID) in any running
+ # container.
+ Admin: false
+
+ # Any user can use "arvados-client shell" to start an
+ # interactive shell (with any user ID) in any running
+ # container that they started, provided it isn't also
+ # associated with a different user's container request.
+ #
+ # Interactive sessions make it easy to alter the container's
+ # runtime environment in ways that aren't recorded or
+ # reproducible. Consider the implications for automatic
+ # container reuse before enabling and using this feature. In
+ # particular, note that starting an interactive session does
+ # not disqualify a container from being reused by a different
+ # user/workflow in the future.
+ User: false
+</code></pre>
+</notextile>
+
+To enable the feature a firewall change may also be required. This feature requires the opening of tcp connections from @arvados-controller@ to the range specified in the @net.ipv4.ip_local_port_range@ sysctl on compute nodes. If that range is unknown or hard to determine, it will be sufficient to allow tcp connections from @arvados-controller@ to port 1024-65535 on compute nodes, while allowing traffic that is part of existing tcp connections.
+
+After changing the configuration, @arvados-controller@ must be restarted for the change to take effect. When enabling, shell access will be enabled for any running containers. When disabling, access is removed immediately for any running containers, as well as any containers started subsequently. Restarting @arvados-controller@ will kill any active connections.
+
+Usage instructions for this feature are available in the "User guide":{{site.baseurl}}/user/debugging/container-shell-access.html.
|_. Distribution|_. State|_. Last supported version|
|CentOS 7|Supported|Latest|
|Debian 10 ("buster")|Supported|Latest|
+|Ubuntu 20.04 ("focal")|Supported|Latest|
|Ubuntu 18.04 ("bionic")|Supported|Latest|
-|Ubuntu 16.04 ("xenial")|Supported|Latest|
-|Debian 9 ("stretch")|EOL|Latest 2.1.X release|
+|Ubuntu 16.04 ("xenial")|EOL|2.1.2|
+|Debian 9 ("stretch")|EOL|2.1.2|
|Debian 8 ("jessie")|EOL|1.4.3|
|Ubuntu 14.04 ("trusty")|EOL|1.4.3|
|Ubuntu 12.04 ("precise")|EOL|8ed7b6dd5d4df93a3f37096afe6d6f81c2a7ef6e (2017-05-03)|
# "Install git and curl":#install-packages
# "Update Git Config":#config-git
# "Create record for VM":#vm-record
-# "Create scoped token":#scoped-token
# "Install arvados-login-sync":#arvados-login-sync
# "Confirm working installation":#confirm-working
# WORKBENCH node:
## arvados workbench
## arvados workbench2
+## arvados webshell
# KEEPPROXY node:
## arvados keepproxy
## arvados keepweb
h2(#multi_host). Multi host install using the provision.sh script
-This is a package-based installation method. The Salt scripts are available from the "tools/salt-install":https://github.com/arvados/arvados/tree/master/tools/salt-install directory in the Arvados git repository.
+{% if site.current_version %}
+{% assign branchname = site.current_version | slice: 1, 5 | append: '-dev' %}
+{% else %}
+{% assign branchname = 'master' %}
+{% endif %}
-This procedure will install all the main Arvados components to get you up and running in a multi host environment.
+This is a package-based installation method. Start with the @provision.sh@ script which is available by cloning the @{{ branchname }}@ branch from "https://git.arvados.org/arvados.git":https://git.arvados.org/arvados.git . The @provision.sh@ script and its supporting files can be found in the "arvados/tools/salt-install":https://git.arvados.org/arvados.git/tree/refs/heads/{{ branchname }}:/tools/salt-install directory in the Arvados git repository.
-We suggest you to use the @provision.sh@ script to deploy Arvados, which is implemented with the @arvados-formula@ in a Saltstack master-less setup. After setting up a few variables in a config file (next step), you'll be ready to run it and get Arvados deployed.
+This procedure will install all the main Arvados components to get you up and running in a multi-host environment.
+
+The @provision.sh@ script will help you deploy Arvados by preparing your environment to be able to run the installer, then running it. The actual installer is located at "arvados-formula":https://git.arvados.org/arvados-formula.git/tree/refs/heads/{{ branchname }} and will be cloned during the running of the @provision.sh@ script. The installer is built using "Saltstack":https://saltproject.io/ and @provision.sh@ performs the install using master-less mode.
+
+After setting up a few variables in a config file (next step), you'll be ready to run it and get Arvados deployed.
h3(#create_a_compute_image). Create a compute image
#. Workbench
<notextile>
<pre><code>scp -r provision.sh local* user@host:
-ssh user@host sudo ./provision.sh --config local.params --roles workbench,workbench2
+ssh user@host sudo ./provision.sh --config local.params --roles workbench,workbench2,webshell
</code></pre>
</notextile>
</code></pre>
</notextile>
-h2(#initial_user). Initial user and login
+h2(#initial_user). Initial user and login
At this point you should be able to log into the Arvados cluster. The initial URL will be:
<b>NOTE: The single host installation is not recommended for production use.</b>
-This is a package-based installation method. The Salt scripts are available from the "tools/salt-install":https://github.com/arvados/arvados/tree/master/tools/salt-install directory in the Arvados git repository.
+{% if site.current_version %}
+{% assign branchname = site.current_version | slice: 1, 5 | append: '-dev' %}
+{% else %}
+{% assign branchname = 'master' %}
+{% endif %}
+
+This is a package-based installation method. Start with the @provision.sh@ script which is available by cloning the @{{ branchname }}@ branch from "https://git.arvados.org/arvados.git":https://git.arvados.org/arvados.git . The @provision.sh@ script and its supporting files can be found in the "arvados/tools/salt-install":https://git.arvados.org/arvados.git/tree/refs/heads/{{ branchname }}:/tools/salt-install directory in the Arvados git repository.
This procedure will install all the main Arvados components to get you up and running in a single host. The whole installation procedure takes somewhere between 15 to 60 minutes, depending on the host resources and its network bandwidth. As a reference, on a virtual machine with 1 core and 1 GB RAM, it takes ~25 minutes to do the initial install.
-We suggest you to use the @provision.sh@ script to deploy Arvados, which is implemented with the @arvados-formula@ in a Saltstack master-less setup. After setting up a few variables in a config file (next step), you'll be ready to run it and get Arvados deployed.
+The @provision.sh@ script will help you deploy Arvados by preparing your environment to be able to run the installer, then running it. The actual installer is located at "arvados-formula":https://git.arvados.org/arvados-formula.git/tree/refs/heads/{{ branchname }} and will be cloned during the running of the @provision.sh@ script. The installer is built using "Saltstack":https://saltproject.io/ and @provision.sh@ performs the install using master-less mode.
+
+After setting up a few variables in a config file (next step), you'll be ready to run it and get Arvados deployed.
h2(#choose_configuration). Choose the desired configuration
</code></pre>
</notextile>
-h2(#initial_user). Initial user and login
+h2(#initial_user). Initial user and login
At this point you should be able to log into the Arvados cluster. The initial URL will be:
Check the LDAP section in the "default config file":{{site.baseurl}}/admin/config.html for more details and configuration options.
-h2(#pam). PAM (experimental)
+h2(#pam). PAM
With this configuration, authentication is done according to the Linux PAM ("Pluggable Authentication Modules") configuration on your controller host.
Check the "default config file":{{site.baseurl}}/admin/config.html for more PAM configuration options.
-The default PAM configuration on most Linux systems uses the local password database in @/etc/shadow@ for all logins. In this case, in order to log in to Arvados, users must have a UNIX account and password on the controller host itself. This can be convenient for a single-user or test cluster. User accounts can have @/dev/false@ as the shell in order to allow the user to log into Arvados but not log into a shell on the controller host.
+The default PAM configuration on most Linux systems uses the local user/password database in @/etc/passwd@ and @/etc/shadow@ for all logins. In this case, in order to log in to Arvados, users must have a UNIX account and password on the controller host itself. This can be convenient for a single-user or test cluster. Configuring a user account with a shell of @/bin/false@ will enable the user to log into Arvados but not log into shell login on the controller host.
-PAM can also be configured to use different backends like LDAP. In a production environment, PAM configuration should use the service name ("arvados" by default) to set a separate policy for Arvados logins: generally, Arvados users should not have shell accounts on the controller node.
+PAM can also be configured to use other authentication systems such such as NIS or Kerberos. In a production environment, PAM configuration should use the service name ("arvados" by default) and set a separate policy for Arvados login. In this case, Arvados users should not have shell accounts on the controller node.
For information about configuring PAM, refer to the "PAM System Administrator's Guide":http://www.linux-pam.org/Linux-PAM-html/Linux-PAM_SAG.html.
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: Analyzing workflow cost (cloud only)
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+{% include 'notebox_begin' %}
+
+This is only applicable when Arvados runs in a cloud environment and @arvados-dispatch-cloud@ is used to dispatch @crunch@ jobs. The per node-hour price for each defined InstanceType most be supplied in "config.yml":{{site.baseurl}}/admin/config.html.
+
+{% include 'notebox_end' %}
+
+The @arvados-client@ program can be used to analyze the cost of a workflow. It can be installed from packages (@apt install arvados-client@ or @yum install arvados-client@). The @arvados-client costanalyzer@ command analyzes the cost accounting information associated with Arvados container requests.
+
+h2(#syntax). Syntax
+
+The @arvados-client costanalyzer@ tool has a number of command line arguments:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arvados-client costanalyzer -h</span>
+Usage:
+ arvados-client costanalyzer [options ...] uuid [uuid ...]
+
+ This program analyzes the cost of Arvados container requests. For each uuid
+ supplied, it creates a CSV report that lists all the containers used to
+ fulfill the container request, together with the machine type and cost of
+ each container. At least one uuid must be specified.
+
+ When supplied with the uuid of a container request, it will calculate the
+ cost of that container request and all its children.
+
+ When supplied with the uuid of a collection, it will see if there is a
+ container_request uuid in the properties of the collection, and if so, it
+ will calculate the cost of that container request and all its children.
+
+ When supplied with a project uuid or when supplied with multiple container
+ request or collection uuids, it will create a CSV report for each supplied
+ uuid, as well as a CSV file with aggregate cost accounting for all supplied
+ uuids. The aggregate cost report takes container reuse into account: if a
+ container was reused between several container requests, its cost will only
+ be counted once.
+
+ Caveats:
+
+ - This program uses the cost data from config.yml at the time of the
+ execution of the container, stored in the 'node.json' file in its log
+ collection. If the cost data was not correctly configured at the time the
+ container was executed, the output from this program will be incorrect.
+
+ - If a container was run on a preemptible ("spot") instance, the cost data
+ reported by this program may be wildly inaccurate, because it does not have
+ access to the spot pricing in effect for the node then the container ran. The
+ UUID report file that is generated when the '-output' option is specified has
+ a column that indicates the preemptible state of the instance that ran the
+ container.
+
+ - This program does not take into account overhead costs like the time spent
+ starting and stopping compute nodes that run containers, the cost of the
+ permanent cloud nodes that provide the Arvados services, the cost of data
+ stored in Arvados, etc.
+
+ - When provided with a project uuid, subprojects will not be considered.
+
+ In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
+ ARVADOS_API_TOKEN environment variables must be set.
+
+ This program prints the total dollar amount from the aggregate cost
+ accounting across all provided uuids on stdout.
+
+ When the '-output' option is specified, a set of CSV files with cost details
+ will be written to the provided directory.
+
+Options:
+ -cache
+ create and use a local disk cache of Arvados objects (default true)
+ -log-level level
+ logging level (debug, info, ...) (default "info")
+ -output directory
+ output directory for the CSV reports
+</code></pre>
+</notextile>
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: Debugging workflows - shell access
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+{% include 'notebox_begin' %}
+
+To use this feature, your Arvados installation must be configured to allow container shell access. See "the install guide":{{site.baseurl}}/install/container-shell-access.html for more information.
+
+{% include 'notebox_end' %}
+
+The @arvados-client@ program can be used to connect to a container in a running workflow. It can be installed from packages (@apt install arvados-client@ or @yum install arvados-client@). The @arvados-client shell@ command provides an ssh connection into a running container.
+
+h2(#syntax). Syntax
+
+The @arvados-client shell@ tool has the following syntax:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arvados-client shell -h</span>
+arvados-client shell: open an interactive shell on a running container.
+
+Usage: arvados-client shell [options] [username@]container-uuid [ssh-options] [remote-command [args...]]
+
+Options:
+ -detach-keys string
+ set detach key sequence, as in docker-attach(1) (default "ctrl-],ctrl-]")
+
+</code></pre>
+</notextile>
+
+The @arvados-client shell@ command calls the ssh binary on your system to make the connection. Everything after _[username@]container-uuid_ is passed through to your OpenSSH client.
+
+h2(#Examples). Examples
+
+Connect to a running container, using the container request UUID:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arvados-client shell ce8i5-xvhdp-e6wnujfslyyqn4b</span>
+root@0f13dcd755fa:~#
+</code></pre>
+</notextile>
+
+The container UUID also works:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arvados-client shell ce8i5-dz642-h1cl0sa62d4i430</span>
+root@0f13dcd755fa:~#
+</code></pre>
+</notextile>
+
+SSH port forwarding is supported:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arvados-client shell ce8i5-dz642-h1cl0sa62d4i430 -L8888:localhost:80</span>
+root@0f13dcd755fa:~# nc -l -p 80
+</code></pre>
+</notextile>
+
+And then, connecting to port 8888 locally:
+
+<notextile>
+<pre><code>~$ <span class="userinput">echo hello | nc localhost 8888</span>
+</code></pre>
+</notextile>
+
+Which appears on the other end:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arvados-client shell ce8i5-dz642-h1cl0sa62d4i430 -L8888:localhost:80</span>
+root@0f13dcd755fa:~# nc -l -p 80
+hello
+</code></pre>
+</notextile>
github.com/src-d/gcfg v1.3.0 // indirect
github.com/xanzy/ssh-agent v0.1.0 // indirect
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
- golang.org/x/net v0.0.0-20201021035429-f5854403a974
+ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
- golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4
- golang.org/x/tools v0.1.0 // indirect
+ golang.org/x/sys v0.0.0-20210510120138-977fb7262007
+ golang.org/x/tools v0.1.2 // indirect
google.golang.org/api v0.13.0
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405
github.com/xanzy/ssh-agent v0.1.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8=
github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA=
+golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
UUIDTTL: 5s
# Block cache entries. Each block consumes up to 64 MiB RAM.
- MaxBlockEntries: 4
+ MaxBlockEntries: 20
# Collection cache entries.
MaxCollectionEntries: 1000
AuthenticationRequestParameters:
SAMPLE: ""
+ # Accept an OIDC access token as an API token if the OIDC
+ # provider's UserInfo endpoint accepts it.
+ #
+ # AcceptAccessTokenScope should also be used when enabling
+ # this feature.
+ AcceptAccessToken: false
+
+ # Before accepting an OIDC access token as an API token, first
+ # check that it is a JWT whose "scope" value includes this
+ # value. Example: "https://zzzzz.example.com/" (your Arvados
+ # API endpoint).
+ #
+ # If this value is empty and AcceptAccessToken is true, all
+ # access tokens will be accepted regardless of scope,
+ # including non-JWT tokens. This is not recommended.
+ AcceptAccessTokenScope: ""
+
PAM:
# (Experimental) Use PAM to authenticate users.
Enable: false
# Default value zero means tokens don't have expiration.
TokenLifetime: 0s
+ # If true (default) tokens issued through login are allowed to create
+ # new tokens.
+ # If false, tokens issued through login are not allowed to
+ # viewing/creating other tokens. New tokens can only be created
+ # by going through login again.
+ IssueTrustedTokens: true
+
# When the token is returned to a client, the token itself may
- # be restricted from manipulating other tokens based on whether
+ # be restricted from viewing/creating other tokens based on whether
# the client is "trusted" or not. The local Workbench1 and
# Workbench2 are trusted by default, but if this is a
# LoginCluster, you probably want to include the other Workbench
# Minimum time between two attempts to run the same container
MinRetryPeriod: 0s
+ # Container runtime: "docker" (default) or "singularity" (experimental)
+ RuntimeEngine: docker
+
Logging:
# When you run the db:delete_old_container_logs task, it will find
# containers that have been finished for at least this many seconds,
<img src="/arvados-logo-big.png" style="width: 20%; float: right; padding: 1em;" />
<h2>Please log in.</h2>
- <p>The "Log in" button below will show you a sign-in
- page. After you log in, you will be redirected back to
- Arvados Workbench.</p>
-
<p>If you have never used Arvados Workbench before, logging in
for the first time will automatically create a new
account.</p>
- <i>Arvados Workbench uses your name and email address only for
+ <i>Arvados Workbench uses your information only for
identification, and does not retrieve any other personal
information.</i>
"Containers.MaxRetryAttempts": true,
"Containers.MinRetryPeriod": true,
"Containers.ReserveExtraRAM": true,
+ "Containers.RuntimeEngine": true,
"Containers.ShellAccess": true,
"Containers.ShellAccess.Admin": true,
"Containers.ShellAccess.User": true,
"Login.LDAP.UsernameAttribute": false,
"Login.LoginCluster": true,
"Login.OpenIDConnect": true,
+ "Login.OpenIDConnect.AcceptAccessToken": false,
+ "Login.OpenIDConnect.AcceptAccessTokenScope": false,
"Login.OpenIDConnect.AuthenticationRequestParameters": false,
"Login.OpenIDConnect.ClientID": false,
"Login.OpenIDConnect.ClientSecret": false,
"Login.Test.Enable": true,
"Login.Test.Users": false,
"Login.TokenLifetime": false,
+ "Login.IssueTrustedTokens": false,
"Login.TrustedClients": false,
"Mail": true,
"Mail.EmailFrom": false,
UUIDTTL: 5s
# Block cache entries. Each block consumes up to 64 MiB RAM.
- MaxBlockEntries: 4
+ MaxBlockEntries: 20
# Collection cache entries.
MaxCollectionEntries: 1000
AuthenticationRequestParameters:
SAMPLE: ""
+ # Accept an OIDC access token as an API token if the OIDC
+ # provider's UserInfo endpoint accepts it.
+ #
+ # AcceptAccessTokenScope should also be used when enabling
+ # this feature.
+ AcceptAccessToken: false
+
+ # Before accepting an OIDC access token as an API token, first
+ # check that it is a JWT whose "scope" value includes this
+ # value. Example: "https://zzzzz.example.com/" (your Arvados
+ # API endpoint).
+ #
+ # If this value is empty and AcceptAccessToken is true, all
+ # access tokens will be accepted regardless of scope,
+ # including non-JWT tokens. This is not recommended.
+ AcceptAccessTokenScope: ""
+
PAM:
# (Experimental) Use PAM to authenticate users.
Enable: false
# Default value zero means tokens don't have expiration.
TokenLifetime: 0s
+ # If true (default) tokens issued through login are allowed to create
+ # new tokens.
+ # If false, tokens issued through login are not allowed to
+ # viewing/creating other tokens. New tokens can only be created
+ # by going through login again.
+ IssueTrustedTokens: true
+
# When the token is returned to a client, the token itself may
- # be restricted from manipulating other tokens based on whether
+ # be restricted from viewing/creating other tokens based on whether
# the client is "trusted" or not. The local Workbench1 and
# Workbench2 are trusted by default, but if this is a
# LoginCluster, you probably want to include the other Workbench
# Minimum time between two attempts to run the same container
MinRetryPeriod: 0s
+ # Container runtime: "docker" (default) or "singularity" (experimental)
+ RuntimeEngine: docker
+
Logging:
# When you run the db:delete_old_container_logs task, it will find
# containers that have been finished for at least this many seconds,
<img src="/arvados-logo-big.png" style="width: 20%; float: right; padding: 1em;" />
<h2>Please log in.</h2>
- <p>The "Log in" button below will show you a sign-in
- page. After you log in, you will be redirected back to
- Arvados Workbench.</p>
-
<p>If you have never used Arvados Workbench before, logging in
for the first time will automatically create a new
account.</p>
- <i>Arvados Workbench uses your name and email address only for
+ <i>Arvados Workbench uses your information only for
identification, and does not retrieve any other personal
information.</i>
cluster.Login.OpenIDConnect.ClientSecret = s.fakeProvider.ValidClientSecret
cluster.Login.OpenIDConnect.EmailClaim = "email"
cluster.Login.OpenIDConnect.EmailVerifiedClaim = "email_verified"
+ cluster.Login.OpenIDConnect.AcceptAccessToken = true
+ cluster.Login.OpenIDConnect.AcceptAccessTokenScope = ""
s.testHandler = &Handler{Cluster: cluster}
s.testServer = newServerFromIntegrationTestEnv(c)
_, err := fn(ctx, conn.cluster.ClusterID, conn.local, opts)
return err
}
+ if opts.ClusterID != "" {
+ // Client explicitly selected cluster
+ _, err := fn(ctx, conn.cluster.ClusterID, conn.chooseBackend(opts.ClusterID), opts)
+ return err
+ }
cannotSplit := false
var matchAllFilters map[string]bool
ClientSecret: ` + s.oidcprovider.ValidClientSecret + `
EmailClaim: email
EmailVerifiedClaim: email_verified
+ AcceptAccessToken: true
+ AcceptAccessTokenScope: ""
`
} else {
yaml += `
}
case wantOpenIDConnect:
return &oidcLoginController{
- Cluster: cluster,
- Parent: parent,
- Issuer: cluster.Login.OpenIDConnect.Issuer,
- ClientID: cluster.Login.OpenIDConnect.ClientID,
- ClientSecret: cluster.Login.OpenIDConnect.ClientSecret,
- AuthParams: cluster.Login.OpenIDConnect.AuthenticationRequestParameters,
- EmailClaim: cluster.Login.OpenIDConnect.EmailClaim,
- EmailVerifiedClaim: cluster.Login.OpenIDConnect.EmailVerifiedClaim,
- UsernameClaim: cluster.Login.OpenIDConnect.UsernameClaim,
+ Cluster: cluster,
+ Parent: parent,
+ Issuer: cluster.Login.OpenIDConnect.Issuer,
+ ClientID: cluster.Login.OpenIDConnect.ClientID,
+ ClientSecret: cluster.Login.OpenIDConnect.ClientSecret,
+ AuthParams: cluster.Login.OpenIDConnect.AuthenticationRequestParameters,
+ EmailClaim: cluster.Login.OpenIDConnect.EmailClaim,
+ EmailVerifiedClaim: cluster.Login.OpenIDConnect.EmailVerifiedClaim,
+ UsernameClaim: cluster.Login.OpenIDConnect.UsernameClaim,
+ AcceptAccessToken: cluster.Login.OpenIDConnect.AcceptAccessToken,
+ AcceptAccessTokenScope: cluster.Login.OpenIDConnect.AcceptAccessTokenScope,
}
case wantSSO:
return &ssoLoginController{Parent: parent}
"golang.org/x/oauth2"
"google.golang.org/api/option"
"google.golang.org/api/people/v1"
+ "gopkg.in/square/go-jose.v2/jwt"
)
var (
)
type oidcLoginController struct {
- Cluster *arvados.Cluster
- Parent *Conn
- Issuer string // OIDC issuer URL, e.g., "https://accounts.google.com"
- ClientID string
- ClientSecret string
- UseGooglePeopleAPI bool // Use Google People API to look up alternate email addresses
- EmailClaim string // OpenID claim to use as email address; typically "email"
- EmailVerifiedClaim string // If non-empty, ensure claim value is true before accepting EmailClaim; typically "email_verified"
- UsernameClaim string // If non-empty, use as preferred username
- AuthParams map[string]string // Additional parameters to pass with authentication request
+ Cluster *arvados.Cluster
+ Parent *Conn
+ Issuer string // OIDC issuer URL, e.g., "https://accounts.google.com"
+ ClientID string
+ ClientSecret string
+ UseGooglePeopleAPI bool // Use Google People API to look up alternate email addresses
+ EmailClaim string // OpenID claim to use as email address; typically "email"
+ EmailVerifiedClaim string // If non-empty, ensure claim value is true before accepting EmailClaim; typically "email_verified"
+ UsernameClaim string // If non-empty, use as preferred username
+ AcceptAccessToken bool // Accept access tokens as API tokens
+ AcceptAccessTokenScope string // If non-empty, don't accept access tokens as API tokens unless they contain this scope
+ AuthParams map[string]string // Additional parameters to pass with authentication request
// override Google People API base URL for testing purposes
// (normally empty, set by google pkg to
if !ok {
return loginError(errors.New("error in OAuth2 exchange: no ID token in OAuth2 token"))
}
+ ctxlog.FromContext(ctx).WithField("rawIDToken", rawIDToken).Debug("oauth2Token provided ID token")
idToken, err := ctrl.verifier.Verify(ctx, rawIDToken)
if err != nil {
return loginError(fmt.Errorf("error verifying ID token: %s", err))
if err != nil {
return fmt.Errorf("error setting up OpenID Connect provider: %s", err)
}
+ if ok, err := ta.checkAccessTokenScope(ctx, tok); err != nil || !ok {
+ ta.cache.Add(tok, time.Now().Add(tokenCacheNegativeTTL))
+ return err
+ }
oauth2Token := &oauth2.Token{
AccessToken: tok,
}
ta.cache.Add(tok, aca)
return nil
}
+
+// Check that the provided access token is a JWT with the required
+// scope. If it is a valid JWT but missing the required scope, we
+// return a 403 error, otherwise true (acceptable as an API token) or
+// false (pass through unmodified).
+//
+// Return false if configured not to accept access tokens at all.
+//
+// Note we don't check signature or expiry here. We are relying on the
+// caller to verify those separately (e.g., by calling the UserInfo
+// endpoint).
+func (ta *oidcTokenAuthorizer) checkAccessTokenScope(ctx context.Context, tok string) (bool, error) {
+ if !ta.ctrl.AcceptAccessToken {
+ return false, nil
+ } else if ta.ctrl.AcceptAccessTokenScope == "" {
+ return true, nil
+ }
+ var claims struct {
+ Scope string `json:"scope"`
+ }
+ if t, err := jwt.ParseSigned(tok); err != nil {
+ ctxlog.FromContext(ctx).WithError(err).Debug("error parsing jwt")
+ return false, nil
+ } else if err = t.UnsafeClaimsWithoutVerification(&claims); err != nil {
+ ctxlog.FromContext(ctx).WithError(err).Debug("error extracting jwt claims")
+ return false, nil
+ }
+ for _, s := range strings.Split(claims.Scope, " ") {
+ if s == ta.ctrl.AcceptAccessTokenScope {
+ return true, nil
+ }
+ }
+ ctxlog.FromContext(ctx).WithFields(logrus.Fields{"have": claims.Scope, "need": ta.ctrl.AcceptAccessTokenScope}).Infof("unacceptable access token scope")
+ return false, httpserver.ErrorWithStatus(errors.New("unacceptable access token scope"), http.StatusUnauthorized)
+}
json.Unmarshal([]byte(fmt.Sprintf("%q", s.fakeProvider.Issuer.URL)), &s.cluster.Login.OpenIDConnect.Issuer)
s.cluster.Login.OpenIDConnect.ClientID = "oidc#client#id"
s.cluster.Login.OpenIDConnect.ClientSecret = "oidc#client#secret"
+ s.cluster.Login.OpenIDConnect.AcceptAccessToken = true
+ s.cluster.Login.OpenIDConnect.AcceptAccessTokenScope = ""
s.fakeProvider.ValidClientID = "oidc#client#id"
s.fakeProvider.ValidClientSecret = "oidc#client#secret"
db := arvadostest.DB(c, s.cluster)
tokenCacheTTL = time.Millisecond
tokenCacheRaceWindow = time.Millisecond
+ tokenCacheNegativeTTL = time.Millisecond
oidcAuthorizer := OIDCAccessTokenAuthorizer(s.cluster, func(context.Context) (*sqlx.DB, error) { return db, nil })
accessToken := s.fakeProvider.ValidAccessToken()
mac := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
io.WriteString(mac, accessToken)
- hmac := fmt.Sprintf("%x", mac.Sum(nil))
+ apiToken := fmt.Sprintf("%x", mac.Sum(nil))
cleanup := func() {
- _, err := db.Exec(`delete from api_client_authorizations where api_token=$1`, hmac)
+ _, err := db.Exec(`delete from api_client_authorizations where api_token=$1`, apiToken)
c.Check(err, check.IsNil)
}
cleanup()
c.Assert(creds.Tokens, check.HasLen, 1)
c.Check(creds.Tokens[0], check.Equals, accessToken)
- err := db.QueryRowContext(ctx, `select expires_at at time zone 'UTC' from api_client_authorizations where api_token=$1`, hmac).Scan(&exp1)
+ err := db.QueryRowContext(ctx, `select expires_at at time zone 'UTC' from api_client_authorizations where api_token=$1`, apiToken).Scan(&exp1)
c.Check(err, check.IsNil)
c.Check(exp1.Sub(time.Now()) > -time.Second, check.Equals, true)
c.Check(exp1.Sub(time.Now()) < time.Second, check.Equals, true)
})(ctx, nil)
// If the token is used again after the in-memory cache
- // expires, oidcAuthorizer must re-checks the token and update
+ // expires, oidcAuthorizer must re-check the token and update
// the expires_at value in the database.
time.Sleep(3 * time.Millisecond)
oidcAuthorizer.WrapCalls(func(ctx context.Context, opts interface{}) (interface{}, error) {
var exp time.Time
- err := db.QueryRowContext(ctx, `select expires_at at time zone 'UTC' from api_client_authorizations where api_token=$1`, hmac).Scan(&exp)
+ err := db.QueryRowContext(ctx, `select expires_at at time zone 'UTC' from api_client_authorizations where api_token=$1`, apiToken).Scan(&exp)
c.Check(err, check.IsNil)
c.Check(exp.Sub(exp1) > 0, check.Equals, true)
c.Check(exp.Sub(exp1) < time.Second, check.Equals, true)
return nil, nil
})(ctx, nil)
+
+ s.fakeProvider.AccessTokenPayload = map[string]interface{}{"scope": "openid profile foobar"}
+ accessToken = s.fakeProvider.ValidAccessToken()
+ ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{accessToken}})
+
+ mac = hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
+ io.WriteString(mac, accessToken)
+ apiToken = fmt.Sprintf("%x", mac.Sum(nil))
+
+ for _, trial := range []struct {
+ configEnable bool
+ configScope string
+ acceptable bool
+ shouldRun bool
+ }{
+ {true, "foobar", true, true},
+ {true, "foo", false, false},
+ {true, "", true, true},
+ {false, "", false, true},
+ {false, "foobar", false, true},
+ } {
+ c.Logf("trial = %+v", trial)
+ cleanup()
+ s.cluster.Login.OpenIDConnect.AcceptAccessToken = trial.configEnable
+ s.cluster.Login.OpenIDConnect.AcceptAccessTokenScope = trial.configScope
+ oidcAuthorizer = OIDCAccessTokenAuthorizer(s.cluster, func(context.Context) (*sqlx.DB, error) { return db, nil })
+ checked := false
+ oidcAuthorizer.WrapCalls(func(ctx context.Context, opts interface{}) (interface{}, error) {
+ var n int
+ err := db.QueryRowContext(ctx, `select count(*) from api_client_authorizations where api_token=$1`, apiToken).Scan(&n)
+ c.Check(err, check.IsNil)
+ if trial.acceptable {
+ c.Check(n, check.Equals, 1)
+ } else {
+ c.Check(n, check.Equals, 0)
+ }
+ checked = true
+ return nil, nil
+ })(ctx, nil)
+ c.Check(checked, check.Equals, trial.shouldRun)
+ }
}
func (s *OIDCLoginSuite) TestGenericOIDCLogin(c *check.C) {
import (
"io"
+ "time"
- "git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"github.com/sirupsen/logrus"
)
-var Command command
+var Command = command{}
-type command struct{}
+type command struct {
+ uuids arrayFlags
+ resultsDir string
+ cache bool
+ begin time.Time
+ end time.Time
+}
type NoPrefixFormatter struct{}
}
// RunCommand implements the subcommand "costanalyzer <collection> <collection> ..."
-func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+func (c command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
var err error
logger := ctxlog.New(stderr, "text", "info")
defer func() {
logger.SetFormatter(new(NoPrefixFormatter))
- loader := config.NewLoader(stdin, logger)
- loader.SkipLegacy = true
-
- exitcode, err := costanalyzer(prog, args, loader, logger, stdout, stderr)
+ exitcode, err := c.costAnalyzer(prog, args, logger, stdout, stderr)
return exitcode
}
"errors"
"flag"
"fmt"
- "git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/keepclient"
"github.com/sirupsen/logrus"
)
+const timestampFormat = "2006-01-02T15:04:05"
+
type nodeInfo struct {
// Legacy (records created by Arvados Node Manager with Arvados <= 1.4.3)
Properties struct {
return nil
}
-func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string, cache bool, err error) {
+func (c *command) parseFlags(prog string, args []string, logger *logrus.Logger, stderr io.Writer) (exitCode int, err error) {
+ var beginStr, endStr string
flags := flag.NewFlagSet("", flag.ContinueOnError)
flags.SetOutput(stderr)
flags.Usage = func() {
fmt.Fprintf(flags.Output(), `
Usage:
- %s [options ...] <uuid> ...
+ %s [options ...] [UUID ...]
+
+ This program analyzes the cost of Arvados container requests and calculates
+ the total cost across all requests. At least one UUID or a timestamp range
+ must be specified.
- This program analyzes the cost of Arvados container requests. For each uuid
- supplied, it creates a CSV report that lists all the containers used to
- fulfill the container request, together with the machine type and cost of
- each container. At least one uuid must be specified.
+ When the '-output' option is specified, a set of CSV files with cost details
+ will be written to the provided directory. Each file is a CSV report that lists
+ all the containers used to fulfill the container request, together with the
+ machine type and cost of each container.
- When supplied with the uuid of a container request, it will calculate the
+ When supplied with the UUID of a container request, it will calculate the
cost of that container request and all its children.
- When supplied with the uuid of a collection, it will see if there is a
- container_request uuid in the properties of the collection, and if so, it
+ When supplied with the UUID of a collection, it will see if there is a
+ container_request UUID in the properties of the collection, and if so, it
will calculate the cost of that container request and all its children.
- When supplied with a project uuid or when supplied with multiple container
- request or collection uuids, it will create a CSV report for each supplied
- uuid, as well as a CSV file with aggregate cost accounting for all supplied
- uuids. The aggregate cost report takes container reuse into account: if a
- container was reused between several container requests, its cost will only
- be counted once.
+ When supplied with a project UUID or when supplied with multiple container
+ request or collection UUIDs, it will calculate the total cost for all
+ supplied UUIDs.
- To get the node costs, the progam queries the Arvados API for current cost
- data for each node type used. This means that the reported cost always
- reflects the cost data as currently defined in the Arvados API configuration
- file.
+ When supplied with a 'begin' and 'end' timestamp (format:
+ %s), it will calculate the cost for all top-level container
+ requests whose containers finished during the specified interval.
+
+ The total cost calculation takes container reuse into account: if a container
+ was reused between several container requests, its cost will only be counted
+ once.
Caveats:
- - the Arvados API configuration cost data may be out of sync with the cloud
- provider.
- - when generating reports for older container requests, the cost data in the
- Arvados API configuration file may have changed since the container request
- was fulfilled. This program uses the cost data stored at the time of the
+
+ - This program uses the cost data from config.yml at the time of the
execution of the container, stored in the 'node.json' file in its log
- collection.
- - if a container was run on a preemptible ("spot") instance, the cost data
+ collection. If the cost data was not correctly configured at the time the
+ container was executed, the output from this program will be incorrect.
+
+ - If a container was run on a preemptible ("spot") instance, the cost data
reported by this program may be wildly inaccurate, because it does not have
access to the spot pricing in effect for the node then the container ran. The
UUID report file that is generated when the '-output' option is specified has
a column that indicates the preemptible state of the instance that ran the
container.
- In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
+ - This program does not take into account overhead costs like the time spent
+ starting and stopping compute nodes that run containers, the cost of the
+ permanent cloud nodes that provide the Arvados services, the cost of data
+ stored in Arvados, etc.
+
+ - When provided with a project UUID, subprojects will not be considered.
+
+ In order to get the data for the UUIDs supplied, the ARVADOS_API_HOST and
ARVADOS_API_TOKEN environment variables must be set.
This program prints the total dollar amount from the aggregate cost
- accounting across all provided uuids on stdout.
-
- When the '-output' option is specified, a set of CSV files with cost details
- will be written to the provided directory.
+ accounting across all provided UUIDs on stdout.
Options:
-`, prog)
+`, prog, timestampFormat)
flags.PrintDefaults()
}
loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)")
- flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports")
- flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects")
+ flags.StringVar(&c.resultsDir, "output", "", "output `directory` for the CSV reports")
+ flags.StringVar(&beginStr, "begin", "", fmt.Sprintf("timestamp `begin` for date range operation (format: %s)", timestampFormat))
+ flags.StringVar(&endStr, "end", "", fmt.Sprintf("timestamp `end` for date range operation (format: %s)", timestampFormat))
+ flags.BoolVar(&c.cache, "cache", true, "create and use a local disk cache of Arvados objects")
err = flags.Parse(args)
if err == flag.ErrHelp {
err = nil
exitCode = 2
return
}
- uuids = flags.Args()
+ c.uuids = flags.Args()
- if len(uuids) < 1 {
+ if (len(beginStr) != 0 && len(endStr) == 0) || (len(beginStr) == 0 && len(endStr) != 0) {
+ flags.Usage()
+ err = fmt.Errorf("When specifying a date range, both begin and end must be specified")
+ exitCode = 2
+ return
+ }
+
+ if len(beginStr) != 0 {
+ var errB, errE error
+ c.begin, errB = time.Parse(timestampFormat, beginStr)
+ c.end, errE = time.Parse(timestampFormat, endStr)
+ if (errB != nil) || (errE != nil) {
+ flags.Usage()
+ err = fmt.Errorf("When specifying a date range, both begin and end must be of the format %s %+v, %+v", timestampFormat, errB, errE)
+ exitCode = 2
+ return
+ }
+ }
+
+ if (len(c.uuids) < 1) && (len(beginStr) == 0) {
flags.Usage()
err = fmt.Errorf("error: no uuid(s) provided")
exitCode = 2
return
}
logger.SetLevel(lvl)
- if !cache {
+ if !c.cache {
logger.Debug("Caching disabled\n")
}
return
var tmpCsv string
var tmpTotalCost float64
var totalCost float64
+ fmt.Printf("Processing %s\n", uuid)
var crUUID = uuid
if strings.Contains(uuid, "-4zz18-") {
if err != nil {
return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
}
+ if len(cr.ContainerUUID) == 0 {
+ // Nothing to do! E.g. a CR in 'Uncommitted' state.
+ logger.Infof("No container associated with container request %s, skipping\n", crUUID)
+ return nil, nil
+ }
var container arvados.Container
err = loadObject(logger, ac, crUUID, cr.ContainerUUID, cache, &container)
if err != nil {
topNode, err := getNode(arv, ac, kc, cr)
if err != nil {
- return nil, fmt.Errorf("error getting node %s: %s", cr.UUID, err)
+ logger.Errorf("Skipping container request %s: error getting node %s: %s", cr.UUID, cr.UUID, err)
+ return nil, nil
}
tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container)
csv += tmpCsv
if err != nil {
return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
}
- logger.Infof("Collecting child containers for container request %s", crUUID)
+ logger.Infof("Collecting child containers for container request %s (%s)", crUUID, container.FinishedAt)
for _, cr2 := range childCrs.Items {
logger.Info(".")
node, err := getNode(arv, ac, kc, cr2)
if err != nil {
- return nil, fmt.Errorf("error getting node %s: %s", cr2.UUID, err)
+ logger.Errorf("Skipping container request %s: error getting node %s: %s", cr2.UUID, cr2.UUID, err)
+ continue
}
logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n")
var c2 arvados.Container
return
}
-func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) {
- exitcode, uuids, resultsDir, cache, err := parseFlags(prog, args, loader, logger, stderr)
+func (c *command) costAnalyzer(prog string, args []string, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) {
+ exitcode, err = c.parseFlags(prog, args, logger, stderr)
+
if exitcode != 0 {
return
}
- if resultsDir != "" {
- err = ensureDirectory(logger, resultsDir)
+ if c.resultsDir != "" {
+ err = ensureDirectory(logger, c.resultsDir)
if err != nil {
exitcode = 3
return
}
}
+ uuidChannel := make(chan string)
+
// Arvados Client setup
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
ac := arvados.NewClientFromEnv()
+ // Populate uuidChannel with the requested uuid list
+ go func() {
+ defer close(uuidChannel)
+ for _, uuid := range c.uuids {
+ uuidChannel <- uuid
+ }
+
+ if !c.begin.IsZero() {
+ initialParams := arvados.ResourceListParams{
+ Filters: []arvados.Filter{{"container.finished_at", ">=", c.begin}, {"container.finished_at", "<", c.end}, {"requesting_container_uuid", "=", nil}},
+ Order: "created_at",
+ }
+ params := initialParams
+ for {
+ // This list variable must be a new one declared
+ // inside the loop: otherwise, items in the API
+ // response would get deep-merged into the items
+ // loaded in previous iterations.
+ var list arvados.ContainerRequestList
+
+ err := ac.RequestAndDecode(&list, "GET", "arvados/v1/container_requests", nil, params)
+ if err != nil {
+ logger.Errorf("Error getting container request list from Arvados API: %s\n", err)
+ break
+ }
+ if len(list.Items) == 0 {
+ break
+ }
+
+ for _, i := range list.Items {
+ uuidChannel <- i.UUID
+ }
+ params.Offset += len(list.Items)
+ }
+
+ }
+ }()
+
cost := make(map[string]float64)
- for _, uuid := range uuids {
+
+ for uuid := range uuidChannel {
+ fmt.Printf("Considering %s\n", uuid)
if strings.Contains(uuid, "-j7d0g-") {
// This is a project (group)
- cost, err = handleProject(logger, uuid, arv, ac, kc, resultsDir, cache)
+ cost, err = handleProject(logger, uuid, arv, ac, kc, c.resultsDir, c.cache)
if err != nil {
exitcode = 1
return
} else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") {
// This is a container request
var crCsv map[string]float64
- crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
+ crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, c.resultsDir, c.cache)
if err != nil {
err = fmt.Errorf("error generating CSV for uuid %s: %s", uuid, err.Error())
exitcode = 2
var csv string
csv = "# Aggregate cost accounting for uuids:\n"
- for _, uuid := range uuids {
+ for _, uuid := range c.uuids {
csv += "# " + uuid + "\n"
}
csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n"
- if resultsDir != "" {
+ if c.resultsDir != "" {
// Write the resulting CSV file
- aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
+ aFile := c.resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
err = ioutil.WriteFile(aFile, []byte(csv), 0644)
if err != nil {
err = fmt.Errorf("error writing file with path %s: %s", aFile, err.Error())
c.Check(stderr.String(), check.Matches, `(?ms).*Usage:.*`)
}
+func (*Suite) TestTimestampRange(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ resultsDir := c.MkDir()
+ // Run costanalyzer with a timestamp range. This should pick up two container requests in "Final" state.
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, "-begin", "2020-11-02T00:00:00", "-end", "2020-11-03T23:59:00"}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv")
+ c.Assert(err, check.IsNil)
+ uuid2Report, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv")
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00916192")
+ c.Check(string(uuid2Report), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00588088")
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,0.01492030")
+}
+
func (*Suite) TestContainerRequestUUID(c *check.C) {
var stdout, stderr bytes.Buffer
resultsDir := c.MkDir()
c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000")
}
+func (*Suite) TestUncommittedContainerRequest(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ // Run costanalyzer with 2 container request uuids, one of which is in the Uncommitted state, without output directory specified
+ exitcode := Command.RunCommand("costanalyzer.test", []string{arvadostest.UncommittedContainerRequestUUID, arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Not(check.Matches), "(?ms).*supplied uuids in .*")
+ c.Assert(stderr.String(), check.Matches, "(?ms).*No container associated with container request .*")
+
+ // Check that the total amount was printed to stdout
+ c.Check(stdout.String(), check.Matches, "0.00588088\n")
+}
+
func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
var stdout, stderr bytes.Buffer
// Run costanalyzer with 2 container request uuids, without output directory specified
keepClient IKeepClient
hostOutputDir string
ctrOutputDir string
- binds []string
+ bindmounts map[string]bindmount
mounts map[string]arvados.Mount
secretMounts map[string]arvados.Mount
logger printfer
})
return nil
}
-
- return fmt.Errorf("Unsupported file type (mode %o) in output dir: %q", fi.Mode(), src)
+ cp.logger.Printf("Skipping unsupported file type (mode %o) in output dir: %q", fi.Mode(), src)
+ return nil
}
// Return the host path that was mounted at the given path in the
if ctrRoot == cp.ctrOutputDir {
return cp.hostOutputDir, nil
}
- for _, bind := range cp.binds {
- tokens := strings.Split(bind, ":")
- if len(tokens) >= 2 && tokens[1] == ctrRoot {
- return tokens[0], nil
- }
+ if mnt, ok := cp.bindmounts[ctrRoot]; ok {
+ return mnt.HostPath, nil
}
return "", fmt.Errorf("not bind-mounted: %q", ctrRoot)
}
package crunchrun
import (
+ "bytes"
"io"
"io/ioutil"
"os"
+ "syscall"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "github.com/sirupsen/logrus"
check "gopkg.in/check.v1"
)
var _ = check.Suite(&copierSuite{})
type copierSuite struct {
- cp copier
+ cp copier
+ log bytes.Buffer
}
func (s *copierSuite) SetUpTest(c *check.C) {
- tmpdir, err := ioutil.TempDir("", "crunch-run.test.")
- c.Assert(err, check.IsNil)
+ tmpdir := c.MkDir()
api, err := arvadosclient.MakeArvadosClient()
c.Assert(err, check.IsNil)
+ s.log = bytes.Buffer{}
s.cp = copier{
client: arvados.NewClientFromEnv(),
arvClient: api,
secretMounts: map[string]arvados.Mount{
"/secret_text": {Kind: "text", Content: "xyzzy"},
},
+ logger: &logrus.Logger{Out: &s.log, Formatter: &logrus.TextFormatter{}, Level: logrus.InfoLevel},
}
}
-func (s *copierSuite) TearDownTest(c *check.C) {
- os.RemoveAll(s.cp.hostOutputDir)
-}
-
func (s *copierSuite) TestEmptyOutput(c *check.C) {
err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
c.Check(err, check.IsNil)
_, err = io.WriteString(f, "foo")
c.Assert(err, check.IsNil)
c.Assert(f.Close(), check.IsNil)
+ err = syscall.Mkfifo(s.cp.hostOutputDir+"/dir1/fifo", 0644)
+ c.Assert(err, check.IsNil)
err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
c.Check(err, check.IsNil)
{src: os.DevNull, dst: "/dir1/dir2/dir3/.keep"},
{src: s.cp.hostOutputDir + "/dir1/foo", dst: "/dir1/foo", size: 3},
})
+ c.Check(s.log.String(), check.Matches, `.* msg="Skipping unsupported file type \(mode 200000644\) in output dir: \\"/ctr/outdir/dir1/fifo\\""\n`)
}
func (s *copierSuite) TestSymlinkCycle(c *check.C) {
PortableDataHash: arvadostest.FooCollectionPDH,
Writable: true,
}
- s.cp.binds = append(s.cp.binds, bindtmp+":/mnt-w")
+ s.cp.bindmounts = map[string]bindmount{
+ "/mnt-w": bindmount{HostPath: bindtmp, ReadOnly: false},
+ }
c.Assert(os.Symlink("../../mnt", s.cp.hostOutputDir+"/l_dir"), check.IsNil)
c.Assert(os.Symlink("/mnt/foo", s.cp.hostOutputDir+"/l_file"), check.IsNil)
"git.arvados.org/arvados.git/sdk/go/keepclient"
"git.arvados.org/arvados.git/sdk/go/manifest"
"golang.org/x/net/context"
-
- dockertypes "github.com/docker/docker/api/types"
- dockercontainer "github.com/docker/docker/api/types/container"
- dockernetwork "github.com/docker/docker/api/types/network"
- dockerclient "github.com/docker/docker/client"
)
type command struct{}
type MkTempDir func(string, string) (string, error)
-// ThinDockerClient is the minimal Docker client interface used by crunch-run.
-type ThinDockerClient interface {
- ContainerAttach(ctx context.Context, container string, options dockertypes.ContainerAttachOptions) (dockertypes.HijackedResponse, error)
- ContainerCreate(ctx context.Context, config *dockercontainer.Config, hostConfig *dockercontainer.HostConfig,
- networkingConfig *dockernetwork.NetworkingConfig, containerName string) (dockercontainer.ContainerCreateCreatedBody, error)
- ContainerStart(ctx context.Context, container string, options dockertypes.ContainerStartOptions) error
- ContainerRemove(ctx context.Context, container string, options dockertypes.ContainerRemoveOptions) error
- ContainerWait(ctx context.Context, container string, condition dockercontainer.WaitCondition) (<-chan dockercontainer.ContainerWaitOKBody, <-chan error)
- ContainerInspect(ctx context.Context, id string) (dockertypes.ContainerJSON, error)
- ImageInspectWithRaw(ctx context.Context, image string) (dockertypes.ImageInspect, []byte, error)
- ImageLoad(ctx context.Context, input io.Reader, quiet bool) (dockertypes.ImageLoadResponse, error)
- ImageRemove(ctx context.Context, image string, options dockertypes.ImageRemoveOptions) ([]dockertypes.ImageDeleteResponseItem, error)
-}
-
type PsProcess interface {
CmdlineSlice() ([]string, error)
}
// ContainerRunner is the main stateful struct used for a single execution of a
// container.
type ContainerRunner struct {
- Docker ThinDockerClient
+ executor containerExecutor
// Dispatcher client is initialized with the Dispatcher token.
// This is a privileged token used to manage container status
ContainerArvClient IArvadosClient
ContainerKeepClient IKeepClient
- Container arvados.Container
- ContainerConfig dockercontainer.Config
- HostConfig dockercontainer.HostConfig
- token string
- ContainerID string
- ExitCode *int
- NewLogWriter NewLogWriter
- loggingDone chan bool
- CrunchLog *ThrottledLogger
- Stdout io.WriteCloser
- Stderr io.WriteCloser
- logUUID string
- logMtx sync.Mutex
- LogCollection arvados.CollectionFileSystem
- LogsPDH *string
- RunArvMount RunArvMount
- MkTempDir MkTempDir
- ArvMount *exec.Cmd
- ArvMountPoint string
- HostOutputDir string
- Binds []string
- Volumes map[string]struct{}
- OutputPDH *string
- SigChan chan os.Signal
- ArvMountExit chan error
- SecretMounts map[string]arvados.Mount
- MkArvClient func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error)
- finalState string
- parentTemp string
+ Container arvados.Container
+ token string
+ ExitCode *int
+ NewLogWriter NewLogWriter
+ CrunchLog *ThrottledLogger
+ Stdout io.WriteCloser
+ Stderr io.WriteCloser
+ logUUID string
+ logMtx sync.Mutex
+ LogCollection arvados.CollectionFileSystem
+ LogsPDH *string
+ RunArvMount RunArvMount
+ MkTempDir MkTempDir
+ ArvMount *exec.Cmd
+ ArvMountPoint string
+ HostOutputDir string
+ Volumes map[string]struct{}
+ OutputPDH *string
+ SigChan chan os.Signal
+ ArvMountExit chan error
+ SecretMounts map[string]arvados.Mount
+ MkArvClient func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error)
+ finalState string
+ parentTemp string
statLogger io.WriteCloser
statReporter *crunchstat.Reporter
cStateLock sync.Mutex
cCancelled bool // StopContainer() invoked
- cRemoved bool // docker confirmed the container no longer exists
- enableNetwork string // one of "default" or "always"
- networkMode string // passed through to HostConfig.NetworkMode
- arvMountLog *ThrottledLogger
+ enableMemoryLimit bool
+ enableNetwork string // one of "default" or "always"
+ networkMode string // "none", "host", or "" -- passed through to executor
+ arvMountLog *ThrottledLogger
containerWatchdogInterval time.Duration
gateway Gateway
}
-// setupSignals sets up signal handling to gracefully terminate the underlying
-// Docker container and update state when receiving a TERM, INT or QUIT signal.
+// setupSignals sets up signal handling to gracefully terminate the
+// underlying container and update state when receiving a TERM, INT or
+// QUIT signal.
func (runner *ContainerRunner) setupSignals() {
runner.SigChan = make(chan os.Signal, 1)
signal.Notify(runner.SigChan, syscall.SIGTERM)
}(runner.SigChan)
}
-// stop the underlying Docker container.
+// stop the underlying container.
func (runner *ContainerRunner) stop(sig os.Signal) {
runner.cStateLock.Lock()
defer runner.cStateLock.Unlock()
if sig != nil {
runner.CrunchLog.Printf("caught signal: %v", sig)
}
- if runner.ContainerID == "" {
- return
- }
runner.cCancelled = true
- runner.CrunchLog.Printf("removing container")
- err := runner.Docker.ContainerRemove(context.TODO(), runner.ContainerID, dockertypes.ContainerRemoveOptions{Force: true})
+ runner.CrunchLog.Printf("stopping container")
+ err := runner.executor.Stop()
if err != nil {
- runner.CrunchLog.Printf("error removing container: %s", err)
- }
- if err == nil || strings.Contains(err.Error(), "No such container: "+runner.ContainerID) {
- runner.cRemoved = true
+ runner.CrunchLog.Printf("error stopping container: %s", err)
}
}
// LoadImage determines the docker image id from the container record and
// checks if it is available in the local Docker image store. If not, it loads
// the image from Keep.
-func (runner *ContainerRunner) LoadImage() (err error) {
-
+func (runner *ContainerRunner) LoadImage() (string, error) {
runner.CrunchLog.Printf("Fetching Docker image from collection '%s'", runner.Container.ContainerImage)
- var collection arvados.Collection
- err = runner.ContainerArvClient.Get("collections", runner.Container.ContainerImage, nil, &collection)
+ d, err := os.Open(runner.ArvMountPoint + "/by_id/" + runner.Container.ContainerImage)
+ if err != nil {
+ return "", err
+ }
+ defer d.Close()
+ allfiles, err := d.Readdirnames(-1)
if err != nil {
- return fmt.Errorf("While getting container image collection: %v", err)
+ return "", err
}
- manifest := manifest.Manifest{Text: collection.ManifestText}
- var img, imageID string
- for ms := range manifest.StreamIter() {
- img = ms.FileStreamSegments[0].Name
- if !strings.HasSuffix(img, ".tar") {
- return fmt.Errorf("First file in the container image collection does not end in .tar")
+ var tarfiles []string
+ for _, fnm := range allfiles {
+ if strings.HasSuffix(fnm, ".tar") {
+ tarfiles = append(tarfiles, fnm)
}
- imageID = img[:len(img)-4]
}
+ if len(tarfiles) == 0 {
+ return "", fmt.Errorf("image collection does not include a .tar image file")
+ }
+ if len(tarfiles) > 1 {
+ return "", fmt.Errorf("cannot choose from multiple tar files in image collection: %v", tarfiles)
+ }
+ imageID := tarfiles[0][:len(tarfiles[0])-4]
+ imageFile := runner.ArvMountPoint + "/by_id/" + runner.Container.ContainerImage + "/" + tarfiles[0]
+ runner.CrunchLog.Printf("Using Docker image id %q", imageID)
- runner.CrunchLog.Printf("Using Docker image id '%s'", imageID)
-
- _, _, err = runner.Docker.ImageInspectWithRaw(context.TODO(), imageID)
- if err != nil {
+ if !runner.executor.ImageLoaded(imageID) {
runner.CrunchLog.Print("Loading Docker image from keep")
-
- var readCloser io.ReadCloser
- readCloser, err = runner.ContainerKeepClient.ManifestFileReader(manifest, img)
+ err = runner.executor.LoadImage(imageFile)
if err != nil {
- return fmt.Errorf("While creating ManifestFileReader for container image: %v", err)
+ return "", err
}
-
- response, err := runner.Docker.ImageLoad(context.TODO(), readCloser, true)
- if err != nil {
- return fmt.Errorf("While loading container image into Docker: %v", err)
- }
-
- defer response.Body.Close()
- rbody, err := ioutil.ReadAll(response.Body)
- if err != nil {
- return fmt.Errorf("Reading response to image load: %v", err)
- }
- runner.CrunchLog.Printf("Docker response: %s", rbody)
} else {
runner.CrunchLog.Print("Docker image is available")
}
-
- runner.ContainerConfig.Image = imageID
-
- runner.ContainerKeepClient.ClearBlockCache()
-
- return nil
+ return imageID, nil
}
func (runner *ContainerRunner) ArvMountCmd(arvMountCmd []string, token string) (c *exec.Cmd, err error) {
}
runner.arvMountLog = NewThrottledLogger(w)
c.Stdout = runner.arvMountLog
- c.Stderr = runner.arvMountLog
+ c.Stderr = io.MultiWriter(runner.arvMountLog, os.Stderr)
runner.CrunchLog.Printf("Running %v", c.Args)
return nil
}
-func (runner *ContainerRunner) SetupMounts() (err error) {
- err = runner.SetupArvMountPoint("keep")
+func (runner *ContainerRunner) SetupMounts() (map[string]bindmount, error) {
+ bindmounts := map[string]bindmount{}
+ err := runner.SetupArvMountPoint("keep")
if err != nil {
- return fmt.Errorf("While creating keep mount temp dir: %v", err)
+ return nil, fmt.Errorf("While creating keep mount temp dir: %v", err)
}
token, err := runner.ContainerToken()
if err != nil {
- return fmt.Errorf("could not get container token: %s", err)
+ return nil, fmt.Errorf("could not get container token: %s", err)
}
+ runner.CrunchLog.Printf("container token %q", token)
pdhOnly := true
tmpcount := 0
}
collectionPaths := []string{}
- runner.Binds = nil
- runner.Volumes = make(map[string]struct{})
needCertMount := true
type copyFile struct {
src string
}
for bind := range runner.SecretMounts {
if _, ok := runner.Container.Mounts[bind]; ok {
- return fmt.Errorf("secret mount %q conflicts with regular mount", bind)
+ return nil, fmt.Errorf("secret mount %q conflicts with regular mount", bind)
}
if runner.SecretMounts[bind].Kind != "json" &&
runner.SecretMounts[bind].Kind != "text" {
- return fmt.Errorf("secret mount %q type is %q but only 'json' and 'text' are permitted",
+ return nil, fmt.Errorf("secret mount %q type is %q but only 'json' and 'text' are permitted",
bind, runner.SecretMounts[bind].Kind)
}
binds = append(binds, bind)
if bind == "stdout" || bind == "stderr" {
// Is it a "file" mount kind?
if mnt.Kind != "file" {
- return fmt.Errorf("unsupported mount kind '%s' for %s: only 'file' is supported", mnt.Kind, bind)
+ return nil, fmt.Errorf("unsupported mount kind '%s' for %s: only 'file' is supported", mnt.Kind, bind)
}
// Does path start with OutputPath?
prefix += "/"
}
if !strings.HasPrefix(mnt.Path, prefix) {
- return fmt.Errorf("%s path does not start with OutputPath: %s, %s", strings.Title(bind), mnt.Path, prefix)
+ return nil, fmt.Errorf("%s path does not start with OutputPath: %s, %s", strings.Title(bind), mnt.Path, prefix)
}
}
if bind == "stdin" {
// Is it a "collection" mount kind?
if mnt.Kind != "collection" && mnt.Kind != "json" {
- return fmt.Errorf("unsupported mount kind '%s' for stdin: only 'collection' and 'json' are supported", mnt.Kind)
+ return nil, fmt.Errorf("unsupported mount kind '%s' for stdin: only 'collection' and 'json' are supported", mnt.Kind)
}
}
if strings.HasPrefix(bind, runner.Container.OutputPath+"/") && bind != runner.Container.OutputPath+"/" {
if mnt.Kind != "collection" && mnt.Kind != "text" && mnt.Kind != "json" {
- return fmt.Errorf("only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path for %q, was %q", bind, mnt.Kind)
+ return nil, fmt.Errorf("only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path for %q, was %q", bind, mnt.Kind)
}
}
case mnt.Kind == "collection" && bind != "stdin":
var src string
if mnt.UUID != "" && mnt.PortableDataHash != "" {
- return fmt.Errorf("cannot specify both 'uuid' and 'portable_data_hash' for a collection mount")
+ return nil, fmt.Errorf("cannot specify both 'uuid' and 'portable_data_hash' for a collection mount")
}
if mnt.UUID != "" {
if mnt.Writable {
- return fmt.Errorf("writing to existing collections currently not permitted")
+ return nil, fmt.Errorf("writing to existing collections currently not permitted")
}
pdhOnly = false
src = fmt.Sprintf("%s/by_id/%s", runner.ArvMountPoint, mnt.UUID)
} else if mnt.PortableDataHash != "" {
if mnt.Writable && !strings.HasPrefix(bind, runner.Container.OutputPath+"/") {
- return fmt.Errorf("can never write to a collection specified by portable data hash")
+ return nil, fmt.Errorf("can never write to a collection specified by portable data hash")
}
idx := strings.Index(mnt.PortableDataHash, "/")
if idx > 0 {
if mnt.Writable {
if bind == runner.Container.OutputPath {
runner.HostOutputDir = src
- runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", src, bind))
+ bindmounts[bind] = bindmount{HostPath: src}
} else if strings.HasPrefix(bind, runner.Container.OutputPath+"/") {
copyFiles = append(copyFiles, copyFile{src, runner.HostOutputDir + bind[len(runner.Container.OutputPath):]})
} else {
- runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", src, bind))
+ bindmounts[bind] = bindmount{HostPath: src}
}
} else {
- runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s:ro", src, bind))
+ bindmounts[bind] = bindmount{HostPath: src, ReadOnly: true}
}
collectionPaths = append(collectionPaths, src)
var tmpdir string
tmpdir, err = runner.MkTempDir(runner.parentTemp, "tmp")
if err != nil {
- return fmt.Errorf("while creating mount temp dir: %v", err)
+ return nil, fmt.Errorf("while creating mount temp dir: %v", err)
}
st, staterr := os.Stat(tmpdir)
if staterr != nil {
- return fmt.Errorf("while Stat on temp dir: %v", staterr)
+ return nil, fmt.Errorf("while Stat on temp dir: %v", staterr)
}
err = os.Chmod(tmpdir, st.Mode()|os.ModeSetgid|0777)
if staterr != nil {
- return fmt.Errorf("while Chmod temp dir: %v", err)
+ return nil, fmt.Errorf("while Chmod temp dir: %v", err)
}
- runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", tmpdir, bind))
+ bindmounts[bind] = bindmount{HostPath: tmpdir}
if bind == runner.Container.OutputPath {
runner.HostOutputDir = tmpdir
}
if mnt.Kind == "json" {
filedata, err = json.Marshal(mnt.Content)
if err != nil {
- return fmt.Errorf("encoding json data: %v", err)
+ return nil, fmt.Errorf("encoding json data: %v", err)
}
} else {
text, ok := mnt.Content.(string)
if !ok {
- return fmt.Errorf("content for mount %q must be a string", bind)
+ return nil, fmt.Errorf("content for mount %q must be a string", bind)
}
filedata = []byte(text)
}
tmpdir, err := runner.MkTempDir(runner.parentTemp, mnt.Kind)
if err != nil {
- return fmt.Errorf("creating temp dir: %v", err)
+ return nil, fmt.Errorf("creating temp dir: %v", err)
}
tmpfn := filepath.Join(tmpdir, "mountdata."+mnt.Kind)
err = ioutil.WriteFile(tmpfn, filedata, 0444)
if err != nil {
- return fmt.Errorf("writing temp file: %v", err)
+ return nil, fmt.Errorf("writing temp file: %v", err)
}
if strings.HasPrefix(bind, runner.Container.OutputPath+"/") {
copyFiles = append(copyFiles, copyFile{tmpfn, runner.HostOutputDir + bind[len(runner.Container.OutputPath):]})
} else {
- runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s:ro", tmpfn, bind))
+ bindmounts[bind] = bindmount{HostPath: tmpfn, ReadOnly: true}
}
case mnt.Kind == "git_tree":
tmpdir, err := runner.MkTempDir(runner.parentTemp, "git_tree")
if err != nil {
- return fmt.Errorf("creating temp dir: %v", err)
+ return nil, fmt.Errorf("creating temp dir: %v", err)
}
err = gitMount(mnt).extractTree(runner.ContainerArvClient, tmpdir, token)
if err != nil {
- return err
+ return nil, err
}
- runner.Binds = append(runner.Binds, tmpdir+":"+bind+":ro")
+ bindmounts[bind] = bindmount{HostPath: tmpdir, ReadOnly: true}
}
}
if runner.HostOutputDir == "" {
- return fmt.Errorf("output path does not correspond to a writable mount point")
+ return nil, fmt.Errorf("output path does not correspond to a writable mount point")
}
if needCertMount && runner.Container.RuntimeConstraints.API {
for _, certfile := range arvadosclient.CertFiles {
_, err := os.Stat(certfile)
if err == nil {
- runner.Binds = append(runner.Binds, fmt.Sprintf("%s:/etc/arvados/ca-certificates.crt:ro", certfile))
+ bindmounts["/etc/arvados/ca-certificates.crt"] = bindmount{HostPath: certfile, ReadOnly: true}
break
}
}
runner.ArvMount, err = runner.RunArvMount(arvMountCmd, token)
if err != nil {
- return fmt.Errorf("while trying to start arv-mount: %v", err)
+ return nil, fmt.Errorf("while trying to start arv-mount: %v", err)
}
for _, p := range collectionPaths {
_, err = os.Stat(p)
if err != nil {
- return fmt.Errorf("while checking that input files exist: %v", err)
+ return nil, fmt.Errorf("while checking that input files exist: %v", err)
}
}
for _, cp := range copyFiles {
st, err := os.Stat(cp.src)
if err != nil {
- return fmt.Errorf("while staging writable file from %q to %q: %v", cp.src, cp.bind, err)
+ return nil, fmt.Errorf("while staging writable file from %q to %q: %v", cp.src, cp.bind, err)
}
if st.IsDir() {
err = filepath.Walk(cp.src, func(walkpath string, walkinfo os.FileInfo, walkerr error) error {
}
}
if err != nil {
- return fmt.Errorf("while staging writable file from %q to %q: %v", cp.src, cp.bind, err)
+ return nil, fmt.Errorf("while staging writable file from %q to %q: %v", cp.src, cp.bind, err)
}
}
- return nil
-}
-
-func (runner *ContainerRunner) ProcessDockerAttach(containerReader io.Reader) {
- // Handle docker log protocol
- // https://docs.docker.com/engine/reference/api/docker_remote_api_v1.15/#attach-to-a-container
- defer close(runner.loggingDone)
-
- header := make([]byte, 8)
- var err error
- for err == nil {
- _, err = io.ReadAtLeast(containerReader, header, 8)
- if err != nil {
- if err == io.EOF {
- err = nil
- }
- break
- }
- readsize := int64(header[7]) | (int64(header[6]) << 8) | (int64(header[5]) << 16) | (int64(header[4]) << 24)
- if header[0] == 1 {
- // stdout
- _, err = io.CopyN(runner.Stdout, containerReader, readsize)
- } else {
- // stderr
- _, err = io.CopyN(runner.Stderr, containerReader, readsize)
- }
- }
-
- if err != nil {
- runner.CrunchLog.Printf("error reading docker logs: %v", err)
- }
-
- err = runner.Stdout.Close()
- if err != nil {
- runner.CrunchLog.Printf("error closing stdout logs: %v", err)
- }
-
- err = runner.Stderr.Close()
- if err != nil {
- runner.CrunchLog.Printf("error closing stderr logs: %v", err)
- }
-
- if runner.statReporter != nil {
- runner.statReporter.Stop()
- err = runner.statLogger.Close()
- if err != nil {
- runner.CrunchLog.Printf("error closing crunchstat logs: %v", err)
- }
- }
+ return bindmounts, nil
}
func (runner *ContainerRunner) stopHoststat() error {
}
runner.statLogger = NewThrottledLogger(w)
runner.statReporter = &crunchstat.Reporter{
- CID: runner.ContainerID,
+ CID: runner.executor.CgroupID(),
Logger: log.New(runner.statLogger, "", 0),
CgroupParent: runner.expectCgroupParent,
CgroupRoot: runner.cgroupRoot,
return true, nil
}
-// AttachStreams connects the docker container stdin, stdout and stderr logs
-// to the Arvados logger which logs to Keep and the API server logs table.
-func (runner *ContainerRunner) AttachStreams() (err error) {
-
- runner.CrunchLog.Print("Attaching container streams")
-
- // If stdin mount is provided, attach it to the docker container
- var stdinRdr arvados.File
- var stdinJSON []byte
- if stdinMnt, ok := runner.Container.Mounts["stdin"]; ok {
- if stdinMnt.Kind == "collection" {
- var stdinColl arvados.Collection
- collID := stdinMnt.UUID
- if collID == "" {
- collID = stdinMnt.PortableDataHash
- }
- err = runner.ContainerArvClient.Get("collections", collID, nil, &stdinColl)
- if err != nil {
- return fmt.Errorf("While getting stdin collection: %v", err)
- }
-
- stdinRdr, err = runner.ContainerKeepClient.ManifestFileReader(
- manifest.Manifest{Text: stdinColl.ManifestText},
- stdinMnt.Path)
- if os.IsNotExist(err) {
- return fmt.Errorf("stdin collection path not found: %v", stdinMnt.Path)
- } else if err != nil {
- return fmt.Errorf("While getting stdin collection path %v: %v", stdinMnt.Path, err)
- }
- } else if stdinMnt.Kind == "json" {
- stdinJSON, err = json.Marshal(stdinMnt.Content)
- if err != nil {
- return fmt.Errorf("While encoding stdin json data: %v", err)
- }
- }
- }
-
- stdinUsed := stdinRdr != nil || len(stdinJSON) != 0
- response, err := runner.Docker.ContainerAttach(context.TODO(), runner.ContainerID,
- dockertypes.ContainerAttachOptions{Stream: true, Stdin: stdinUsed, Stdout: true, Stderr: true})
- if err != nil {
- return fmt.Errorf("While attaching container stdout/stderr streams: %v", err)
- }
-
- runner.loggingDone = make(chan bool)
-
- if stdoutMnt, ok := runner.Container.Mounts["stdout"]; ok {
- stdoutFile, err := runner.getStdoutFile(stdoutMnt.Path)
- if err != nil {
- return err
- }
- runner.Stdout = stdoutFile
- } else if w, err := runner.NewLogWriter("stdout"); err != nil {
- return err
- } else {
- runner.Stdout = NewThrottledLogger(w)
- }
-
- if stderrMnt, ok := runner.Container.Mounts["stderr"]; ok {
- stderrFile, err := runner.getStdoutFile(stderrMnt.Path)
- if err != nil {
- return err
- }
- runner.Stderr = stderrFile
- } else if w, err := runner.NewLogWriter("stderr"); err != nil {
- return err
- } else {
- runner.Stderr = NewThrottledLogger(w)
- }
-
- if stdinRdr != nil {
- go func() {
- _, err := io.Copy(response.Conn, stdinRdr)
- if err != nil {
- runner.CrunchLog.Printf("While writing stdin collection to docker container: %v", err)
- runner.stop(nil)
- }
- stdinRdr.Close()
- response.CloseWrite()
- }()
- } else if len(stdinJSON) != 0 {
- go func() {
- _, err := io.Copy(response.Conn, bytes.NewReader(stdinJSON))
- if err != nil {
- runner.CrunchLog.Printf("While writing stdin json to docker container: %v", err)
- runner.stop(nil)
- }
- response.CloseWrite()
- }()
- }
-
- go runner.ProcessDockerAttach(response.Reader)
-
- return nil
-}
-
func (runner *ContainerRunner) getStdoutFile(mntPath string) (*os.File, error) {
stdoutPath := mntPath[len(runner.Container.OutputPath):]
index := strings.LastIndex(stdoutPath, "/")
}
// CreateContainer creates the docker container.
-func (runner *ContainerRunner) CreateContainer() error {
- runner.CrunchLog.Print("Creating Docker container")
-
- runner.ContainerConfig.Cmd = runner.Container.Command
- if runner.Container.Cwd != "." {
- runner.ContainerConfig.WorkingDir = runner.Container.Cwd
+func (runner *ContainerRunner) CreateContainer(imageID string, bindmounts map[string]bindmount) error {
+ var stdin io.ReadCloser
+ if mnt, ok := runner.Container.Mounts["stdin"]; ok {
+ switch mnt.Kind {
+ case "collection":
+ var collID string
+ if mnt.UUID != "" {
+ collID = mnt.UUID
+ } else {
+ collID = mnt.PortableDataHash
+ }
+ path := runner.ArvMountPoint + "/by_id/" + collID + "/" + mnt.Path
+ f, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ stdin = f
+ case "json":
+ j, err := json.Marshal(mnt.Content)
+ if err != nil {
+ return fmt.Errorf("error encoding stdin json data: %v", err)
+ }
+ stdin = ioutil.NopCloser(bytes.NewReader(j))
+ default:
+ return fmt.Errorf("stdin mount has unsupported kind %q", mnt.Kind)
+ }
}
- for k, v := range runner.Container.Environment {
- runner.ContainerConfig.Env = append(runner.ContainerConfig.Env, k+"="+v)
+ var stdout, stderr io.WriteCloser
+ if mnt, ok := runner.Container.Mounts["stdout"]; ok {
+ f, err := runner.getStdoutFile(mnt.Path)
+ if err != nil {
+ return err
+ }
+ stdout = f
+ } else if w, err := runner.NewLogWriter("stdout"); err != nil {
+ return err
+ } else {
+ stdout = NewThrottledLogger(w)
}
- runner.ContainerConfig.Volumes = runner.Volumes
-
- maxRAM := int64(runner.Container.RuntimeConstraints.RAM)
- minDockerRAM := int64(16)
- if maxRAM < minDockerRAM*1024*1024 {
- // Docker daemon won't let you set a limit less than ~10 MiB
- maxRAM = minDockerRAM * 1024 * 1024
- }
- runner.HostConfig = dockercontainer.HostConfig{
- Binds: runner.Binds,
- LogConfig: dockercontainer.LogConfig{
- Type: "none",
- },
- Resources: dockercontainer.Resources{
- CgroupParent: runner.setCgroupParent,
- NanoCPUs: int64(runner.Container.RuntimeConstraints.VCPUs) * 1000000000,
- Memory: maxRAM, // RAM
- MemorySwap: maxRAM, // RAM+swap
- KernelMemory: maxRAM, // kernel portion
- },
+ if mnt, ok := runner.Container.Mounts["stderr"]; ok {
+ f, err := runner.getStdoutFile(mnt.Path)
+ if err != nil {
+ return err
+ }
+ stderr = f
+ } else if w, err := runner.NewLogWriter("stderr"); err != nil {
+ return err
+ } else {
+ stderr = NewThrottledLogger(w)
}
+ env := runner.Container.Environment
+ enableNetwork := runner.enableNetwork == "always"
if runner.Container.RuntimeConstraints.API {
+ enableNetwork = true
tok, err := runner.ContainerToken()
if err != nil {
return err
}
- runner.ContainerConfig.Env = append(runner.ContainerConfig.Env,
- "ARVADOS_API_TOKEN="+tok,
- "ARVADOS_API_HOST="+os.Getenv("ARVADOS_API_HOST"),
- "ARVADOS_API_HOST_INSECURE="+os.Getenv("ARVADOS_API_HOST_INSECURE"),
- )
- runner.HostConfig.NetworkMode = dockercontainer.NetworkMode(runner.networkMode)
- } else {
- if runner.enableNetwork == "always" {
- runner.HostConfig.NetworkMode = dockercontainer.NetworkMode(runner.networkMode)
- } else {
- runner.HostConfig.NetworkMode = dockercontainer.NetworkMode("none")
- }
- }
-
- _, stdinUsed := runner.Container.Mounts["stdin"]
- runner.ContainerConfig.OpenStdin = stdinUsed
- runner.ContainerConfig.StdinOnce = stdinUsed
- runner.ContainerConfig.AttachStdin = stdinUsed
- runner.ContainerConfig.AttachStdout = true
- runner.ContainerConfig.AttachStderr = true
-
- createdBody, err := runner.Docker.ContainerCreate(context.TODO(), &runner.ContainerConfig, &runner.HostConfig, nil, runner.Container.UUID)
- if err != nil {
- return fmt.Errorf("While creating container: %v", err)
- }
-
- runner.ContainerID = createdBody.ID
-
- return runner.AttachStreams()
+ env = map[string]string{}
+ for k, v := range runner.Container.Environment {
+ env[k] = v
+ }
+ env["ARVADOS_API_TOKEN"] = tok
+ env["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST")
+ env["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE")
+ }
+ workdir := runner.Container.Cwd
+ if workdir == "." {
+ // both "" and "." mean default
+ workdir = ""
+ }
+ ram := runner.Container.RuntimeConstraints.RAM
+ if !runner.enableMemoryLimit {
+ ram = 0
+ }
+ return runner.executor.Create(containerSpec{
+ Image: imageID,
+ VCPUs: runner.Container.RuntimeConstraints.VCPUs,
+ RAM: ram,
+ WorkingDir: workdir,
+ Env: env,
+ BindMounts: bindmounts,
+ Command: runner.Container.Command,
+ EnableNetwork: enableNetwork,
+ NetworkMode: runner.networkMode,
+ CgroupParent: runner.setCgroupParent,
+ Stdin: stdin,
+ Stdout: stdout,
+ Stderr: stderr,
+ })
}
// StartContainer starts the docker container created by CreateContainer.
func (runner *ContainerRunner) StartContainer() error {
- runner.CrunchLog.Printf("Starting Docker container id '%s'", runner.ContainerID)
+ runner.CrunchLog.Printf("Starting container")
runner.cStateLock.Lock()
defer runner.cStateLock.Unlock()
if runner.cCancelled {
return ErrCancelled
}
- err := runner.Docker.ContainerStart(context.TODO(), runner.ContainerID,
- dockertypes.ContainerStartOptions{})
+ err := runner.executor.Start()
if err != nil {
var advice string
if m, e := regexp.MatchString("(?ms).*(exec|System error).*(no such file or directory|file not found).*", err.Error()); m && e == nil {
// WaitFinish waits for the container to terminate, capture the exit code, and
// close the stdout/stderr logging.
func (runner *ContainerRunner) WaitFinish() error {
- var runTimeExceeded <-chan time.Time
runner.CrunchLog.Print("Waiting for container to finish")
-
- waitOk, waitErr := runner.Docker.ContainerWait(context.TODO(), runner.ContainerID, dockercontainer.WaitConditionNotRunning)
- arvMountExit := runner.ArvMountExit
- if timeout := runner.Container.SchedulingParameters.MaxRunTime; timeout > 0 {
- runTimeExceeded = time.After(time.Duration(timeout) * time.Second)
+ var timeout <-chan time.Time
+ if s := runner.Container.SchedulingParameters.MaxRunTime; s > 0 {
+ timeout = time.After(time.Duration(s) * time.Second)
}
-
- containerGone := make(chan struct{})
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
go func() {
- defer close(containerGone)
- if runner.containerWatchdogInterval < 1 {
- runner.containerWatchdogInterval = time.Minute
- }
- for range time.NewTicker(runner.containerWatchdogInterval).C {
- ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(runner.containerWatchdogInterval))
- ctr, err := runner.Docker.ContainerInspect(ctx, runner.ContainerID)
- cancel()
- runner.cStateLock.Lock()
- done := runner.cRemoved || runner.ExitCode != nil
- runner.cStateLock.Unlock()
- if done {
- return
- } else if err != nil {
- runner.CrunchLog.Printf("Error inspecting container: %s", err)
- runner.checkBrokenNode(err)
- return
- } else if ctr.State == nil || !(ctr.State.Running || ctr.State.Status == "created") {
- runner.CrunchLog.Printf("Container is not running: State=%v", ctr.State)
- return
- }
- }
- }()
-
- for {
select {
- case waitBody := <-waitOk:
- runner.CrunchLog.Printf("Container exited with code: %v", waitBody.StatusCode)
- code := int(waitBody.StatusCode)
- runner.ExitCode = &code
-
- // wait for stdout/stderr to complete
- <-runner.loggingDone
- return nil
-
- case err := <-waitErr:
- return fmt.Errorf("container wait: %v", err)
-
- case <-arvMountExit:
- runner.CrunchLog.Printf("arv-mount exited while container is still running. Stopping container.")
- runner.stop(nil)
- // arvMountExit will always be ready now that
- // it's closed, but that doesn't interest us.
- arvMountExit = nil
-
- case <-runTimeExceeded:
+ case <-timeout:
runner.CrunchLog.Printf("maximum run time exceeded. Stopping container.")
runner.stop(nil)
- runTimeExceeded = nil
+ case <-runner.ArvMountExit:
+ runner.CrunchLog.Printf("arv-mount exited while container is still running. Stopping container.")
+ runner.stop(nil)
+ case <-ctx.Done():
+ }
+ }()
+ exitcode, err := runner.executor.Wait(ctx)
+ if err != nil {
+ runner.checkBrokenNode(err)
+ return err
+ }
+ runner.ExitCode = &exitcode
- case <-containerGone:
- return errors.New("docker client never returned status")
+ if runner.statReporter != nil {
+ runner.statReporter.Stop()
+ err = runner.statLogger.Close()
+ if err != nil {
+ runner.CrunchLog.Printf("error closing crunchstat logs: %v", err)
}
}
+ return nil
}
func (runner *ContainerRunner) updateLogs() {
// CaptureOutput saves data from the container's output directory if
// needed, and updates the container output accordingly.
-func (runner *ContainerRunner) CaptureOutput() error {
+func (runner *ContainerRunner) CaptureOutput(bindmounts map[string]bindmount) error {
if runner.Container.RuntimeConstraints.API {
// Output may have been set directly by the container, so
// refresh the container record to check.
keepClient: runner.ContainerKeepClient,
hostOutputDir: runner.HostOutputDir,
ctrOutputDir: runner.Container.OutputPath,
- binds: runner.Binds,
+ bindmounts: bindmounts,
mounts: runner.Container.Mounts,
secretMounts: runner.SecretMounts,
logger: runner.CrunchLog,
return fmt.Errorf("dispatch error detected: container %q has state %q", runner.Container.UUID, runner.Container.State)
}
+ var bindmounts map[string]bindmount
defer func() {
// checkErr prints e (unless it's nil) and sets err to
// e (unless err is already non-nil). Thus, if err
// capture partial output and write logs
}
- checkErr("CaptureOutput", runner.CaptureOutput())
+ if bindmounts != nil {
+ checkErr("CaptureOutput", runner.CaptureOutput(bindmounts))
+ }
checkErr("stopHoststat", runner.stopHoststat())
checkErr("CommitLogs", runner.CommitLogs())
checkErr("UpdateContainerFinal", runner.UpdateContainerFinal())
return
}
+ // set up FUSE mount and binds
+ bindmounts, err = runner.SetupMounts()
+ if err != nil {
+ runner.finalState = "Cancelled"
+ err = fmt.Errorf("While setting up mounts: %v", err)
+ return
+ }
+
// check for and/or load image
- err = runner.LoadImage()
+ imageID, err := runner.LoadImage()
if err != nil {
if !runner.checkBrokenNode(err) {
// Failed to load image but not due to a "broken node"
return
}
- // set up FUSE mount and binds
- err = runner.SetupMounts()
- if err != nil {
- runner.finalState = "Cancelled"
- err = fmt.Errorf("While setting up mounts: %v", err)
- return
- }
-
- err = runner.CreateContainer()
+ err = runner.CreateContainer(imageID, bindmounts)
if err != nil {
return
}
func NewContainerRunner(dispatcherClient *arvados.Client,
dispatcherArvClient IArvadosClient,
dispatcherKeepClient IKeepClient,
- docker ThinDockerClient,
containerUUID string) (*ContainerRunner, error) {
cr := &ContainerRunner{
dispatcherClient: dispatcherClient,
DispatcherArvClient: dispatcherArvClient,
DispatcherKeepClient: dispatcherKeepClient,
- Docker: docker,
}
cr.NewLogWriter = cr.NewArvLogWriter
cr.RunArvMount = cr.ArvMountCmd
sleep := flags.Duration("sleep", 0, "Delay before starting (testing use only)")
kill := flags.Int("kill", -1, "Send signal to an existing crunch-run process for given UUID")
list := flags.Bool("list", false, "List UUIDs of existing crunch-run processes")
- enableNetwork := flags.String("container-enable-networking", "default",
- `Specify if networking should be enabled for container. One of 'default', 'always':
- default: only enable networking if container requests it.
- always: containers always have networking enabled
- `)
- networkMode := flags.String("container-network-mode", "default",
- `Set networking mode for container. Corresponds to Docker network mode (--net).
- `)
+ enableMemoryLimit := flags.Bool("enable-memory-limit", true, "tell container runtime to limit container's memory usage")
+ enableNetwork := flags.String("container-enable-networking", "default", "enable networking \"always\" (for all containers) or \"default\" (for containers that request it)")
+ networkMode := flags.String("container-network-mode", "default", `Docker network mode for container (use any argument valid for docker --net)`)
memprofile := flags.String("memprofile", "", "write memory profile to `file` after running container")
+ runtimeEngine := flags.String("runtime-engine", "docker", "container runtime: docker or singularity")
flags.Duration("check-containerd", 0, "Ignored. Exists for compatibility with older versions.")
ignoreDetachFlag := false
}
}
- containerID := flags.Arg(0)
+ containerUUID := flags.Arg(0)
switch {
case *detach && !ignoreDetachFlag:
- return Detach(containerID, prog, args, os.Stdout, os.Stderr)
+ return Detach(containerUUID, prog, args, os.Stdout, os.Stderr)
case *kill >= 0:
- return KillProcess(containerID, syscall.Signal(*kill), os.Stdout, os.Stderr)
+ return KillProcess(containerUUID, syscall.Signal(*kill), os.Stdout, os.Stderr)
case *list:
return ListProcesses(os.Stdout, os.Stderr)
}
- if containerID == "" {
+ if containerUUID == "" {
log.Printf("usage: %s [options] UUID", prog)
return 1
}
api, err := arvadosclient.MakeArvadosClient()
if err != nil {
- log.Printf("%s: %v", containerID, err)
+ log.Printf("%s: %v", containerUUID, err)
return 1
}
api.Retries = 8
kc, kcerr := keepclient.MakeKeepClient(api)
if kcerr != nil {
- log.Printf("%s: %v", containerID, kcerr)
+ log.Printf("%s: %v", containerUUID, kcerr)
return 1
}
kc.BlockCache = &keepclient.BlockCache{MaxBlocks: 2}
kc.Retries = 4
- // API version 1.21 corresponds to Docker 1.9, which is currently the
- // minimum version we want to support.
- docker, dockererr := dockerclient.NewClient(dockerclient.DefaultDockerHost, "1.21", nil, nil)
-
- cr, err := NewContainerRunner(arvados.NewClientFromEnv(), api, kc, docker, containerID)
+ cr, err := NewContainerRunner(arvados.NewClientFromEnv(), api, kc, containerUUID)
if err != nil {
log.Print(err)
return 1
}
- if dockererr != nil {
- cr.CrunchLog.Printf("%s: %v", containerID, dockererr)
- cr.checkBrokenNode(dockererr)
+
+ switch *runtimeEngine {
+ case "docker":
+ cr.executor, err = newDockerExecutor(containerUUID, cr.CrunchLog.Printf, cr.containerWatchdogInterval)
+ case "singularity":
+ cr.executor, err = newSingularityExecutor(cr.CrunchLog.Printf)
+ default:
+ cr.CrunchLog.Printf("%s: unsupported RuntimeEngine %q", containerUUID, *runtimeEngine)
cr.CrunchLog.Close()
return 1
}
-
- cr.gateway = Gateway{
- Address: os.Getenv("GatewayAddress"),
- AuthSecret: os.Getenv("GatewayAuthSecret"),
- ContainerUUID: containerID,
- DockerContainerID: &cr.ContainerID,
- Log: cr.CrunchLog,
- ContainerIPAddress: dockerContainerIPAddress(&cr.ContainerID),
+ if err != nil {
+ cr.CrunchLog.Printf("%s: %v", containerUUID, err)
+ cr.checkBrokenNode(err)
+ cr.CrunchLog.Close()
+ return 1
}
+ defer cr.executor.Close()
+
+ gwAuthSecret := os.Getenv("GatewayAuthSecret")
os.Unsetenv("GatewayAuthSecret")
- if cr.gateway.Address != "" {
+ if gwAuthSecret == "" {
+ // not safe to run a gateway service without an auth
+ // secret
+ cr.CrunchLog.Printf("Not starting a gateway server (GatewayAuthSecret was not provided by dispatcher)")
+ } else if gwListen := os.Getenv("GatewayAddress"); gwListen == "" {
+ // dispatcher did not tell us which external IP
+ // address to advertise --> no gateway service
+ cr.CrunchLog.Printf("Not starting a gateway server (GatewayAddress was not provided by dispatcher)")
+ } else if de, ok := cr.executor.(*dockerExecutor); ok {
+ cr.gateway = Gateway{
+ Address: gwListen,
+ AuthSecret: gwAuthSecret,
+ ContainerUUID: containerUUID,
+ DockerContainerID: &de.containerID,
+ Log: cr.CrunchLog,
+ ContainerIPAddress: dockerContainerIPAddress(&de.containerID),
+ }
err = cr.gateway.Start()
if err != nil {
log.Printf("error starting gateway server: %s", err)
}
}
- parentTemp, tmperr := cr.MkTempDir("", "crunch-run."+containerID+".")
+ parentTemp, tmperr := cr.MkTempDir("", "crunch-run."+containerUUID+".")
if tmperr != nil {
- log.Printf("%s: %v", containerID, tmperr)
+ log.Printf("%s: %v", containerUUID, tmperr)
return 1
}
cr.statInterval = *statInterval
cr.cgroupRoot = *cgroupRoot
cr.expectCgroupParent = *cgroupParent
+ cr.enableMemoryLimit = *enableMemoryLimit
cr.enableNetwork = *enableNetwork
cr.networkMode = *networkMode
if *cgroupParentSubsystem != "" {
}
if runerr != nil {
- log.Printf("%s: %v", containerID, runerr)
+ log.Printf("%s: %v", containerUUID, runerr)
return 1
}
return 0
package crunchrun
import (
- "bufio"
"bytes"
"crypto/md5"
"encoding/json"
"fmt"
"io"
"io/ioutil"
- "net"
"os"
"os/exec"
+ "regexp"
"runtime/pprof"
- "sort"
"strings"
"sync"
"syscall"
"git.arvados.org/arvados.git/sdk/go/manifest"
"golang.org/x/net/context"
- dockertypes "github.com/docker/docker/api/types"
- dockercontainer "github.com/docker/docker/api/types/container"
- dockernetwork "github.com/docker/docker/api/types/network"
. "gopkg.in/check.v1"
)
TestingT(t)
}
-// Gocheck boilerplate
var _ = Suite(&TestSuite{})
type TestSuite struct {
- client *arvados.Client
- docker *TestDockerClient
- runner *ContainerRunner
+ client *arvados.Client
+ api *ArvTestClient
+ runner *ContainerRunner
+ executor *stubExecutor
+ keepmount string
}
func (s *TestSuite) SetUpTest(c *C) {
+ *brokenNodeHook = ""
s.client = arvados.NewClientFromEnv()
- s.docker = NewTestDockerClient()
+ s.executor = &stubExecutor{}
+ var err error
+ s.api = &ArvTestClient{}
+ s.runner, err = NewContainerRunner(s.client, s.api, &KeepTestClient{}, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ c.Assert(err, IsNil)
+ s.runner.executor = s.executor
+ s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
+ return s.api, &KeepTestClient{}, s.client, nil
+ }
+ s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
+ s.runner.ArvMountPoint = s.keepmount
+ return nil, nil
+ }
+ s.keepmount = c.MkDir()
+ err = os.Mkdir(s.keepmount+"/by_id", 0755)
+ c.Assert(err, IsNil)
+ err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
+ c.Assert(err, IsNil)
+ err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
+ err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
+ c.Assert(err, IsNil)
+ err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
+ c.Assert(err, IsNil)
+ s.runner.ArvMountPoint = s.keepmount
}
type ArvTestClient struct {
Content []byte
}
+type stubExecutor struct {
+ imageLoaded bool
+ loaded string
+ loadErr error
+ exitCode int
+ createErr error
+ created containerSpec
+ startErr error
+ waitSleep time.Duration
+ waitErr error
+ stopErr error
+ stopped bool
+ closed bool
+ runFunc func()
+ exit chan int
+}
+
+func (e *stubExecutor) ImageLoaded(imageID string) bool { return e.imageLoaded }
+func (e *stubExecutor) LoadImage(filename string) error { e.loaded = filename; return e.loadErr }
+func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
+func (e *stubExecutor) Start() error { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
+func (e *stubExecutor) CgroupID() string { return "cgroupid" }
+func (e *stubExecutor) Stop() error { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
+func (e *stubExecutor) Close() { e.closed = true }
+func (e *stubExecutor) Wait(context.Context) (int, error) {
+ defer e.created.Stdout.Close()
+ defer e.created.Stderr.Close()
+ return <-e.exit, e.waitErr
+}
+
+const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
+
var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
-type TestDockerClient struct {
- imageLoaded string
- logReader io.ReadCloser
- logWriter io.WriteCloser
- fn func(t *TestDockerClient)
- exitCode int
- stop chan bool
- cwd string
- env []string
- api *ArvTestClient
- realTemp string
- calledWait bool
- ctrExited bool
-}
-
-func NewTestDockerClient() *TestDockerClient {
- t := &TestDockerClient{}
- t.logReader, t.logWriter = io.Pipe()
- t.stop = make(chan bool, 1)
- t.cwd = "/"
- return t
-}
-
-type MockConn struct {
- net.Conn
-}
-
-func (m *MockConn) Write(b []byte) (int, error) {
- return len(b), nil
-}
-
-func NewMockConn() *MockConn {
- c := &MockConn{}
- return c
-}
-
-func (t *TestDockerClient) ContainerAttach(ctx context.Context, container string, options dockertypes.ContainerAttachOptions) (dockertypes.HijackedResponse, error) {
- return dockertypes.HijackedResponse{Conn: NewMockConn(), Reader: bufio.NewReader(t.logReader)}, nil
-}
-
-func (t *TestDockerClient) ContainerCreate(ctx context.Context, config *dockercontainer.Config, hostConfig *dockercontainer.HostConfig, networkingConfig *dockernetwork.NetworkingConfig, containerName string) (dockercontainer.ContainerCreateCreatedBody, error) {
- if config.WorkingDir != "" {
- t.cwd = config.WorkingDir
- }
- t.env = config.Env
- return dockercontainer.ContainerCreateCreatedBody{ID: "abcde"}, nil
-}
-
-func (t *TestDockerClient) ContainerStart(ctx context.Context, container string, options dockertypes.ContainerStartOptions) error {
- if t.exitCode == 3 {
- return errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
- }
- if t.exitCode == 4 {
- return errors.New(`panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`)
- }
- if t.exitCode == 5 {
- return errors.New(`Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`)
- }
- if t.exitCode == 6 {
- return errors.New(`Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`)
- }
-
- if container == "abcde" {
- // t.fn gets executed in ContainerWait
- return nil
- }
- return errors.New("Invalid container id")
-}
-
-func (t *TestDockerClient) ContainerRemove(ctx context.Context, container string, options dockertypes.ContainerRemoveOptions) error {
- t.stop <- true
- return nil
-}
-
-func (t *TestDockerClient) ContainerWait(ctx context.Context, container string, condition dockercontainer.WaitCondition) (<-chan dockercontainer.ContainerWaitOKBody, <-chan error) {
- t.calledWait = true
- body := make(chan dockercontainer.ContainerWaitOKBody, 1)
- err := make(chan error)
- go func() {
- t.fn(t)
- body <- dockercontainer.ContainerWaitOKBody{StatusCode: int64(t.exitCode)}
- }()
- return body, err
-}
-
-func (t *TestDockerClient) ContainerInspect(ctx context.Context, id string) (c dockertypes.ContainerJSON, err error) {
- c.ContainerJSONBase = &dockertypes.ContainerJSONBase{}
- c.ID = "abcde"
- if t.ctrExited {
- c.State = &dockertypes.ContainerState{Status: "exited", Dead: true}
- } else {
- c.State = &dockertypes.ContainerState{Status: "running", Pid: 1234, Running: true}
- }
- return
-}
-
-func (t *TestDockerClient) ImageInspectWithRaw(ctx context.Context, image string) (dockertypes.ImageInspect, []byte, error) {
- if t.exitCode == 2 {
- return dockertypes.ImageInspect{}, nil, fmt.Errorf("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
- }
-
- if t.imageLoaded == image {
- return dockertypes.ImageInspect{}, nil, nil
- }
- return dockertypes.ImageInspect{}, nil, errors.New("")
-}
-
-func (t *TestDockerClient) ImageLoad(ctx context.Context, input io.Reader, quiet bool) (dockertypes.ImageLoadResponse, error) {
- if t.exitCode == 2 {
- return dockertypes.ImageLoadResponse{}, fmt.Errorf("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
- }
- _, err := io.Copy(ioutil.Discard, input)
- if err != nil {
- return dockertypes.ImageLoadResponse{}, err
- }
- t.imageLoaded = hwImageID
- return dockertypes.ImageLoadResponse{Body: ioutil.NopCloser(input)}, nil
-}
-
-func (*TestDockerClient) ImageRemove(ctx context.Context, image string, options dockertypes.ImageRemoveOptions) ([]dockertypes.ImageDeleteResponseItem, error) {
- return nil, nil
-}
-
func (client *ArvTestClient) Create(resourceType string,
parameters arvadosclient.Dict,
output interface{}) error {
} else {
j = []byte(`{
"command": ["sleep", "1"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "` + arvadostest.DockerImage112PDH + `",
"cwd": ".",
"environment": {},
"mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
}
func (s *TestSuite) TestLoadImage(c *C) {
- cr, err := NewContainerRunner(s.client, &ArvTestClient{},
- &KeepTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
- c.Assert(err, IsNil)
-
- kc := &KeepTestClient{}
- defer kc.Close()
- cr.ContainerArvClient = &ArvTestClient{}
- cr.ContainerKeepClient = kc
-
- _, err = cr.Docker.ImageRemove(nil, hwImageID, dockertypes.ImageRemoveOptions{})
- c.Check(err, IsNil)
-
- _, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageID)
- c.Check(err, NotNil)
-
- cr.Container.ContainerImage = hwPDH
-
- // (1) Test loading image from keep
- c.Check(kc.Called, Equals, false)
- c.Check(cr.ContainerConfig.Image, Equals, "")
-
- err = cr.LoadImage()
-
- c.Check(err, IsNil)
- defer func() {
- cr.Docker.ImageRemove(nil, hwImageID, dockertypes.ImageRemoveOptions{})
- }()
+ s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
+ s.runner.Container.Mounts = map[string]arvados.Mount{
+ "/out": {Kind: "tmp", Writable: true},
+ }
+ s.runner.Container.OutputPath = "/out"
- c.Check(kc.Called, Equals, true)
- c.Check(cr.ContainerConfig.Image, Equals, hwImageID)
+ _, err := s.runner.SetupMounts()
+ c.Assert(err, IsNil)
- _, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageID)
+ imageID, err := s.runner.LoadImage()
c.Check(err, IsNil)
-
- // (2) Test using image that's already loaded
- kc.Called = false
- cr.ContainerConfig.Image = ""
-
- err = cr.LoadImage()
+ c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
+ c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
+
+ s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
+ s.executor.imageLoaded = false
+ s.executor.loaded = ""
+ s.executor.loadErr = errors.New("bork")
+ imageID, err = s.runner.LoadImage()
+ c.Check(err, ErrorMatches, ".*bork")
+ c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
+
+ s.runner.Container.ContainerImage = fakeInputCollectionPDH
+ s.executor.imageLoaded = false
+ s.executor.loaded = ""
+ s.executor.loadErr = nil
+ imageID, err = s.runner.LoadImage()
+ c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
+ c.Check(s.executor.loaded, Equals, "")
+
+ // if executor reports image is already loaded, LoadImage should not be called
+ s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
+ s.executor.imageLoaded = true
+ s.executor.loaded = ""
+ s.executor.loadErr = nil
+ imageID, err = s.runner.LoadImage()
c.Check(err, IsNil)
- c.Check(kc.Called, Equals, false)
- c.Check(cr.ContainerConfig.Image, Equals, hwImageID)
-
+ c.Check(s.executor.loaded, Equals, "")
+ c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
}
type ArvErrorTestClient struct{}
return ErrorReader{}, nil
}
-func (s *TestSuite) TestLoadImageArvError(c *C) {
- // (1) Arvados error
- kc := &KeepTestClient{}
- defer kc.Close()
- cr, err := NewContainerRunner(s.client, &ArvErrorTestClient{}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
- c.Assert(err, IsNil)
-
- cr.ContainerArvClient = &ArvErrorTestClient{}
- cr.ContainerKeepClient = &KeepTestClient{}
-
- cr.Container.ContainerImage = hwPDH
-
- err = cr.LoadImage()
- c.Check(err.Error(), Equals, "While getting container image collection: ArvError")
-}
-
-func (s *TestSuite) TestLoadImageKeepError(c *C) {
- // (2) Keep error
- kc := &KeepErrorTestClient{}
- cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
- c.Assert(err, IsNil)
-
- cr.ContainerArvClient = &ArvTestClient{}
- cr.ContainerKeepClient = &KeepErrorTestClient{}
-
- cr.Container.ContainerImage = hwPDH
-
- err = cr.LoadImage()
- c.Assert(err, NotNil)
- c.Check(err.Error(), Equals, "While creating ManifestFileReader for container image: KeepError")
-}
-
-func (s *TestSuite) TestLoadImageCollectionError(c *C) {
- // (3) Collection doesn't contain image
- kc := &KeepReadErrorTestClient{}
- cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
- c.Assert(err, IsNil)
- cr.Container.ContainerImage = otherPDH
-
- cr.ContainerArvClient = &ArvTestClient{}
- cr.ContainerKeepClient = &KeepReadErrorTestClient{}
-
- err = cr.LoadImage()
- c.Check(err.Error(), Equals, "First file in the container image collection does not end in .tar")
-}
-
-func (s *TestSuite) TestLoadImageKeepReadError(c *C) {
- // (4) Collection doesn't contain image
- kc := &KeepReadErrorTestClient{}
- cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
- c.Assert(err, IsNil)
- cr.Container.ContainerImage = hwPDH
- cr.ContainerArvClient = &ArvTestClient{}
- cr.ContainerKeepClient = &KeepReadErrorTestClient{}
-
- err = cr.LoadImage()
- c.Check(err, NotNil)
-}
-
type ClosableBuffer struct {
bytes.Buffer
}
}
func (s *TestSuite) TestRunContainer(c *C) {
- s.docker.fn = func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, "Hello world\n"))
- t.logWriter.Close()
+ s.executor.runFunc = func() {
+ fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
+ s.executor.created.Stdout.Close()
+ s.executor.created.Stderr.Close()
+ s.executor.exit <- 0
}
- kc := &KeepTestClient{}
- defer kc.Close()
- cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
- c.Assert(err, IsNil)
-
- cr.ContainerArvClient = &ArvTestClient{}
- cr.ContainerKeepClient = &KeepTestClient{}
var logs TestLogs
- cr.NewLogWriter = logs.NewTestLoggingWriter
- cr.Container.ContainerImage = hwPDH
- cr.Container.Command = []string{"./hw"}
- err = cr.LoadImage()
- c.Check(err, IsNil)
+ s.runner.NewLogWriter = logs.NewTestLoggingWriter
+ s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
+ s.runner.Container.Command = []string{"./hw"}
- err = cr.CreateContainer()
- c.Check(err, IsNil)
+ imageID, err := s.runner.LoadImage()
+ c.Assert(err, IsNil)
- err = cr.StartContainer()
- c.Check(err, IsNil)
+ err = s.runner.CreateContainer(imageID, nil)
+ c.Assert(err, IsNil)
- err = cr.WaitFinish()
- c.Check(err, IsNil)
+ err = s.runner.StartContainer()
+ c.Assert(err, IsNil)
+
+ err = s.runner.WaitFinish()
+ c.Assert(err, IsNil)
- c.Check(strings.HasSuffix(logs.Stdout.String(), "Hello world\n"), Equals, true)
+ c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
c.Check(logs.Stderr.String(), Equals, "")
}
api := &ArvTestClient{}
kc := &KeepTestClient{}
defer kc.Close()
- cr, err := NewContainerRunner(s.client, api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
c.Assert(err, IsNil)
cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
api := &ArvTestClient{}
kc := &KeepTestClient{}
defer kc.Close()
- cr, err := NewContainerRunner(s.client, api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
c.Assert(err, IsNil)
err = cr.UpdateContainerRunning()
api := &ArvTestClient{}
kc := &KeepTestClient{}
defer kc.Close()
- cr, err := NewContainerRunner(s.client, api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
c.Assert(err, IsNil)
cr.LogsPDH = new(string)
api := &ArvTestClient{}
kc := &KeepTestClient{}
defer kc.Close()
- cr, err := NewContainerRunner(s.client, api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
c.Assert(err, IsNil)
cr.cCancelled = true
cr.finalState = "Cancelled"
// Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
// dress rehearsal of the Run() function, starting from a JSON container record.
-func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, realTemp string) {
- rec := arvados.Container{}
- err := json.Unmarshal([]byte(record), &rec)
- c.Check(err, IsNil)
+func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
+ err := json.Unmarshal([]byte(record), &s.api.Container)
+ c.Assert(err, IsNil)
+ initialState := s.api.Container.State
var sm struct {
SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
err = json.Unmarshal([]byte(record), &sm)
c.Check(err, IsNil)
secretMounts, err := json.Marshal(sm)
- c.Logf("%s %q", sm, secretMounts)
- c.Check(err, IsNil)
-
- s.docker.exitCode = exitCode
- s.docker.fn = fn
- s.docker.ImageRemove(nil, hwImageID, dockertypes.ImageRemoveOptions{})
-
- api = &ArvTestClient{Container: rec}
- s.docker.api = api
- kc := &KeepTestClient{}
- defer kc.Close()
- cr, err = NewContainerRunner(s.client, api, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
c.Assert(err, IsNil)
- s.runner = cr
- cr.statInterval = 100 * time.Millisecond
- cr.containerWatchdogInterval = time.Second
- am := &ArvMountCmdLine{}
- cr.RunArvMount = am.ArvMountTest
+ c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
- realTemp, err = ioutil.TempDir("", "crunchrun_test1-")
- c.Assert(err, IsNil)
- defer os.RemoveAll(realTemp)
+ s.executor.runFunc = func() {
+ fn()
+ s.executor.exit <- exitCode
+ }
- s.docker.realTemp = realTemp
+ s.runner.statInterval = 100 * time.Millisecond
+ s.runner.containerWatchdogInterval = time.Second
+ am := &ArvMountCmdLine{}
+ s.runner.RunArvMount = am.ArvMountTest
+ realTemp := c.MkDir()
tempcount := 0
- cr.MkTempDir = func(_ string, prefix string) (string, error) {
+ s.runner.MkTempDir = func(_, prefix string) (string, error) {
tempcount++
d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
err := os.Mkdir(d, os.ModePerm)
}
return d, err
}
- cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
+ s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
return &ArvTestClient{secretMounts: secretMounts}, &KeepTestClient{}, nil, nil
}
if extraMounts != nil && len(extraMounts) > 0 {
- err := cr.SetupArvMountPoint("keep")
+ err := s.runner.SetupArvMountPoint("keep")
c.Check(err, IsNil)
for _, m := range extraMounts {
- os.MkdirAll(cr.ArvMountPoint+"/by_id/"+m, os.ModePerm)
+ os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
}
}
- err = cr.Run()
- if api.CalledWith("container.state", "Complete") != nil {
+ err = s.runner.Run()
+ if s.api.CalledWith("container.state", "Complete") != nil {
c.Check(err, IsNil)
}
- if exitCode != 2 {
- c.Check(api.WasSetRunning, Equals, true)
+ if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
+ c.Check(s.api.WasSetRunning, Equals, true)
var lastupdate arvadosclient.Dict
- for _, content := range api.Content {
+ for _, content := range s.api.Content {
if content["container"] != nil {
lastupdate = content["container"].(arvadosclient.Dict)
}
}
if lastupdate["log"] == nil {
- c.Errorf("no container update with non-nil log -- updates were: %v", api.Content)
+ c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
}
}
if err != nil {
- for k, v := range api.Logs {
+ for k, v := range s.api.Logs {
c.Log(k)
c.Log(v.String())
}
}
- return
+ return s.api, s.runner, realTemp
}
func (s *TestSuite) TestFullRunHello(c *C) {
- api, _, _ := s.fullRunHelper(c, `{
+ s.runner.enableMemoryLimit = true
+ s.runner.networkMode = "default"
+ s.fullRunHelper(c, `{
"command": ["echo", "hello world"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "`+arvadostest.DockerImage112PDH+`",
"cwd": ".",
- "environment": {},
+ "environment": {"foo":"bar","baz":"waz"},
"mounts": {"/tmp": {"kind": "tmp"} },
"output_path": "/tmp",
"priority": 1,
- "runtime_constraints": {},
+ "runtime_constraints": {"vcpus":1,"ram":1000000},
"state": "Locked"
-}`, nil, 0, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, "hello world\n"))
- t.logWriter.Close()
+}`, nil, 0, func() {
+ c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
+ c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
+ c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
+ c.Check(s.executor.created.VCPUs, Equals, 1)
+ c.Check(s.executor.created.RAM, Equals, int64(1000000))
+ c.Check(s.executor.created.NetworkMode, Equals, "default")
+ c.Check(s.executor.created.EnableNetwork, Equals, false)
+ fmt.Fprintln(s.executor.created.Stdout, "hello world")
})
- c.Check(api.CalledWith("container.exit_code", 0), NotNil)
- c.Check(api.CalledWith("container.state", "Complete"), NotNil)
- c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "hello world\n"), Equals, true)
+ c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
+ c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
}
func (s *TestSuite) TestRunAlreadyRunning(c *C) {
var ran bool
- api, _, _ := s.fullRunHelper(c, `{
+ s.fullRunHelper(c, `{
"command": ["sleep", "3"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "`+arvadostest.DockerImage112PDH+`",
"cwd": ".",
"environment": {},
"mounts": {"/tmp": {"kind": "tmp"} },
"runtime_constraints": {},
"scheduling_parameters":{"max_run_time": 1},
"state": "Running"
-}`, nil, 2, func(t *TestDockerClient) {
+}`, nil, 2, func() {
ran = true
})
-
- c.Check(api.CalledWith("container.state", "Cancelled"), IsNil)
- c.Check(api.CalledWith("container.state", "Complete"), IsNil)
+ c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
+ c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
c.Check(ran, Equals, false)
}
func (s *TestSuite) TestRunTimeExceeded(c *C) {
- api, _, _ := s.fullRunHelper(c, `{
+ s.fullRunHelper(c, `{
"command": ["sleep", "3"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "`+arvadostest.DockerImage112PDH+`",
"cwd": ".",
"environment": {},
"mounts": {"/tmp": {"kind": "tmp"} },
"runtime_constraints": {},
"scheduling_parameters":{"max_run_time": 1},
"state": "Locked"
-}`, nil, 0, func(t *TestDockerClient) {
+}`, nil, 0, func() {
time.Sleep(3 * time.Second)
- t.logWriter.Close()
})
- c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
- c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
+ c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
+ c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
}
func (s *TestSuite) TestContainerWaitFails(c *C) {
- api, _, _ := s.fullRunHelper(c, `{
+ s.fullRunHelper(c, `{
"command": ["sleep", "3"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "`+arvadostest.DockerImage112PDH+`",
"cwd": ".",
"mounts": {"/tmp": {"kind": "tmp"} },
"output_path": "/tmp",
"priority": 1,
"state": "Locked"
-}`, nil, 0, func(t *TestDockerClient) {
- t.ctrExited = true
- time.Sleep(10 * time.Second)
- t.logWriter.Close()
+}`, nil, 0, func() {
+ s.executor.waitErr = errors.New("Container is not running")
})
- c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
- c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
+ c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
+ c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
}
func (s *TestSuite) TestCrunchstat(c *C) {
- api, _, _ := s.fullRunHelper(c, `{
+ s.fullRunHelper(c, `{
"command": ["sleep", "1"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "`+arvadostest.DockerImage112PDH+`",
"cwd": ".",
"environment": {},
"mounts": {"/tmp": {"kind": "tmp"} },
"priority": 1,
"runtime_constraints": {},
"state": "Locked"
- }`, nil, 0, func(t *TestDockerClient) {
+ }`, nil, 0, func() {
time.Sleep(time.Second)
- t.logWriter.Close()
})
- c.Check(api.CalledWith("container.exit_code", 0), NotNil)
- c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+ c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
// We didn't actually start a container, so crunchstat didn't
// find accounting files and therefore didn't log any stats.
// It should have logged a "can't find accounting files"
// message after one poll interval, though, so we can confirm
// it's alive:
- c.Assert(api.Logs["crunchstat"], NotNil)
- c.Check(api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
+ c.Assert(s.api.Logs["crunchstat"], NotNil)
+ c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
// The "files never appeared" log assures us that we called
// (*crunchstat.Reporter)Stop(), and that we set it up with
// the correct container ID "abcde":
- c.Check(api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for abcde\n`)
+ c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
}
func (s *TestSuite) TestNodeInfoLog(c *C) {
os.Setenv("SLURMD_NODENAME", "compute2")
- api, _, _ := s.fullRunHelper(c, `{
+ s.fullRunHelper(c, `{
"command": ["sleep", "1"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "`+arvadostest.DockerImage112PDH+`",
"cwd": ".",
"environment": {},
"mounts": {"/tmp": {"kind": "tmp"} },
"runtime_constraints": {},
"state": "Locked"
}`, nil, 0,
- func(t *TestDockerClient) {
+ func() {
time.Sleep(time.Second)
- t.logWriter.Close()
})
- c.Check(api.CalledWith("container.exit_code", 0), NotNil)
- c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+ c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
- c.Assert(api.Logs["node"], NotNil)
- json := api.Logs["node"].String()
+ c.Assert(s.api.Logs["node"], NotNil)
+ json := s.api.Logs["node"].String()
c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
c.Check(json, Not(Matches), `(?ms).*"info":.*`)
- c.Assert(api.Logs["node-info"], NotNil)
- json = api.Logs["node-info"].String()
+ c.Assert(s.api.Logs["node-info"], NotNil)
+ json = s.api.Logs["node-info"].String()
c.Check(json, Matches, `(?ms).*Host Information.*`)
c.Check(json, Matches, `(?ms).*CPU Information.*`)
c.Check(json, Matches, `(?ms).*Memory Information.*`)
}
func (s *TestSuite) TestContainerRecordLog(c *C) {
- api, _, _ := s.fullRunHelper(c, `{
+ s.fullRunHelper(c, `{
"command": ["sleep", "1"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "`+arvadostest.DockerImage112PDH+`",
"cwd": ".",
"environment": {},
"mounts": {"/tmp": {"kind": "tmp"} },
"runtime_constraints": {},
"state": "Locked"
}`, nil, 0,
- func(t *TestDockerClient) {
+ func() {
time.Sleep(time.Second)
- t.logWriter.Close()
})
- c.Check(api.CalledWith("container.exit_code", 0), NotNil)
- c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+ c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
- c.Assert(api.Logs["container"], NotNil)
- c.Check(api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
+ c.Assert(s.api.Logs["container"], NotNil)
+ c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
}
func (s *TestSuite) TestFullRunStderr(c *C) {
- api, _, _ := s.fullRunHelper(c, `{
+ s.fullRunHelper(c, `{
"command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "`+arvadostest.DockerImage112PDH+`",
"cwd": ".",
"environment": {},
"mounts": {"/tmp": {"kind": "tmp"} },
"priority": 1,
"runtime_constraints": {},
"state": "Locked"
-}`, nil, 1, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, "hello\n"))
- t.logWriter.Write(dockerLog(2, "world\n"))
- t.logWriter.Close()
+}`, nil, 1, func() {
+ fmt.Fprintln(s.executor.created.Stdout, "hello")
+ fmt.Fprintln(s.executor.created.Stderr, "world")
})
- final := api.CalledWith("container.state", "Complete")
+ final := s.api.CalledWith("container.state", "Complete")
c.Assert(final, NotNil)
c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
- c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "hello\n"), Equals, true)
- c.Check(strings.HasSuffix(api.Logs["stderr"].String(), "world\n"), Equals, true)
+ c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
+ c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
}
func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
- api, _, _ := s.fullRunHelper(c, `{
+ s.fullRunHelper(c, `{
"command": ["pwd"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "`+arvadostest.DockerImage112PDH+`",
"cwd": ".",
"environment": {},
"mounts": {"/tmp": {"kind": "tmp"} },
"priority": 1,
"runtime_constraints": {},
"state": "Locked"
-}`, nil, 0, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
- t.logWriter.Close()
+}`, nil, 0, func() {
+ fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
})
- c.Check(api.CalledWith("container.exit_code", 0), NotNil)
- c.Check(api.CalledWith("container.state", "Complete"), NotNil)
- c.Log(api.Logs["stdout"])
- c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "/\n"), Equals, true)
+ c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
+ c.Log(s.api.Logs["stdout"])
+ c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
}
func (s *TestSuite) TestFullRunSetCwd(c *C) {
- api, _, _ := s.fullRunHelper(c, `{
+ s.fullRunHelper(c, `{
"command": ["pwd"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "`+arvadostest.DockerImage112PDH+`",
"cwd": "/bin",
"environment": {},
"mounts": {"/tmp": {"kind": "tmp"} },
"priority": 1,
"runtime_constraints": {},
"state": "Locked"
-}`, nil, 0, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
- t.logWriter.Close()
+}`, nil, 0, func() {
+ fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
})
- c.Check(api.CalledWith("container.exit_code", 0), NotNil)
- c.Check(api.CalledWith("container.state", "Complete"), NotNil)
- c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "/bin\n"), Equals, true)
+ c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
+ c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
}
func (s *TestSuite) TestStopOnSignal(c *C) {
- s.testStopContainer(c, func(cr *ContainerRunner) {
- go func() {
- for !s.docker.calledWait {
- time.Sleep(time.Millisecond)
- }
- cr.SigChan <- syscall.SIGINT
- }()
- })
+ s.executor.runFunc = func() {
+ s.executor.created.Stdout.Write([]byte("foo\n"))
+ s.runner.SigChan <- syscall.SIGINT
+ }
+ s.testStopContainer(c)
}
func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
- s.testStopContainer(c, func(cr *ContainerRunner) {
- cr.ArvMountExit = make(chan error)
- go func() {
- cr.ArvMountExit <- exec.Command("true").Run()
- close(cr.ArvMountExit)
- }()
- })
+ s.executor.runFunc = func() {
+ s.executor.created.Stdout.Write([]byte("foo\n"))
+ s.runner.ArvMountExit <- nil
+ close(s.runner.ArvMountExit)
+ }
+ s.runner.ArvMountExit = make(chan error)
+ s.testStopContainer(c)
}
-func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) {
+func (s *TestSuite) testStopContainer(c *C) {
record := `{
"command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "` + arvadostest.DockerImage112PDH + `",
"cwd": ".",
"environment": {},
"mounts": {"/tmp": {"kind": "tmp"} },
"state": "Locked"
}`
- rec := arvados.Container{}
- err := json.Unmarshal([]byte(record), &rec)
- c.Check(err, IsNil)
-
- s.docker.fn = func(t *TestDockerClient) {
- <-t.stop
- t.logWriter.Write(dockerLog(1, "foo\n"))
- t.logWriter.Close()
- }
- s.docker.ImageRemove(nil, hwImageID, dockertypes.ImageRemoveOptions{})
-
- api := &ArvTestClient{Container: rec}
- kc := &KeepTestClient{}
- defer kc.Close()
- cr, err := NewContainerRunner(s.client, api, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ err := json.Unmarshal([]byte(record), &s.api.Container)
c.Assert(err, IsNil)
- cr.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
- cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
+
+ s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
+ s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
return &ArvTestClient{}, &KeepTestClient{}, nil, nil
}
- setup(cr)
done := make(chan error)
go func() {
- done <- cr.Run()
+ done <- s.runner.Run()
}()
select {
case <-time.After(20 * time.Second):
case err = <-done:
c.Check(err, IsNil)
}
- for k, v := range api.Logs {
+ for k, v := range s.api.Logs {
c.Log(k)
- c.Log(v.String())
+ c.Log(v.String(), "\n")
}
- c.Check(api.CalledWith("container.log", nil), NotNil)
- c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
- c.Check(api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
+ c.Check(s.api.CalledWith("container.log", nil), NotNil)
+ c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
+ c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
}
func (s *TestSuite) TestFullRunSetEnv(c *C) {
- api, _, _ := s.fullRunHelper(c, `{
+ s.fullRunHelper(c, `{
"command": ["/bin/sh", "-c", "echo $FROBIZ"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "`+arvadostest.DockerImage112PDH+`",
"cwd": "/bin",
"environment": {"FROBIZ": "bilbo"},
"mounts": {"/tmp": {"kind": "tmp"} },
"priority": 1,
"runtime_constraints": {},
"state": "Locked"
-}`, nil, 0, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
- t.logWriter.Close()
+}`, nil, 0, func() {
+ fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
})
- c.Check(api.CalledWith("container.exit_code", 0), NotNil)
- c.Check(api.CalledWith("container.state", "Complete"), NotNil)
- c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "bilbo\n"), Equals, true)
+ c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
+ c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
}
type ArvMountCmdLine struct {
}
func (s *TestSuite) TestSetupMounts(c *C) {
- api := &ArvTestClient{}
- kc := &KeepTestClient{}
- defer kc.Close()
- cr, err := NewContainerRunner(s.client, api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
- c.Assert(err, IsNil)
+ cr := s.runner
am := &ArvMountCmdLine{}
cr.RunArvMount = am.ArvMountTest
cr.ContainerArvClient = &ArvTestClient{}
cr.ContainerKeepClient = &KeepTestClient{}
- realTemp, err := ioutil.TempDir("", "crunchrun_test1-")
- c.Assert(err, IsNil)
- certTemp, err := ioutil.TempDir("", "crunchrun_test2-")
- c.Assert(err, IsNil)
+ realTemp := c.MkDir()
+ certTemp := c.MkDir()
stubCertPath := stubCert(certTemp)
-
cr.parentTemp = realTemp
- defer os.RemoveAll(realTemp)
- defer os.RemoveAll(certTemp)
-
i := 0
cr.MkTempDir = func(_ string, prefix string) (string, error) {
i++
cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
cr.Container.OutputPath = "/tmp"
cr.statInterval = 5 * time.Second
- err := cr.SetupMounts()
+ bindmounts, err := cr.SetupMounts()
c.Check(err, IsNil)
c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
"--read-write", "--crunchstat-interval=5",
"--mount-by-pdh", "by_id", realTemp + "/keep1"})
- c.Check(cr.Binds, DeepEquals, []string{realTemp + "/tmp2:/tmp"})
+ c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
checkEmpty()
cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
cr.Container.OutputPath = "/out"
- err := cr.SetupMounts()
+ bindmounts, err := cr.SetupMounts()
c.Check(err, IsNil)
c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
"--read-write", "--crunchstat-interval=5",
"--mount-by-pdh", "by_id", realTemp + "/keep1"})
- c.Check(cr.Binds, DeepEquals, []string{realTemp + "/tmp2:/out", realTemp + "/tmp3:/tmp"})
+ c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
checkEmpty()
cr.Container.OutputPath = "/tmp"
cr.Container.RuntimeConstraints.API = true
- err := cr.SetupMounts()
+ bindmounts, err := cr.SetupMounts()
c.Check(err, IsNil)
c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
"--read-write", "--crunchstat-interval=5",
"--mount-by-pdh", "by_id", realTemp + "/keep1"})
- c.Check(cr.Binds, DeepEquals, []string{realTemp + "/tmp2:/tmp", stubCertPath + ":/etc/arvados/ca-certificates.crt:ro"})
+ c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
checkEmpty()
os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
- err := cr.SetupMounts()
+ bindmounts, err := cr.SetupMounts()
c.Check(err, IsNil)
c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
"--read-write", "--crunchstat-interval=5",
"--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
- c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/tmp0:/keeptmp"})
+ c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
checkEmpty()
os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
- err := cr.SetupMounts()
+ bindmounts, err := cr.SetupMounts()
c.Check(err, IsNil)
c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
"--read-write", "--crunchstat-interval=5",
"--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
- sort.StringSlice(cr.Binds).Sort()
- c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53:/keepinp:ro",
- realTemp + "/keep1/tmp0:/keepout"})
+ c.Check(bindmounts, DeepEquals, map[string]bindmount{
+ "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
+ "/keepout": {realTemp + "/keep1/tmp0", false},
+ })
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
checkEmpty()
os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
- err := cr.SetupMounts()
+ bindmounts, err := cr.SetupMounts()
c.Check(err, IsNil)
c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
"--read-write", "--crunchstat-interval=5",
"--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
- sort.StringSlice(cr.Binds).Sort()
- c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53:/keepinp:ro",
- realTemp + "/keep1/tmp0:/keepout"})
+ c.Check(bindmounts, DeepEquals, map[string]bindmount{
+ "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
+ "/keepout": {realTemp + "/keep1/tmp0", false},
+ })
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
checkEmpty()
cr.Container.Mounts = map[string]arvados.Mount{
"/mnt/test.json": {Kind: "json", Content: test.in},
}
- err := cr.SetupMounts()
+ bindmounts, err := cr.SetupMounts()
c.Check(err, IsNil)
- sort.StringSlice(cr.Binds).Sort()
- c.Check(cr.Binds, DeepEquals, []string{realTemp + "/json2/mountdata.json:/mnt/test.json:ro"})
+ c.Check(bindmounts, DeepEquals, map[string]bindmount{
+ "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
+ })
content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
c.Check(err, IsNil)
c.Check(content, DeepEquals, []byte(test.out))
cr.Container.Mounts = map[string]arvados.Mount{
"/mnt/test.txt": {Kind: "text", Content: test.in},
}
- err := cr.SetupMounts()
+ bindmounts, err := cr.SetupMounts()
if test.out == "error" {
c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
} else {
c.Check(err, IsNil)
- sort.StringSlice(cr.Binds).Sort()
- c.Check(cr.Binds, DeepEquals, []string{realTemp + "/text2/mountdata.text:/mnt/test.txt:ro"})
+ c.Check(bindmounts, DeepEquals, map[string]bindmount{
+ "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
+ })
content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
c.Check(err, IsNil)
c.Check(content, DeepEquals, []byte(test.out))
os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
- err := cr.SetupMounts()
+ bindmounts, err := cr.SetupMounts()
c.Check(err, IsNil)
c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
"--read-write", "--crunchstat-interval=5",
"--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
- c.Check(cr.Binds, DeepEquals, []string{realTemp + "/tmp2:/tmp", realTemp + "/keep1/tmp0:/tmp/foo:ro"})
+ c.Check(bindmounts, DeepEquals, map[string]bindmount{
+ "/tmp": {realTemp + "/tmp2", false},
+ "/tmp/foo": {realTemp + "/keep1/tmp0", true},
+ })
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
checkEmpty()
rf.Write([]byte("bar"))
rf.Close()
- err := cr.SetupMounts()
+ _, err := cr.SetupMounts()
c.Check(err, IsNil)
_, err = os.Stat(cr.HostOutputDir + "/foo")
c.Check(err, IsNil)
}
cr.Container.OutputPath = "/tmp"
- err := cr.SetupMounts()
+ _, err := cr.SetupMounts()
c.Check(err, NotNil)
c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
os.RemoveAll(cr.ArvMountPoint)
"stdin": {Kind: "tmp"},
}
- err := cr.SetupMounts()
+ _, err := cr.SetupMounts()
c.Check(err, NotNil)
c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
os.RemoveAll(cr.ArvMountPoint)
}
cr.Container.OutputPath = "/tmp"
- err := cr.SetupMounts()
+ bindmounts, err := cr.SetupMounts()
c.Check(err, IsNil)
- // dirMap[mountpoint] == tmpdir
- dirMap := make(map[string]string)
- for _, bind := range cr.Binds {
- tokens := strings.Split(bind, ":")
- dirMap[tokens[1]] = tokens[0]
-
- if cr.Container.Mounts[tokens[1]].Writable {
- c.Check(len(tokens), Equals, 2)
- } else {
- c.Check(len(tokens), Equals, 3)
- c.Check(tokens[2], Equals, "ro")
- }
+ for path, mount := range bindmounts {
+ c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
}
- data, err := ioutil.ReadFile(dirMap["/tip"] + "/dir1/dir2/file with mode 0644")
+ data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
c.Check(err, IsNil)
c.Check(string(data), Equals, "\000\001\002\003")
- _, err = ioutil.ReadFile(dirMap["/tip"] + "/file only on testbranch")
+ _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
c.Check(err, FitsTypeOf, &os.PathError{})
c.Check(os.IsNotExist(err), Equals, true)
- data, err = ioutil.ReadFile(dirMap["/non-tip"] + "/dir1/dir2/file with mode 0644")
+ data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
c.Check(err, IsNil)
c.Check(string(data), Equals, "\000\001\002\003")
- data, err = ioutil.ReadFile(dirMap["/non-tip"] + "/file only on testbranch")
+ data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
c.Check(err, IsNil)
c.Check(string(data), Equals, "testfile\n")
func (s *TestSuite) TestStdout(c *C) {
helperRecord := `{
"command": ["/bin/sh", "-c", "echo $FROBIZ"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "` + arvadostest.DockerImage112PDH + `",
"cwd": "/bin",
"environment": {"FROBIZ": "bilbo"},
"mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
"state": "Locked"
}`
- api, cr, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
- t.logWriter.Close()
+ s.fullRunHelper(c, helperRecord, nil, 0, func() {
+ fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
})
- c.Check(api.CalledWith("container.exit_code", 0), NotNil)
- c.Check(api.CalledWith("container.state", "Complete"), NotNil)
- c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
+ c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
+ c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
}
// Used by the TestStdoutWithWrongPath*()
-func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, err error) {
- rec := arvados.Container{}
- err = json.Unmarshal([]byte(record), &rec)
- c.Check(err, IsNil)
-
- s.docker.fn = fn
- s.docker.ImageRemove(nil, hwImageID, dockertypes.ImageRemoveOptions{})
-
- api = &ArvTestClient{Container: rec}
- kc := &KeepTestClient{}
- defer kc.Close()
- cr, err = NewContainerRunner(s.client, api, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
+ err := json.Unmarshal([]byte(record), &s.api.Container)
c.Assert(err, IsNil)
- am := &ArvMountCmdLine{}
- cr.RunArvMount = am.ArvMountTest
- cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
- return &ArvTestClient{}, &KeepTestClient{}, nil, nil
+ s.executor.runFunc = fn
+ s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
+ s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
+ return s.api, &KeepTestClient{}, nil, nil
}
-
- err = cr.Run()
- return
+ return s.api, s.runner, s.runner.Run()
}
func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
"mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
"output_path": "/tmp",
"state": "Locked"
-}`, func(t *TestDockerClient) {})
-
- c.Check(err, NotNil)
- c.Check(strings.Contains(err.Error(), "Stdout path does not start with OutputPath"), Equals, true)
+}`, func() {})
+ c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
}
func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
"mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
"output_path": "/tmp",
"state": "Locked"
-}`, func(t *TestDockerClient) {})
-
- c.Check(err, NotNil)
- c.Check(strings.Contains(err.Error(), "unsupported mount kind 'tmp' for stdout"), Equals, true)
+}`, func() {})
+ c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
}
func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
"mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
"output_path": "/tmp",
"state": "Locked"
-}`, func(t *TestDockerClient) {})
-
- c.Check(err, NotNil)
- c.Check(strings.Contains(err.Error(), "unsupported mount kind 'collection' for stdout"), Equals, true)
+}`, func() {})
+ c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
}
func (s *TestSuite) TestFullRunWithAPI(c *C) {
- defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
- os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
- api, _, _ := s.fullRunHelper(c, `{
- "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ s.fullRunHelper(c, `{
+ "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
+ "container_image": "`+arvadostest.DockerImage112PDH+`",
"cwd": "/bin",
"environment": {},
"mounts": {"/tmp": {"kind": "tmp"} },
"priority": 1,
"runtime_constraints": {"API": true},
"state": "Locked"
-}`, nil, 0, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, t.env[1][17:]+"\n"))
- t.logWriter.Close()
+}`, nil, 0, func() {
+ c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
+ s.executor.exit <- 3
})
-
- c.Check(api.CalledWith("container.exit_code", 0), NotNil)
- c.Check(api.CalledWith("container.state", "Complete"), NotNil)
- c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "test.arvados.org\n"), Equals, true)
- c.Check(api.CalledWith("container.output", "d41d8cd98f00b204e9800998ecf8427e+0"), NotNil)
+ c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
+ c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
}
func (s *TestSuite) TestFullRunSetOutput(c *C) {
defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
- api, _, _ := s.fullRunHelper(c, `{
+ s.fullRunHelper(c, `{
"command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "`+arvadostest.DockerImage112PDH+`",
"cwd": "/bin",
"environment": {},
"mounts": {"/tmp": {"kind": "tmp"} },
"priority": 1,
"runtime_constraints": {"API": true},
"state": "Locked"
-}`, nil, 0, func(t *TestDockerClient) {
- t.api.Container.Output = "d4ab34d3d4f8a72f5c4973051ae69fab+122"
- t.logWriter.Close()
+}`, nil, 0, func() {
+ s.api.Container.Output = arvadostest.DockerImage112PDH
})
- c.Check(api.CalledWith("container.exit_code", 0), NotNil)
- c.Check(api.CalledWith("container.state", "Complete"), NotNil)
- c.Check(api.CalledWith("container.output", "d4ab34d3d4f8a72f5c4973051ae69fab+122"), NotNil)
+ c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
+ c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
}
func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
helperRecord := `{
"command": ["/bin/sh", "-c", "echo $FROBIZ"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "` + arvadostest.DockerImage112PDH + `",
"cwd": "/bin",
"environment": {"FROBIZ": "bilbo"},
"mounts": {
extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
- api, cr, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
- t.logWriter.Close()
+ s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
+ fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
})
- c.Check(api.CalledWith("container.exit_code", 0), NotNil)
- c.Check(api.CalledWith("container.state", "Complete"), NotNil)
- c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
+ c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
+ c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
}
func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
helperRecord := `{
"command": ["/bin/sh", "-c", "echo $FROBIZ"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "` + arvadostest.DockerImage112PDH + `",
"cwd": "/bin",
"environment": {"FROBIZ": "bilbo"},
"mounts": {
"a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
}
- api, runner, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
- t.logWriter.Close()
+ api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
+ fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
})
- c.Check(runner.Binds, DeepEquals, []string{realtemp + "/tmp2:/tmp",
- realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt:/tmp/foo/bar:ro",
- realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt:/tmp/foo/baz/sub2file2:ro",
- realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1:/tmp/foo/sub1:ro",
- realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt:/tmp/foo/sub1file2:ro",
+ c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
+ "/tmp": {realtemp + "/tmp1", false},
+ "/tmp/foo/bar": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
+ "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
+ "/tmp/foo/sub1": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
+ "/tmp/foo/sub1file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
})
c.Check(api.CalledWith("container.exit_code", 0), NotNil)
func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
helperRecord := `{
"command": ["/bin/sh", "-c", "echo $FROBIZ"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "` + arvadostest.DockerImage112PDH + `",
"cwd": "/bin",
"environment": {"FROBIZ": "bilbo"},
"mounts": {
"b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
}
- api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
- t.logWriter.Close()
+ s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
+ fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
})
- c.Check(api.CalledWith("container.exit_code", 0), NotNil)
- c.Check(api.CalledWith("container.state", "Complete"), NotNil)
- for _, v := range api.Content {
+ c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
+ for _, v := range s.api.Content {
if v["collection"] != nil {
collection := v["collection"].(arvadosclient.Dict)
if strings.Index(collection["name"].(string), "output") == 0 {
func (s *TestSuite) TestOutputError(c *C) {
helperRecord := `{
"command": ["/bin/sh", "-c", "echo $FROBIZ"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "` + arvadostest.DockerImage112PDH + `",
"cwd": "/bin",
"environment": {"FROBIZ": "bilbo"},
"mounts": {
"runtime_constraints": {},
"state": "Locked"
}`
-
- extraMounts := []string{}
-
- api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
- os.Symlink("/etc/hosts", t.realTemp+"/tmp2/baz")
- t.logWriter.Close()
+ s.fullRunHelper(c, helperRecord, nil, 0, func() {
+ os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
})
- c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
+ c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
}
func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
helperRecord := `{
"command": ["/bin/sh", "-c", "echo $FROBIZ"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "` + arvadostest.DockerImage112PDH + `",
"cwd": "/bin",
"environment": {"FROBIZ": "bilbo"},
"mounts": {
"b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
}
- api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
- t.logWriter.Close()
+ api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
+ fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
})
c.Check(api.CalledWith("container.exit_code", 0), NotNil)
func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
helperRecord := `{
"command": ["/bin/sh", "-c", "echo $FROBIZ"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "` + arvadostest.DockerImage112PDH + `",
"cwd": "/bin",
"environment": {"FROBIZ": "bilbo"},
"mounts": {
"state": "Locked"
}`
- api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
- t.logWriter.Close()
+ api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
+ fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
})
c.Check(api.CalledWith("container.exit_code", 0), NotNil)
func (s *TestSuite) TestStderrMount(c *C) {
api, cr, _ := s.fullRunHelper(c, `{
"command": ["/bin/sh", "-c", "echo hello;exit 1"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "`+arvadostest.DockerImage112PDH+`",
"cwd": ".",
"environment": {},
"mounts": {"/tmp": {"kind": "tmp"},
"priority": 1,
"runtime_constraints": {},
"state": "Locked"
-}`, nil, 1, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, "hello\n"))
- t.logWriter.Write(dockerLog(2, "oops\n"))
- t.logWriter.Close()
+}`, nil, 1, func() {
+ fmt.Fprintln(s.executor.created.Stdout, "hello")
+ fmt.Fprintln(s.executor.created.Stderr, "oops")
})
final := api.CalledWith("container.state", "Complete")
}
func (s *TestSuite) TestNumberRoundTrip(c *C) {
- kc := &KeepTestClient{}
- defer kc.Close()
- cr, err := NewContainerRunner(s.client, &ArvTestClient{callraw: true}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ s.api.callraw = true
+ err := s.runner.fetchContainerRecord()
c.Assert(err, IsNil)
- cr.fetchContainerRecord()
-
- jsondata, err := json.Marshal(cr.Container.Mounts["/json"].Content)
-
+ jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
+ c.Logf("%#v", s.runner.Container)
c.Check(err, IsNil)
c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
}
-func (s *TestSuite) TestFullBrokenDocker1(c *C) {
- tf, err := ioutil.TempFile("", "brokenNodeHook-")
- c.Assert(err, IsNil)
- defer os.Remove(tf.Name())
-
- tf.Write([]byte(`#!/bin/sh
-exec echo killme
-`))
- tf.Close()
- os.Chmod(tf.Name(), 0700)
-
- ech := tf.Name()
- brokenNodeHook = &ech
-
- api, _, _ := s.fullRunHelper(c, `{
- "command": ["echo", "hello world"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
- "cwd": ".",
- "environment": {},
- "mounts": {"/tmp": {"kind": "tmp"} },
- "output_path": "/tmp",
- "priority": 1,
- "runtime_constraints": {},
- "state": "Locked"
-}`, nil, 2, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, "hello world\n"))
- t.logWriter.Close()
- })
-
- c.Check(api.CalledWith("container.state", "Queued"), NotNil)
- c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
- c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
- c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
-
-}
-
-func (s *TestSuite) TestFullBrokenDocker2(c *C) {
- ech := ""
- brokenNodeHook = &ech
-
- api, _, _ := s.fullRunHelper(c, `{
- "command": ["echo", "hello world"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
- "cwd": ".",
- "environment": {},
- "mounts": {"/tmp": {"kind": "tmp"} },
- "output_path": "/tmp",
- "priority": 1,
- "runtime_constraints": {},
- "state": "Locked"
-}`, nil, 2, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, "hello world\n"))
- t.logWriter.Close()
- })
-
- c.Check(api.CalledWith("container.state", "Queued"), NotNil)
- c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
- c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
-}
-
-func (s *TestSuite) TestFullBrokenDocker3(c *C) {
- ech := ""
- brokenNodeHook = &ech
-
- api, _, _ := s.fullRunHelper(c, `{
- "command": ["echo", "hello world"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
- "cwd": ".",
- "environment": {},
- "mounts": {"/tmp": {"kind": "tmp"} },
- "output_path": "/tmp",
- "priority": 1,
- "runtime_constraints": {},
- "state": "Locked"
-}`, nil, 3, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, "hello world\n"))
- t.logWriter.Close()
- })
-
- c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
- c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
-}
-
-func (s *TestSuite) TestBadCommand1(c *C) {
- ech := ""
- brokenNodeHook = &ech
-
- api, _, _ := s.fullRunHelper(c, `{
- "command": ["echo", "hello world"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
- "cwd": ".",
- "environment": {},
- "mounts": {"/tmp": {"kind": "tmp"} },
- "output_path": "/tmp",
- "priority": 1,
- "runtime_constraints": {},
- "state": "Locked"
-}`, nil, 4, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, "hello world\n"))
- t.logWriter.Close()
- })
-
- c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
- c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
-}
-
-func (s *TestSuite) TestBadCommand2(c *C) {
- ech := ""
- brokenNodeHook = &ech
-
- api, _, _ := s.fullRunHelper(c, `{
+func (s *TestSuite) TestFullBrokenDocker(c *C) {
+ nextState := ""
+ for _, setup := range []func(){
+ func() {
+ c.Log("// waitErr = ocl runtime error")
+ s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
+ nextState = "Cancelled"
+ },
+ func() {
+ c.Log("// loadErr = cannot connect")
+ s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
+ *brokenNodeHook = c.MkDir() + "/broken-node-hook"
+ err := ioutil.WriteFile(*brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
+ c.Assert(err, IsNil)
+ nextState = "Queued"
+ },
+ } {
+ s.SetUpTest(c)
+ setup()
+ s.fullRunHelper(c, `{
"command": ["echo", "hello world"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "`+arvadostest.DockerImage112PDH+`",
"cwd": ".",
"environment": {},
"mounts": {"/tmp": {"kind": "tmp"} },
"priority": 1,
"runtime_constraints": {},
"state": "Locked"
-}`, nil, 5, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, "hello world\n"))
- t.logWriter.Close()
- })
-
- c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
- c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
+}`, nil, 0, func() {})
+ c.Check(s.api.CalledWith("container.state", nextState), NotNil)
+ c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
+ if *brokenNodeHook != "" {
+ c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
+ c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
+ c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
+ } else {
+ c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
+ }
+ }
}
-func (s *TestSuite) TestBadCommand3(c *C) {
- ech := ""
- brokenNodeHook = &ech
-
- api, _, _ := s.fullRunHelper(c, `{
+func (s *TestSuite) TestBadCommand(c *C) {
+ for _, startError := range []string{
+ `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
+ `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
+ `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
+ } {
+ s.SetUpTest(c)
+ s.executor.startErr = errors.New(startError)
+ s.fullRunHelper(c, `{
"command": ["echo", "hello world"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "`+arvadostest.DockerImage112PDH+`",
"cwd": ".",
"environment": {},
"mounts": {"/tmp": {"kind": "tmp"} },
"priority": 1,
"runtime_constraints": {},
"state": "Locked"
-}`, nil, 6, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, "hello world\n"))
- t.logWriter.Close()
- })
-
- c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
- c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
+}`, nil, 0, func() {})
+ c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
+ c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
+ }
}
func (s *TestSuite) TestSecretTextMountPoint(c *C) {
- // under normal mounts, gets captured in output, oops
helperRecord := `{
"command": ["true"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "` + arvadostest.DockerImage112PDH + `",
"cwd": "/bin",
"mounts": {
"/tmp": {"kind": "tmp"},
"state": "Locked"
}`
- api, cr, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
- content, err := ioutil.ReadFile(t.realTemp + "/tmp2/secret.conf")
+ s.fullRunHelper(c, helperRecord, nil, 0, func() {
+ content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
c.Check(err, IsNil)
- c.Check(content, DeepEquals, []byte("mypassword"))
- t.logWriter.Close()
+ c.Check(string(content), Equals, "mypassword")
})
- c.Check(api.CalledWith("container.exit_code", 0), NotNil)
- c.Check(api.CalledWith("container.state", "Complete"), NotNil)
- c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
- c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
+ c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
+ c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
+ c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
// under secret mounts, not captured in output
helperRecord = `{
"command": ["true"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "container_image": "` + arvadostest.DockerImage112PDH + `",
"cwd": "/bin",
"mounts": {
"/tmp": {"kind": "tmp"}
"state": "Locked"
}`
- api, cr, _ = s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
- content, err := ioutil.ReadFile(t.realTemp + "/tmp2/secret.conf")
+ s.SetUpTest(c)
+ s.fullRunHelper(c, helperRecord, nil, 0, func() {
+ content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
c.Check(err, IsNil)
- c.Check(content, DeepEquals, []byte("mypassword"))
- t.logWriter.Close()
+ c.Check(string(content), Equals, "mypassword")
})
- c.Check(api.CalledWith("container.exit_code", 0), NotNil)
- c.Check(api.CalledWith("container.state", "Complete"), NotNil)
- c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
- c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
+ c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
+ c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
+ c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
}
type FakeProcess struct {
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+package crunchrun
+
+import (
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "strings"
+ "time"
+
+ dockertypes "github.com/docker/docker/api/types"
+ dockercontainer "github.com/docker/docker/api/types/container"
+ dockerclient "github.com/docker/docker/client"
+ "golang.org/x/net/context"
+)
+
+// Docker daemon won't let you set a limit less than ~10 MiB
+const minDockerRAM = int64(16 * 1024 * 1024)
+
+type dockerExecutor struct {
+ containerUUID string
+ logf func(string, ...interface{})
+ watchdogInterval time.Duration
+ dockerclient *dockerclient.Client
+ containerID string
+ doneIO chan struct{}
+ errIO error
+}
+
+func newDockerExecutor(containerUUID string, logf func(string, ...interface{}), watchdogInterval time.Duration) (*dockerExecutor, error) {
+ // API version 1.21 corresponds to Docker 1.9, which is
+ // currently the minimum version we want to support.
+ client, err := dockerclient.NewClient(dockerclient.DefaultDockerHost, "1.21", nil, nil)
+ if watchdogInterval < 1 {
+ watchdogInterval = time.Minute
+ }
+ return &dockerExecutor{
+ containerUUID: containerUUID,
+ logf: logf,
+ watchdogInterval: watchdogInterval,
+ dockerclient: client,
+ }, err
+}
+
+func (e *dockerExecutor) ImageLoaded(imageID string) bool {
+ _, _, err := e.dockerclient.ImageInspectWithRaw(context.TODO(), imageID)
+ return err == nil
+}
+
+func (e *dockerExecutor) LoadImage(filename string) error {
+ f, err := os.Open(filename)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ resp, err := e.dockerclient.ImageLoad(context.TODO(), f, true)
+ if err != nil {
+ return fmt.Errorf("While loading container image into Docker: %v", err)
+ }
+ defer resp.Body.Close()
+ buf, _ := ioutil.ReadAll(resp.Body)
+ e.logf("loaded image: response %s", buf)
+ return nil
+}
+
+func (e *dockerExecutor) Create(spec containerSpec) error {
+ e.logf("Creating Docker container")
+ cfg := dockercontainer.Config{
+ Image: spec.Image,
+ Cmd: spec.Command,
+ WorkingDir: spec.WorkingDir,
+ Volumes: map[string]struct{}{},
+ OpenStdin: spec.Stdin != nil,
+ StdinOnce: spec.Stdin != nil,
+ AttachStdin: spec.Stdin != nil,
+ AttachStdout: true,
+ AttachStderr: true,
+ }
+ if cfg.WorkingDir == "." {
+ cfg.WorkingDir = ""
+ }
+ for k, v := range spec.Env {
+ cfg.Env = append(cfg.Env, k+"="+v)
+ }
+ if spec.RAM > 0 && spec.RAM < minDockerRAM {
+ spec.RAM = minDockerRAM
+ }
+ hostCfg := dockercontainer.HostConfig{
+ LogConfig: dockercontainer.LogConfig{
+ Type: "none",
+ },
+ NetworkMode: dockercontainer.NetworkMode("none"),
+ Resources: dockercontainer.Resources{
+ CgroupParent: spec.CgroupParent,
+ NanoCPUs: int64(spec.VCPUs) * 1000000000,
+ Memory: spec.RAM, // RAM
+ MemorySwap: spec.RAM, // RAM+swap
+ KernelMemory: spec.RAM, // kernel portion
+ },
+ }
+ for path, mount := range spec.BindMounts {
+ bind := mount.HostPath + ":" + path
+ if mount.ReadOnly {
+ bind += ":ro"
+ }
+ hostCfg.Binds = append(hostCfg.Binds, bind)
+ }
+ if spec.EnableNetwork {
+ hostCfg.NetworkMode = dockercontainer.NetworkMode(spec.NetworkMode)
+ }
+
+ created, err := e.dockerclient.ContainerCreate(context.TODO(), &cfg, &hostCfg, nil, e.containerUUID)
+ if err != nil {
+ return fmt.Errorf("While creating container: %v", err)
+ }
+ e.containerID = created.ID
+ return e.startIO(spec.Stdin, spec.Stdout, spec.Stderr)
+}
+
+func (e *dockerExecutor) CgroupID() string {
+ return e.containerID
+}
+
+func (e *dockerExecutor) Start() error {
+ return e.dockerclient.ContainerStart(context.TODO(), e.containerID, dockertypes.ContainerStartOptions{})
+}
+
+func (e *dockerExecutor) Stop() error {
+ err := e.dockerclient.ContainerRemove(context.TODO(), e.containerID, dockertypes.ContainerRemoveOptions{Force: true})
+ if err != nil && strings.Contains(err.Error(), "No such container: "+e.containerID) {
+ err = nil
+ }
+ return err
+}
+
+// Wait for the container to terminate, capture the exit code, and
+// wait for stdout/stderr logging to finish.
+func (e *dockerExecutor) Wait(ctx context.Context) (int, error) {
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ watchdogErr := make(chan error, 1)
+ go func() {
+ ticker := time.NewTicker(e.watchdogInterval)
+ defer ticker.Stop()
+ for range ticker.C {
+ dctx, dcancel := context.WithDeadline(ctx, time.Now().Add(e.watchdogInterval))
+ ctr, err := e.dockerclient.ContainerInspect(dctx, e.containerID)
+ dcancel()
+ if ctx.Err() != nil {
+ // Either the container already
+ // exited, or our caller is trying to
+ // kill it.
+ return
+ } else if err != nil {
+ e.logf("Error inspecting container: %s", err)
+ watchdogErr <- err
+ return
+ } else if ctr.State == nil || !(ctr.State.Running || ctr.State.Status == "created") {
+ watchdogErr <- fmt.Errorf("Container is not running: State=%v", ctr.State)
+ return
+ }
+ }
+ }()
+
+ waitOk, waitErr := e.dockerclient.ContainerWait(ctx, e.containerID, dockercontainer.WaitConditionNotRunning)
+ for {
+ select {
+ case waitBody := <-waitOk:
+ e.logf("Container exited with code: %v", waitBody.StatusCode)
+ // wait for stdout/stderr to complete
+ <-e.doneIO
+ return int(waitBody.StatusCode), nil
+
+ case err := <-waitErr:
+ return -1, fmt.Errorf("container wait: %v", err)
+
+ case <-ctx.Done():
+ return -1, ctx.Err()
+
+ case err := <-watchdogErr:
+ return -1, err
+ }
+ }
+}
+
+func (e *dockerExecutor) startIO(stdin io.ReadCloser, stdout, stderr io.WriteCloser) error {
+ resp, err := e.dockerclient.ContainerAttach(context.TODO(), e.containerID, dockertypes.ContainerAttachOptions{
+ Stream: true,
+ Stdin: stdin != nil,
+ Stdout: true,
+ Stderr: true,
+ })
+ if err != nil {
+ return fmt.Errorf("error attaching container stdin/stdout/stderr streams: %v", err)
+ }
+ var errStdin error
+ if stdin != nil {
+ go func() {
+ errStdin = e.handleStdin(stdin, resp.Conn, resp.CloseWrite)
+ }()
+ }
+ e.doneIO = make(chan struct{})
+ go func() {
+ e.errIO = e.handleStdoutStderr(stdout, stderr, resp.Reader)
+ if e.errIO == nil && errStdin != nil {
+ e.errIO = errStdin
+ }
+ close(e.doneIO)
+ }()
+ return nil
+}
+
+func (e *dockerExecutor) handleStdin(stdin io.ReadCloser, conn io.Writer, closeConn func() error) error {
+ defer stdin.Close()
+ defer closeConn()
+ _, err := io.Copy(conn, stdin)
+ if err != nil {
+ return fmt.Errorf("While writing to docker container on stdin: %v", err)
+ }
+ return nil
+}
+
+// Handle docker log protocol; see
+// https://docs.docker.com/engine/reference/api/docker_remote_api_v1.15/#attach-to-a-container
+func (e *dockerExecutor) handleStdoutStderr(stdout, stderr io.WriteCloser, reader io.Reader) error {
+ header := make([]byte, 8)
+ var err error
+ for err == nil {
+ _, err = io.ReadAtLeast(reader, header, 8)
+ if err != nil {
+ if err == io.EOF {
+ err = nil
+ }
+ break
+ }
+ readsize := int64(header[7]) | (int64(header[6]) << 8) | (int64(header[5]) << 16) | (int64(header[4]) << 24)
+ if header[0] == 1 {
+ _, err = io.CopyN(stdout, reader, readsize)
+ } else {
+ // stderr
+ _, err = io.CopyN(stderr, reader, readsize)
+ }
+ }
+ if err != nil {
+ return fmt.Errorf("error copying stdout/stderr from docker: %v", err)
+ }
+ err = stdout.Close()
+ if err != nil {
+ return fmt.Errorf("error writing stdout: close: %v", err)
+ }
+ err = stderr.Close()
+ if err != nil {
+ return fmt.Errorf("error writing stderr: close: %v", err)
+ }
+ return nil
+}
+
+func (e *dockerExecutor) Close() {
+ e.dockerclient.ContainerRemove(context.TODO(), e.containerID, dockertypes.ContainerRemoveOptions{Force: true})
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package crunchrun
+
+import (
+ "os/exec"
+ "time"
+
+ . "gopkg.in/check.v1"
+)
+
+var _ = Suite(&dockerSuite{})
+
+type dockerSuite struct {
+ executorSuite
+}
+
+func (s *dockerSuite) SetUpSuite(c *C) {
+ _, err := exec.LookPath("docker")
+ if err != nil {
+ c.Skip("looks like docker is not installed")
+ }
+ s.newExecutor = func(c *C) {
+ exec.Command("docker", "rm", "zzzzz-zzzzz-zzzzzzzzzzzzzzz").Run()
+ var err error
+ s.executor, err = newDockerExecutor("zzzzz-zzzzz-zzzzzzzzzzzzzzz", c.Logf, time.Second/2)
+ c.Assert(err, IsNil)
+ }
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+package crunchrun
+
+import (
+ "io"
+
+ "golang.org/x/net/context"
+)
+
+type bindmount struct {
+ HostPath string
+ ReadOnly bool
+}
+
+type containerSpec struct {
+ Image string
+ VCPUs int
+ RAM int64
+ WorkingDir string
+ Env map[string]string
+ BindMounts map[string]bindmount
+ Command []string
+ EnableNetwork bool
+ NetworkMode string // docker network mode, normally "default"
+ CgroupParent string
+ Stdin io.ReadCloser
+ Stdout io.WriteCloser
+ Stderr io.WriteCloser
+}
+
+// containerExecutor is an interface to a container runtime
+// (docker/singularity).
+type containerExecutor interface {
+ // ImageLoaded determines whether the given image is already
+ // available to use without calling ImageLoad.
+ ImageLoaded(imageID string) bool
+
+ // ImageLoad loads the image from the given tarball such that
+ // it can be used to create/start a container.
+ LoadImage(filename string) error
+
+ // Wait for the container process to finish, and return its
+ // exit code. If applicable, also remove the stopped container
+ // before returning.
+ Wait(context.Context) (int, error)
+
+ // Create a container, but don't start it yet.
+ Create(containerSpec) error
+
+ // Start the container
+ Start() error
+
+ // CID the container will belong to
+ CgroupID() string
+
+ // Stop the container immediately
+ Stop() error
+
+ // Release resources (temp dirs, stopped containers)
+ Close()
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package crunchrun
+
+import (
+ "bytes"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "os"
+ "strings"
+ "time"
+
+ "golang.org/x/net/context"
+ . "gopkg.in/check.v1"
+)
+
+func busyboxDockerImage(c *C) string {
+ fnm := "busybox_uclibc.tar"
+ cachedir := c.MkDir()
+ cachefile := cachedir + "/" + fnm
+ if _, err := os.Stat(cachefile); err == nil {
+ return cachefile
+ }
+
+ f, err := ioutil.TempFile(cachedir, "")
+ c.Assert(err, IsNil)
+ defer f.Close()
+ defer os.Remove(f.Name())
+
+ resp, err := http.Get("https://cache.arvados.org/" + fnm)
+ c.Assert(err, IsNil)
+ defer resp.Body.Close()
+ _, err = io.Copy(f, resp.Body)
+ c.Assert(err, IsNil)
+ err = f.Close()
+ c.Assert(err, IsNil)
+ err = os.Rename(f.Name(), cachefile)
+ c.Assert(err, IsNil)
+
+ return cachefile
+}
+
+type nopWriteCloser struct{ io.Writer }
+
+func (nopWriteCloser) Close() error { return nil }
+
+// embedded by dockerSuite and singularitySuite so they can share
+// tests.
+type executorSuite struct {
+ newExecutor func(*C) // embedding struct's SetUpSuite method must set this
+ executor containerExecutor
+ spec containerSpec
+ stdout bytes.Buffer
+ stderr bytes.Buffer
+}
+
+func (s *executorSuite) SetUpTest(c *C) {
+ s.newExecutor(c)
+ s.stdout = bytes.Buffer{}
+ s.stderr = bytes.Buffer{}
+ s.spec = containerSpec{
+ Image: "busybox:uclibc",
+ VCPUs: 1,
+ WorkingDir: "",
+ Env: map[string]string{"PATH": "/bin:/usr/bin"},
+ NetworkMode: "default",
+ Stdout: nopWriteCloser{&s.stdout},
+ Stderr: nopWriteCloser{&s.stderr},
+ }
+ err := s.executor.LoadImage(busyboxDockerImage(c))
+ c.Assert(err, IsNil)
+}
+
+func (s *executorSuite) TearDownTest(c *C) {
+ s.executor.Close()
+}
+
+func (s *executorSuite) TestExecTrivialContainer(c *C) {
+ s.spec.Command = []string{"echo", "ok"}
+ s.checkRun(c, 0)
+ c.Check(s.stdout.String(), Equals, "ok\n")
+ c.Check(s.stderr.String(), Equals, "")
+}
+
+func (s *executorSuite) TestExecStop(c *C) {
+ s.spec.Command = []string{"sh", "-c", "sleep 10; echo ok"}
+ err := s.executor.Create(s.spec)
+ c.Assert(err, IsNil)
+ err = s.executor.Start()
+ c.Assert(err, IsNil)
+ go func() {
+ time.Sleep(time.Second / 10)
+ s.executor.Stop()
+ }()
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(10*time.Second))
+ defer cancel()
+ code, err := s.executor.Wait(ctx)
+ c.Check(code, Not(Equals), 0)
+ c.Check(err, IsNil)
+ c.Check(s.stdout.String(), Equals, "")
+ c.Check(s.stderr.String(), Equals, "")
+}
+
+func (s *executorSuite) TestExecCleanEnv(c *C) {
+ s.spec.Command = []string{"env"}
+ s.checkRun(c, 0)
+ c.Check(s.stderr.String(), Equals, "")
+ got := map[string]string{}
+ for _, kv := range strings.Split(s.stdout.String(), "\n") {
+ if kv == "" {
+ continue
+ }
+ kv := strings.SplitN(kv, "=", 2)
+ switch kv[0] {
+ case "HOSTNAME", "HOME":
+ // docker sets these by itself
+ case "LD_LIBRARY_PATH", "SINGULARITY_NAME", "PWD", "LANG", "SHLVL", "SINGULARITY_INIT", "SINGULARITY_CONTAINER":
+ // singularity sets these by itself (cf. https://sylabs.io/guides/3.5/user-guide/environment_and_metadata.html)
+ case "PROMPT_COMMAND", "PS1", "SINGULARITY_APPNAME":
+ // singularity also sets these by itself (as of v3.5.2)
+ default:
+ got[kv[0]] = kv[1]
+ }
+ }
+ c.Check(got, DeepEquals, s.spec.Env)
+}
+func (s *executorSuite) TestExecEnableNetwork(c *C) {
+ for _, enable := range []bool{false, true} {
+ s.SetUpTest(c)
+ s.spec.Command = []string{"ip", "route"}
+ s.spec.EnableNetwork = enable
+ s.checkRun(c, 0)
+ if enable {
+ c.Check(s.stdout.String(), Matches, "(?ms).*default via.*")
+ } else {
+ c.Check(s.stdout.String(), Equals, "")
+ }
+ }
+}
+
+func (s *executorSuite) TestExecStdoutStderr(c *C) {
+ s.spec.Command = []string{"sh", "-c", "echo foo; echo -n bar >&2; echo baz; echo waz >&2"}
+ s.checkRun(c, 0)
+ c.Check(s.stdout.String(), Equals, "foo\nbaz\n")
+ c.Check(s.stderr.String(), Equals, "barwaz\n")
+}
+
+func (s *executorSuite) checkRun(c *C, expectCode int) {
+ c.Assert(s.executor.Create(s.spec), IsNil)
+ c.Assert(s.executor.Start(), IsNil)
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(10*time.Second))
+ defer cancel()
+ code, err := s.executor.Wait(ctx)
+ c.Assert(err, IsNil)
+ c.Check(code, Equals, expectCode)
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package crunchrun
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "strings"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
+ . "gopkg.in/check.v1"
+)
+
+var _ = Suite(&integrationSuite{})
+
+type integrationSuite struct {
+ engine string
+ image arvados.Collection
+ input arvados.Collection
+ stdin bytes.Buffer
+ stdout bytes.Buffer
+ stderr bytes.Buffer
+ cr arvados.ContainerRequest
+ client *arvados.Client
+ ac *arvadosclient.ArvadosClient
+ kc *keepclient.KeepClient
+}
+
+func (s *integrationSuite) SetUpSuite(c *C) {
+ _, err := exec.LookPath("docker")
+ if err != nil {
+ c.Skip("looks like docker is not installed")
+ }
+
+ arvadostest.StartKeep(2, true)
+
+ out, err := exec.Command("docker", "load", "--input", busyboxDockerImage(c)).CombinedOutput()
+ c.Log(string(out))
+ c.Assert(err, IsNil)
+ out, err = exec.Command("arv-keepdocker", "--no-resume", "busybox:uclibc").Output()
+ imageUUID := strings.TrimSpace(string(out))
+ c.Logf("image uuid %s", imageUUID)
+ c.Assert(err, IsNil)
+ err = arvados.NewClientFromEnv().RequestAndDecode(&s.image, "GET", "arvados/v1/collections/"+imageUUID, nil, nil)
+ c.Assert(err, IsNil)
+ c.Logf("image pdh %s", s.image.PortableDataHash)
+
+ s.client = arvados.NewClientFromEnv()
+ s.ac, err = arvadosclient.New(s.client)
+ c.Assert(err, IsNil)
+ s.kc = keepclient.New(s.ac)
+ fs, err := s.input.FileSystem(s.client, s.kc)
+ c.Assert(err, IsNil)
+ f, err := fs.OpenFile("inputfile", os.O_CREATE|os.O_WRONLY, 0755)
+ c.Assert(err, IsNil)
+ _, err = f.Write([]byte("inputdata"))
+ c.Assert(err, IsNil)
+ err = f.Close()
+ c.Assert(err, IsNil)
+ s.input.ManifestText, err = fs.MarshalManifest(".")
+ c.Assert(err, IsNil)
+ err = s.client.RequestAndDecode(&s.input, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "ensure_unique_name": true,
+ "collection": map[string]interface{}{
+ "manifest_text": s.input.ManifestText,
+ },
+ })
+ c.Assert(err, IsNil)
+ c.Logf("input pdh %s", s.input.PortableDataHash)
+}
+
+func (s *integrationSuite) TearDownSuite(c *C) {
+ if s.client == nil {
+ // didn't set up
+ return
+ }
+ err := s.client.RequestAndDecode(nil, "POST", "database/reset", nil, nil)
+ c.Check(err, IsNil)
+}
+
+func (s *integrationSuite) SetUpTest(c *C) {
+ s.engine = "docker"
+ s.stdin = bytes.Buffer{}
+ s.stdout = bytes.Buffer{}
+ s.stderr = bytes.Buffer{}
+ s.cr = arvados.ContainerRequest{
+ Priority: 1,
+ State: "Committed",
+ OutputPath: "/mnt/out",
+ ContainerImage: s.image.PortableDataHash,
+ Mounts: map[string]arvados.Mount{
+ "/mnt/json": {
+ Kind: "json",
+ Content: []interface{}{
+ "foo",
+ map[string]string{"foo": "bar"},
+ nil,
+ },
+ },
+ "/mnt/in": {
+ Kind: "collection",
+ PortableDataHash: s.input.PortableDataHash,
+ },
+ "/mnt/out": {
+ Kind: "tmp",
+ Capacity: 1000,
+ },
+ },
+ RuntimeConstraints: arvados.RuntimeConstraints{
+ RAM: 128000000,
+ VCPUs: 1,
+ API: true,
+ },
+ }
+}
+
+func (s *integrationSuite) setup(c *C) {
+ err := s.client.RequestAndDecode(&s.cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{
+ "priority": s.cr.Priority,
+ "state": s.cr.State,
+ "command": s.cr.Command,
+ "output_path": s.cr.OutputPath,
+ "container_image": s.cr.ContainerImage,
+ "mounts": s.cr.Mounts,
+ "runtime_constraints": s.cr.RuntimeConstraints,
+ "use_existing": false,
+ }})
+ c.Assert(err, IsNil)
+ c.Assert(s.cr.ContainerUUID, Not(Equals), "")
+ err = s.client.RequestAndDecode(nil, "POST", "arvados/v1/containers/"+s.cr.ContainerUUID+"/lock", nil, nil)
+ c.Assert(err, IsNil)
+}
+
+func (s *integrationSuite) TestRunTrivialContainerWithDocker(c *C) {
+ s.engine = "docker"
+ s.testRunTrivialContainer(c)
+}
+
+func (s *integrationSuite) TestRunTrivialContainerWithSingularity(c *C) {
+ s.engine = "singularity"
+ s.testRunTrivialContainer(c)
+}
+
+func (s *integrationSuite) testRunTrivialContainer(c *C) {
+ if err := exec.Command("which", s.engine).Run(); err != nil {
+ c.Skip(fmt.Sprintf("%s: %s", s.engine, err))
+ }
+ s.cr.Command = []string{"sh", "-c", "cat /mnt/in/inputfile >/mnt/out/inputfile && cat /mnt/json >/mnt/out/json && ! touch /mnt/in/shouldbereadonly && mkdir /mnt/out/emptydir"}
+ s.setup(c)
+ code := command{}.RunCommand("crunch-run", []string{
+ "-runtime-engine=" + s.engine,
+ "-enable-memory-limit=false",
+ s.cr.ContainerUUID,
+ }, &s.stdin, io.MultiWriter(&s.stdout, os.Stderr), io.MultiWriter(&s.stderr, os.Stderr))
+ c.Check(code, Equals, 0)
+ err := s.client.RequestAndDecode(&s.cr, "GET", "arvados/v1/container_requests/"+s.cr.UUID, nil, nil)
+ c.Assert(err, IsNil)
+ c.Logf("Finished container request: %#v", s.cr)
+
+ var log arvados.Collection
+ err = s.client.RequestAndDecode(&log, "GET", "arvados/v1/collections/"+s.cr.LogUUID, nil, nil)
+ c.Assert(err, IsNil)
+ fs, err := log.FileSystem(s.client, s.kc)
+ c.Assert(err, IsNil)
+ if d, err := fs.Open("/"); c.Check(err, IsNil) {
+ fis, err := d.Readdir(-1)
+ c.Assert(err, IsNil)
+ for _, fi := range fis {
+ if fi.IsDir() {
+ continue
+ }
+ f, err := fs.Open(fi.Name())
+ c.Assert(err, IsNil)
+ buf, err := ioutil.ReadAll(f)
+ c.Assert(err, IsNil)
+ c.Logf("\n===== %s =====\n%s", fi.Name(), buf)
+ }
+ }
+
+ var output arvados.Collection
+ err = s.client.RequestAndDecode(&output, "GET", "arvados/v1/collections/"+s.cr.OutputUUID, nil, nil)
+ c.Assert(err, IsNil)
+ fs, err = output.FileSystem(s.client, s.kc)
+ c.Assert(err, IsNil)
+ if f, err := fs.Open("inputfile"); c.Check(err, IsNil) {
+ defer f.Close()
+ buf, err := ioutil.ReadAll(f)
+ c.Check(err, IsNil)
+ c.Check(string(buf), Equals, "inputdata")
+ }
+ if f, err := fs.Open("json"); c.Check(err, IsNil) {
+ defer f.Close()
+ buf, err := ioutil.ReadAll(f)
+ c.Check(err, IsNil)
+ c.Check(string(buf), Equals, `["foo",{"foo":"bar"},null]`)
+ }
+ if fi, err := fs.Stat("emptydir"); c.Check(err, IsNil) {
+ c.Check(fi.IsDir(), Equals, true)
+ }
+ if d, err := fs.Open("emptydir"); c.Check(err, IsNil) {
+ defer d.Close()
+ fis, err := d.Readdir(-1)
+ c.Assert(err, IsNil)
+ // crunch-run still saves a ".keep" file to preserve
+ // empty dirs even though that shouldn't be
+ // necessary. Ideally we would do:
+ // c.Check(fis, HasLen, 0)
+ for _, fi := range fis {
+ c.Check(fi.Name(), Equals, ".keep")
+ }
+ }
+}
api := &ArvTestClient{}
kc := &KeepTestClient{}
defer kc.Close()
- cr, err := NewContainerRunner(s.client, api, kc, nil, "zzzzz-zzzzzzzzzzzzzzz")
+ cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzzzzzzzzzzzz")
c.Assert(err, IsNil)
cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
api := &ArvTestClient{}
kc := &KeepTestClient{}
defer kc.Close()
- cr, err := NewContainerRunner(s.client, api, kc, nil, "zzzzz-zzzzzzzzzzzzzzz")
+ cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzzzzzzzzzzzz")
c.Assert(err, IsNil)
cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
cr.CrunchLog.Immediate = nil
api := &ArvTestClient{}
kc := &KeepTestClient{}
defer kc.Close()
- cr, err := NewContainerRunner(s.client, api, kc, nil, "zzzzz-zzzzzzzzzzzzzzz")
+ cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzzzzzzzzzzzz")
c.Assert(err, IsNil)
ts := &TestTimestamper{}
cr.CrunchLog.Timestamper = ts.Timestamp
api := &ArvTestClient{}
kc := &KeepTestClient{}
defer kc.Close()
- cr, err := NewContainerRunner(s.client, api, kc, nil, "zzzzz-zzzzzzzzzzzzzzz")
+ cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzzzzzzzzzzzz")
c.Assert(err, IsNil)
ts := &TestTimestamper{}
cr.CrunchLog.Timestamper = ts.Timestamp
api := &ArvTestClient{}
kc := &KeepTestClient{}
defer kc.Close()
- cr, err := NewContainerRunner(s.client, api, kc, nil, "zzzzz-zzzzzzzzzzzzzzz")
+ cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzzzzzzzzzzzz")
c.Assert(err, IsNil)
cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package crunchrun
+
+import (
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "syscall"
+
+ "golang.org/x/net/context"
+)
+
+type singularityExecutor struct {
+ logf func(string, ...interface{})
+ spec containerSpec
+ tmpdir string
+ child *exec.Cmd
+ imageFilename string // "sif" image
+}
+
+func newSingularityExecutor(logf func(string, ...interface{})) (*singularityExecutor, error) {
+ tmpdir, err := ioutil.TempDir("", "crunch-run-singularity-")
+ if err != nil {
+ return nil, err
+ }
+ return &singularityExecutor{
+ logf: logf,
+ tmpdir: tmpdir,
+ }, nil
+}
+
+func (e *singularityExecutor) ImageLoaded(string) bool {
+ return false
+}
+
+// LoadImage will satisfy ContainerExecuter interface transforming
+// containerImage into a sif file for later use.
+func (e *singularityExecutor) LoadImage(imageTarballPath string) error {
+ e.logf("building singularity image")
+ // "singularity build" does not accept a
+ // docker-archive://... filename containing a ":" character,
+ // as in "/path/to/sha256:abcd...1234.tar". Workaround: make a
+ // symlink that doesn't have ":" chars.
+ err := os.Symlink(imageTarballPath, e.tmpdir+"/image.tar")
+ if err != nil {
+ return err
+ }
+ e.imageFilename = e.tmpdir + "/image.sif"
+ build := exec.Command("singularity", "build", e.imageFilename, "docker-archive://"+e.tmpdir+"/image.tar")
+ e.logf("%v", build.Args)
+ out, err := build.CombinedOutput()
+ // INFO: Starting build...
+ // Getting image source signatures
+ // Copying blob ab15617702de done
+ // Copying config 651e02b8a2 done
+ // Writing manifest to image destination
+ // Storing signatures
+ // 2021/04/22 14:42:14 info unpack layer: sha256:21cbfd3a344c52b197b9fa36091e66d9cbe52232703ff78d44734f85abb7ccd3
+ // INFO: Creating SIF file...
+ // INFO: Build complete: arvados-jobs.latest.sif
+ e.logf("%s", out)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (e *singularityExecutor) Create(spec containerSpec) error {
+ e.spec = spec
+ return nil
+}
+
+func (e *singularityExecutor) Start() error {
+ args := []string{"singularity", "exec", "--containall", "--no-home", "--cleanenv"}
+ if !e.spec.EnableNetwork {
+ args = append(args, "--net", "--network=none")
+ }
+ readonlyflag := map[bool]string{
+ false: "rw",
+ true: "ro",
+ }
+ for path, mount := range e.spec.BindMounts {
+ args = append(args, "--bind", mount.HostPath+":"+path+":"+readonlyflag[mount.ReadOnly])
+ }
+ args = append(args, e.imageFilename)
+ args = append(args, e.spec.Command...)
+
+ // This is for singularity 3.5.2. There are some behaviors
+ // that will change in singularity 3.6, please see:
+ // https://sylabs.io/guides/3.7/user-guide/environment_and_metadata.html
+ // https://sylabs.io/guides/3.5/user-guide/environment_and_metadata.html
+ env := make([]string, 0, len(e.spec.Env))
+ for k, v := range e.spec.Env {
+ env = append(env, "SINGULARITYENV_"+k+"="+v)
+ }
+
+ path, err := exec.LookPath(args[0])
+ if err != nil {
+ return err
+ }
+ child := &exec.Cmd{
+ Path: path,
+ Args: args,
+ Env: env,
+ Stdin: e.spec.Stdin,
+ Stdout: e.spec.Stdout,
+ Stderr: e.spec.Stderr,
+ }
+ err = child.Start()
+ if err != nil {
+ return err
+ }
+ e.child = child
+ return nil
+}
+
+func (e *singularityExecutor) CgroupID() string {
+ return ""
+}
+
+func (e *singularityExecutor) Stop() error {
+ if err := e.child.Process.Signal(syscall.Signal(0)); err != nil {
+ // process already exited
+ return nil
+ }
+ return e.child.Process.Signal(syscall.SIGKILL)
+}
+
+func (e *singularityExecutor) Wait(context.Context) (int, error) {
+ err := e.child.Wait()
+ if err, ok := err.(*exec.ExitError); ok {
+ return err.ProcessState.ExitCode(), nil
+ }
+ if err != nil {
+ return 0, err
+ }
+ return e.child.ProcessState.ExitCode(), nil
+}
+
+func (e *singularityExecutor) Close() {
+ err := os.RemoveAll(e.tmpdir)
+ if err != nil {
+ e.logf("error removing temp dir: %s", err)
+ }
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package crunchrun
+
+import (
+ "os/exec"
+
+ . "gopkg.in/check.v1"
+)
+
+var _ = Suite(&singularitySuite{})
+
+type singularitySuite struct {
+ executorSuite
+}
+
+func (s *singularitySuite) SetUpSuite(c *C) {
+ _, err := exec.LookPath("singularity")
+ if err != nil {
+ c.Skip("looks like singularity is not installed")
+ }
+ s.newExecutor = func(c *C) {
+ var err error
+ s.executor, err = newSingularityExecutor(c.Logf)
+ c.Assert(err, IsNil)
+ }
+}
CrunchRunArgumentsList: []string{"--foo", "--extra='args'"},
DispatchPrivateKey: string(dispatchprivraw),
StaleLockTimeout: arvados.Duration(5 * time.Millisecond),
+ RuntimeEngine: "stub",
CloudVMs: arvados.CloudVMsConfig{
Driver: "test",
SyncInterval: arvados.Duration(10 * time.Millisecond),
stubvm.CrunchRunDetachDelay = time.Duration(rand.Int63n(int64(10 * time.Millisecond)))
stubvm.ExecuteContainer = executeContainer
stubvm.CrashRunningContainer = finishContainer
- stubvm.ExtraCrunchRunArgs = "'--foo' '--extra='\\''args'\\'''"
+ stubvm.ExtraCrunchRunArgs = "'--runtime-engine=stub' '--foo' '--extra='\\''args'\\'''"
switch n % 7 {
case 0:
stubvm.Broken = time.Now().Add(time.Duration(rand.Int63n(90)) * time.Millisecond)
installPublicKey: installPublicKey,
tagKeyPrefix: cluster.Containers.CloudVMs.TagKeyPrefix,
runnerCmdDefault: cluster.Containers.CrunchRunCommand,
- runnerArgs: cluster.Containers.CrunchRunArgumentsList,
+ runnerArgs: append([]string{"--runtime-engine=" + cluster.Containers.RuntimeEngine}, cluster.Containers.CrunchRunArgumentsList...),
stop: make(chan bool),
}
wp.registerMetrics(reg)
"r-cran-roxygen2",
"r-cran-xml",
"sudo",
+ "uuid-dev",
"wget",
"xvfb",
)
+ if dev || test {
+ pkgs = append(pkgs,
+ "squashfs-tools", // for singularity
+ )
+ }
switch {
case osv.Debian && osv.Major >= 10:
pkgs = append(pkgs, "libcurl4")
} else {
err = inst.runBash(`
cd /tmp
+rm -rf /var/lib/arvados/go/
wget --progress=dot:giga -O- https://storage.googleapis.com/golang/go`+goversion+`.linux-amd64.tar.gz | tar -C /var/lib/arvados -xzf -
ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
`, stdout, stderr)
}
}
+ singularityversion := "3.5.2"
+ if havesingularityversion, err := exec.Command("/var/lib/arvados/bin/singularity", "--version").CombinedOutput(); err == nil && strings.Contains(string(havesingularityversion), singularityversion) {
+ logger.Print("singularity " + singularityversion + " already installed")
+ } else if dev || test {
+ err = inst.runBash(`
+S=`+singularityversion+`
+tmp=/var/lib/arvados/tmp/singularity
+trap "rm -r ${tmp}" ERR EXIT
+cd /var/lib/arvados/tmp
+git clone https://github.com/sylabs/singularity
+cd singularity
+git checkout v${S}
+./mconfig --prefix=/var/lib/arvados
+make -C ./builddir
+make -C ./builddir install
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+
// The entry in /etc/locale.gen is "en_US.UTF-8"; once
// it's installed, locale -a reports it as
// "en_US.utf8".
doc: "Test issue 17462 - secondary file objects on file defaults are not resolved"
- job: null
- output: {}
+ output: {
+ "stuff": {
+ "location": "bar.txt",
+ "basename": "bar.txt",
+ "class": "File",
+ "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
+ "size": 0
+ }
+ }
tool: 17521-dot-slash-glob.cwl
doc: "Test issue 17521 - bug with leading './' capturing files in subdirectories"
- job: null
- output: {}
+ output: {
+ "stuff": {
+ "basename": "foo",
+ "class": "Directory",
+ "listing": [
+ {
+ "basename": "bar.txt",
+ "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
+ "class": "File",
+ "location": "foo/bar.txt",
+ "size": 0
+ }
+ ],
+ "location": "foo"
+ }
+ }
tool: 10380-trailing-slash-dir.cwl
doc: "Test issue 10380 - bug with trailing slash when capturing an output directory"
EmailClaim string
EmailVerifiedClaim string
UsernameClaim string
+ AcceptAccessToken bool
+ AcceptAccessTokenScope string
AuthenticationRequestParameters map[string]string
}
PAM struct {
RemoteTokenRefresh Duration
TokenLifetime Duration
TrustedClients map[string]struct{}
+ IssueTrustedTokens bool
}
Mail struct {
MailchimpAPIKey string
StaleLockTimeout Duration
SupportedDockerImageFormats StringSet
UsePreemptibleInstances bool
+ RuntimeEngine string
JobsAPI struct {
Enable string
CompletedDiagnosticsHasher2ContainerUUID = "zzzzz-dz642-diagcomphasher2"
CompletedDiagnosticsHasher3ContainerUUID = "zzzzz-dz642-diagcomphasher3"
+ UncommittedContainerRequestUUID = "zzzzz-xvhdp-cr4uncommittedc"
+
Hasher1LogCollectionUUID = "zzzzz-4zz18-dlogcollhash001"
Hasher2LogCollectionUUID = "zzzzz-4zz18-dlogcollhash002"
Hasher3LogCollectionUUID = "zzzzz-4zz18-dlogcollhash003"
LogCollectionUUID = "zzzzz-4zz18-logcollection01"
LogCollectionUUID2 = "zzzzz-4zz18-logcollection02"
+
+ DockerImage112PDH = "d740a57097711e08eb9b2a93518f20ab+174"
+ DockerImage112Filename = "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678.tar"
)
// PathologicalManifest : A valid manifest designed to test
"gopkg.in/check.v1"
"gopkg.in/square/go-jose.v2"
+ "gopkg.in/square/go-jose.v2/jwt"
)
type OIDCProvider struct {
ValidClientID string
ValidClientSecret string
// desired response from token endpoint
- AuthEmail string
- AuthEmailVerified bool
- AuthName string
+ AuthEmail string
+ AuthEmailVerified bool
+ AuthName string
+ AccessTokenPayload map[string]interface{}
PeopleAPIResponse map[string]interface{}
c.Assert(err, check.IsNil)
p.Issuer = httptest.NewServer(http.HandlerFunc(p.serveOIDC))
p.PeopleAPI = httptest.NewServer(http.HandlerFunc(p.servePeopleAPI))
+ p.AccessTokenPayload = map[string]interface{}{"sub": "example"}
return p
}
func (p *OIDCProvider) ValidAccessToken() string {
- return p.fakeToken([]byte("fake access token"))
+ buf, _ := json.Marshal(p.AccessTokenPayload)
+ return p.fakeToken(buf)
}
func (p *OIDCProvider) serveOIDC(w http.ResponseWriter, req *http.Request) {
case "/auth":
w.WriteHeader(http.StatusInternalServerError)
case "/userinfo":
- if authhdr := req.Header.Get("Authorization"); strings.TrimPrefix(authhdr, "Bearer ") != p.ValidAccessToken() {
+ authhdr := req.Header.Get("Authorization")
+ if _, err := jwt.ParseSigned(strings.TrimPrefix(authhdr, "Bearer ")); err != nil {
p.c.Logf("OIDCProvider: bad auth %q", authhdr)
w.WriteHeader(http.StatusUnauthorized)
return
},
"Login": {
"SSO": {
+ "Enable": True,
"ProviderAppID": "arvados-server",
"ProviderAppSecret": "608dbf356a327e2d0d4932b60161e212c2d8d8f5e25690d7b622f850a990cd33",
},
# SPDX-License-Identifier: Apache-2.0
require 'google/api_client'
-# Monkeypatch google-api-client gem to avoid sending newline characters
-# on headers to make ruby-2.3.7+ happy.
-# See: https://dev.arvados.org/issues/13920
-Google::APIClient::ENV::OS_VERSION.strip!
-
require 'json'
require 'tempfile'
gem 'themes_for_rails', git: 'https://github.com/arvados/themes_for_rails'
-# Import arvados gem. Note: actual git commit is pinned via Gemfile.lock
-gem 'arvados', git: 'https://github.com/arvados/arvados.git', glob: 'sdk/ruby/arvados.gemspec'
+# Import arvados gem.
+gem 'arvados', '~> 2.1.5'
gem 'httpclient'
gem 'sshkey'
-GIT
- remote: https://github.com/arvados/arvados.git
- revision: 81725af5d5d2e6cd18ba7099ba5fb1fc520f4f8c
- glob: sdk/ruby/arvados.gemspec
- specs:
- arvados (1.5.0.pre20200114202620)
- activesupport (>= 3)
- andand (~> 1.3, >= 1.3.3)
- arvados-google-api-client (>= 0.7, < 0.8.9)
- faraday (< 0.16)
- i18n (~> 0)
- json (>= 1.7.7, < 3)
- jwt (>= 0.1.5, < 2)
-
GIT
remote: https://github.com/arvados/themes_for_rails
revision: ddf6e592b3b6493ea0c2de7b5d3faa120ed35be0
GEM
remote: https://rubygems.org/
specs:
- actioncable (5.2.4.5)
- actionpack (= 5.2.4.5)
+ actioncable (5.2.6)
+ actionpack (= 5.2.6)
nio4r (~> 2.0)
websocket-driver (>= 0.6.1)
- actionmailer (5.2.4.5)
- actionpack (= 5.2.4.5)
- actionview (= 5.2.4.5)
- activejob (= 5.2.4.5)
+ actionmailer (5.2.6)
+ actionpack (= 5.2.6)
+ actionview (= 5.2.6)
+ activejob (= 5.2.6)
mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 2.0)
- actionpack (5.2.4.5)
- actionview (= 5.2.4.5)
- activesupport (= 5.2.4.5)
+ actionpack (5.2.6)
+ actionview (= 5.2.6)
+ activesupport (= 5.2.6)
rack (~> 2.0, >= 2.0.8)
rack-test (>= 0.6.3)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.2)
- actionview (5.2.4.5)
- activesupport (= 5.2.4.5)
+ actionview (5.2.6)
+ activesupport (= 5.2.6)
builder (~> 3.1)
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.3)
- activejob (5.2.4.5)
- activesupport (= 5.2.4.5)
+ activejob (5.2.6)
+ activesupport (= 5.2.6)
globalid (>= 0.3.6)
- activemodel (5.2.4.5)
- activesupport (= 5.2.4.5)
- activerecord (5.2.4.5)
- activemodel (= 5.2.4.5)
- activesupport (= 5.2.4.5)
+ activemodel (5.2.6)
+ activesupport (= 5.2.6)
+ activerecord (5.2.6)
+ activemodel (= 5.2.6)
+ activesupport (= 5.2.6)
arel (>= 9.0)
- activestorage (5.2.4.5)
- actionpack (= 5.2.4.5)
- activerecord (= 5.2.4.5)
- marcel (~> 0.3.1)
- activesupport (5.2.4.5)
+ activestorage (5.2.6)
+ actionpack (= 5.2.6)
+ activerecord (= 5.2.6)
+ marcel (~> 1.0.0)
+ activesupport (5.2.6)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 0.7, < 2)
minitest (~> 5.1)
public_suffix (>= 2.0.2, < 5.0)
andand (1.3.3)
arel (9.0.0)
+ arvados (2.1.5)
+ activesupport (>= 3)
+ andand (~> 1.3, >= 1.3.3)
+ arvados-google-api-client (>= 0.7, < 0.8.9)
+ faraday (< 0.16)
+ i18n (~> 0)
+ json (>= 1.7.7, < 3)
+ jwt (>= 0.1.5, < 2)
arvados-google-api-client (0.8.7.4)
activesupport (>= 3.2, < 5.3)
addressable (~> 2.3)
net-sftp (>= 2.0.0)
net-ssh (>= 2.0.14)
net-ssh-gateway (>= 1.1.0)
- concurrent-ruby (1.1.8)
+ concurrent-ruby (1.1.9)
crass (1.0.6)
erubi (1.10.0)
execjs (2.7.0)
rails-dom-testing (>= 1, < 3)
railties (>= 4.2.0)
thor (>= 0.14, < 2.0)
- json (2.3.0)
+ json (2.5.1)
jwt (1.5.6)
- launchy (2.4.3)
- addressable (~> 2.3)
+ launchy (2.5.0)
+ addressable (~> 2.7)
libv8 (3.16.14.19)
listen (3.2.1)
rb-fsevent (~> 0.10, >= 0.10.3)
railties (>= 4)
request_store (~> 1.0)
logstash-event (1.2.02)
- loofah (2.9.0)
+ loofah (2.10.0)
crass (~> 1.0.2)
nokogiri (>= 1.5.9)
mail (2.7.1)
mini_mime (>= 0.1.1)
- marcel (0.3.3)
- mimemagic (~> 0.3.2)
+ marcel (1.0.1)
memoist (0.16.2)
metaclass (0.0.4)
method_source (1.0.0)
- mimemagic (0.3.8)
- nokogiri (~> 1)
- mini_mime (1.0.2)
- mini_portile2 (2.5.0)
+ mini_mime (1.1.0)
+ mini_portile2 (2.5.3)
minitest (5.10.3)
mocha (1.8.0)
metaclass (~> 0.0.1)
msgpack (1.3.3)
- multi_json (1.14.1)
+ multi_json (1.15.0)
multi_xml (0.6.0)
multipart-post (2.1.1)
net-scp (2.0.0)
net-ssh-gateway (2.0.0)
net-ssh (>= 4.0.0)
nio4r (2.5.7)
- nokogiri (1.11.2)
+ nokogiri (1.11.7)
mini_portile2 (~> 2.5.0)
racc (~> 1.4)
oauth2 (1.4.1)
oauth2 (~> 1.1)
omniauth (~> 1.2)
optimist (3.0.0)
- os (1.0.1)
+ os (1.1.1)
passenger (6.0.2)
rack
rake (>= 0.8.1)
pg (1.1.4)
power_assert (1.1.4)
- public_suffix (4.0.3)
+ public_suffix (4.0.6)
racc (1.5.2)
rack (2.2.3)
rack-test (1.1.0)
rack (>= 1.0, < 3)
- rails (5.2.4.5)
- actioncable (= 5.2.4.5)
- actionmailer (= 5.2.4.5)
- actionpack (= 5.2.4.5)
- actionview (= 5.2.4.5)
- activejob (= 5.2.4.5)
- activemodel (= 5.2.4.5)
- activerecord (= 5.2.4.5)
- activestorage (= 5.2.4.5)
- activesupport (= 5.2.4.5)
+ rails (5.2.6)
+ actioncable (= 5.2.6)
+ actionmailer (= 5.2.6)
+ actionpack (= 5.2.6)
+ actionview (= 5.2.6)
+ activejob (= 5.2.6)
+ activemodel (= 5.2.6)
+ activerecord (= 5.2.6)
+ activestorage (= 5.2.6)
+ activesupport (= 5.2.6)
bundler (>= 1.3.0)
- railties (= 5.2.4.5)
+ railties (= 5.2.6)
sprockets-rails (>= 2.0.0)
rails-controller-testing (1.0.4)
actionpack (>= 5.0.1.x)
rails-observers (0.1.5)
activemodel (>= 4.0)
rails-perftest (0.0.7)
- railties (5.2.4.5)
- actionpack (= 5.2.4.5)
- activesupport (= 5.2.4.5)
+ railties (5.2.6)
+ actionpack (= 5.2.6)
+ activesupport (= 5.2.6)
method_source
rake (>= 0.8.7)
thor (>= 0.19.0, < 2.0)
uglifier (2.7.2)
execjs (>= 0.3.0)
json (>= 1.8.0)
- websocket-driver (0.7.3)
+ websocket-driver (0.7.4)
websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5)
DEPENDENCIES
acts_as_api
andand
- arvados!
+ arvados (~> 2.1.5)
bootsnap
byebug
factory_bot_rails
end
def is_trusted
- (from_trusted_url && Rails.configuration.Login.TokenLifetime == 0) || super
+ (from_trusted_url && Rails.configuration.Login.IssueTrustedTokens) || super
end
protected
protected
def clamp_token_expiration
- if !current_user.andand.is_admin && Rails.configuration.API.MaxTokenLifetime > 0
+ if Rails.configuration.API.MaxTokenLifetime > 0
max_token_expiration = db_current_time + Rails.configuration.API.MaxTokenLifetime
- if (self.new_record? || self.expires_at_changed?) && (self.expires_at.nil? || self.expires_at > max_token_expiration)
+ if (self.new_record? || self.expires_at_changed?) && (self.expires_at.nil? || (self.expires_at > max_token_expiration && !current_user.andand.is_admin))
self.expires_at = max_token_expiration
end
end
if self.new_record? || self.state_was == Uncommitted
# Allow create-and-commit in a single operation.
permitted.push(*AttrsPermittedBeforeCommit)
+ elsif mounts_changed? && mounts_was.keys == mounts.keys
+ # Ignore the updated mounts if the only changes are default/zero
+ # values as added by controller, see 17774
+ only_defaults = true
+ mounts.each do |path, mount|
+ (mount.to_a - mounts_was[path].to_a).each do |k, v|
+ if !["", false, nil].index(v)
+ only_defaults = false
+ end
+ end
+ end
+ if only_defaults
+ clear_attribute_change("mounts")
+ end
end
case self.state
output_path: test
command: ["echo", "hello"]
container_uuid: zzzzz-dz642-runningcontainr
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
runtime_constraints:
vcpus: 1
ram: 123
- mounts: {}
requester_for_running:
uuid: zzzzz-xvhdp-req4runningcntr
command: ["echo", "hello"]
container_uuid: zzzzz-dz642-logscontainer03
requesting_container_uuid: zzzzz-dz642-runningcontainr
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
runtime_constraints:
vcpus: 1
ram: 123
- mounts: {}
running_older:
uuid: zzzzz-xvhdp-cr4runningcntn2
output_path: test
command: ["echo", "hello"]
container_uuid: zzzzz-dz642-runningcontain2
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
runtime_constraints:
vcpus: 1
ram: 123
- mounts: {}
completed:
uuid: zzzzz-xvhdp-cr4completedctr
output_path: test
command: ["echo", "hello"]
container_uuid: zzzzz-dz642-runningcontain2
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
runtime_constraints:
vcpus: 1
ram: 123
- mounts: {}
cr_for_failed:
uuid: zzzzz-xvhdp-cr4failedcontnr
output_path: test
command: ["echo", "hello"]
container_uuid: zzzzz-dz642-runningcontainr
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
runtime_constraints:
vcpus: 1
ram: 123
- mounts: {}
running_to_be_deleted:
uuid: zzzzz-xvhdp-cr5runningcntnr
cwd: test
output_path: test
command: ["echo", "hello"]
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
container_uuid: zzzzz-dz642-runnincntrtodel
runtime_constraints:
vcpus: 1
ram: 123
- mounts: {}
completed_with_input_mounts:
uuid: zzzzz-xvhdp-crwithinputmnts
updated_at: <%= 1.minute.ago.to_s(:db) %>
started_at: <%= 1.minute.ago.to_s(:db) %>
container_image: test
- cwd: test
- output_path: test
+ cwd: /tmp
+ output_path: /tmp
command: ["echo", "hello"]
runtime_constraints:
ram: 12000000000
vcpus: 4
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
secret_mounts:
/secret/6x9:
kind: text
updated_at: <%= 2.minute.ago.to_s(:db) %>
started_at: <%= 2.minute.ago.to_s(:db) %>
container_image: test
- cwd: test
- output_path: test
+ cwd: /tmp
+ output_path: /tmp
command: ["echo", "hello"]
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
runtime_constraints:
ram: 12000000000
vcpus: 4
cwd: test
output_path: test
command: ["echo", "hello"]
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
runtime_constraints:
ram: 12000000000
vcpus: 4
cwd: test
output_path: test
command: ["echo", "hello"]
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
runtime_constraints:
ram: 12000000000
vcpus: 4
end
end
- test "expires_at can be set to #{desired_expiration.nil? ? 'nil' : 'exceed the limit'} by admins when API.MaxTokenLifetime is set" do
+ test "behavior when expires_at is set to #{desired_expiration.nil? ? 'nil' : 'exceed the limit'} by admins when API.MaxTokenLifetime is set" do
Rails.configuration.API.MaxTokenLifetime = 1.hour
# Test token creation
headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin_trustedclient).api_token}"}
assert_response 200
if desired_expiration.nil?
- assert json_response['expires_at'].nil?
+ # When expires_at is nil, default to MaxTokenLifetime
+ assert_operator (json_response['expires_at'].to_time.to_i - (db_current_time + Rails.configuration.API.MaxTokenLifetime).to_i).abs, :<, 2
else
assert_equal json_response['expires_at'].to_time.to_i, desired_expiration.to_i
end
# Test token update (reverse the above behavior)
- previous_expiration = json_response['expires_at']
token_uuid = json_response['uuid']
- if previous_expiration.nil?
- desired_updated_expiration = db_current_time + Rails.configuration.API.MaxTokenLifetime + 1.hour
+ if desired_expiration.nil?
+ submitted_updated_expiration = db_current_time + Rails.configuration.API.MaxTokenLifetime + 1.hour
else
- desired_updated_expiration = nil
+ submitted_updated_expiration = nil
end
put "/arvados/v1/api_client_authorizations/#{token_uuid}",
params: {
:api_client_authorization => {
- :expires_at => desired_updated_expiration,
+ :expires_at => submitted_updated_expiration,
}
},
headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin_trustedclient).api_token}"}
assert_response 200
- if desired_updated_expiration.nil?
- assert json_response['expires_at'].nil?
+ if submitted_updated_expiration.nil?
+ assert_operator (json_response['expires_at'].to_time.to_i - (db_current_time + Rails.configuration.API.MaxTokenLifetime).to_i).abs, :<, 2
else
- assert_equal json_response['expires_at'].to_time.to_i, desired_updated_expiration.to_i
+ assert_equal json_response['expires_at'].to_time.to_i, submitted_updated_expiration.to_i
end
end
end
[true, false].each do |token_lifetime_enabled|
test "configured workbench is trusted when token lifetime is#{token_lifetime_enabled ? '': ' not'} enabled" do
Rails.configuration.Login.TokenLifetime = token_lifetime_enabled ? 8.hours : 0
+ Rails.configuration.Login.IssueTrustedTokens = !token_lifetime_enabled;
Rails.configuration.Services.Workbench1.ExternalURL = URI("http://wb1.example.com")
Rails.configuration.Services.Workbench2.ExternalURL = URI("https://wb2.example.com:443")
Rails.configuration.Login.TrustedClients = ActiveSupport::OrderedOptions.new
['Committed', false, {container_count: 2}],
['Committed', false, {container_count: 0}],
['Committed', false, {container_count: nil}],
+ ['Committed', true, {priority: 0, mounts: {"/out" => {"kind" => "tmp", "capacity" => 1000000}}}],
+ ['Committed', true, {priority: 0, mounts: {"/out" => {"capacity" => 1000000, "kind" => "tmp"}}}],
+ # Addition of default values for mounts / runtime_constraints /
+ # scheduling_parameters, as happens in a round-trip through
+ # controller, does not have any real effect and should be
+ # accepted/ignored rather than causing an error when the CR state
+ # dictates those attributes are not allowed to change.
+ ['Committed', true, {priority: 0, mounts: {"/out" => {"capacity" => 1000000, "kind" => "tmp", "exclude_from_output": false}}}],
+ ['Committed', true, {priority: 0, mounts: {"/out" => {"capacity" => 1000000, "kind" => "tmp", "repository_name": ""}}}],
+ ['Committed', true, {priority: 0, mounts: {"/out" => {"capacity" => 1000000, "kind" => "tmp", "content": nil}}}],
+ ['Committed', false, {priority: 0, mounts: {"/out" => {"capacity" => 1000000, "kind" => "tmp", "content": {}}}}],
+ ['Committed', false, {priority: 0, mounts: {"/out" => {"capacity" => 1000000, "kind" => "tmp", "repository_name": "foo"}}}],
+ ['Committed', false, {priority: 0, mounts: {"/out" => {"kind" => "tmp", "capacity" => 1234567}}}],
+ ['Committed', false, {priority: 0, mounts: {}}],
+ ['Committed', true, {priority: 0, runtime_constraints: {"vcpus" => 1, "ram" => 2}}],
+ ['Committed', true, {priority: 0, runtime_constraints: {"vcpus" => 1, "ram" => 2, "keep_cache_ram" => 0}}],
+ ['Committed', true, {priority: 0, runtime_constraints: {"vcpus" => 1, "ram" => 2, "API" => false}}],
+ ['Committed', false, {priority: 0, runtime_constraints: {"vcpus" => 1, "ram" => 2, "keep_cache_ram" => 1}}],
+ ['Committed', false, {priority: 0, runtime_constraints: {"vcpus" => 1, "ram" => 2, "API" => true}}],
+ ['Committed', true, {priority: 0, scheduling_parameters: {"preemptible" => false}}],
+ ['Committed', true, {priority: 0, scheduling_parameters: {"partitions" => []}}],
+ ['Committed', true, {priority: 0, scheduling_parameters: {"max_run_time" => 0}}],
+ ['Committed', false, {priority: 0, scheduling_parameters: {"preemptible" => true}}],
+ ['Committed', false, {priority: 0, scheduling_parameters: {"partitions" => ["foo"]}}],
+ ['Committed', false, {priority: 0, scheduling_parameters: {"max_run_time" => 1}}],
['Final', false, {state: ContainerRequest::Committed, name: "foobar"}],
['Final', false, {name: "foobar", priority: 123}],
['Final', false, {name: "foobar", output_uuid: "zzzzz-4zz18-znfnqtbbv4spc3w"}],
},
})
if err == nil {
- c.collections.Add(client.AuthToken+"\000"+coll.PortableDataHash, &cachedCollection{
+ c.collections.Add(client.AuthToken+"\000"+updated.PortableDataHash, &cachedCollection{
expire: time.Now().Add(time.Duration(c.config.TTL)),
collection: &updated,
})
c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
}
+// Writing to a collection shouldn't affect its entry in the
+// PDH-to-manifest cache.
+func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.Equals, nil)
+ arv.ApiToken = arvadostest.ActiveToken
+
+ u := mustParseURL("http://x.example/testfile")
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
+ }
+
+ checkWithID := func(id string, status int) {
+ req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
+ req.Host = req.URL.Host
+ resp := httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, status)
+ }
+
+ var colls [2]arvados.Collection
+ for i := range colls {
+ err := arv.Create("collections",
+ map[string]interface{}{
+ "ensure_unique_name": true,
+ "collection": map[string]interface{}{
+ "name": "test collection",
+ },
+ }, &colls[i])
+ c.Assert(err, check.Equals, nil)
+ }
+
+ // Populate cache with empty collection
+ checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
+
+ // write a file to colls[0]
+ reqPut := *req
+ reqPut.Method = "PUT"
+ reqPut.URL.Host = colls[0].UUID + ".example"
+ reqPut.Host = req.URL.Host
+ reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
+ resp := httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, &reqPut)
+ c.Check(resp.Code, check.Equals, http.StatusCreated)
+
+ // new file should not appear in colls[1]
+ checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
+ checkWithID(colls[1].UUID, http.StatusNotFound)
+
+ checkWithID(colls[0].UUID, http.StatusOK)
+}
+
func copyHeader(h http.Header) http.Header {
hc := http.Header{}
for k, v := range h {
EXTRA2+=" -var public_key_file=$PUBLIC_KEY_FILE"
fi
+echo
+packer version
+echo
echo packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE
packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE
$SUDO systemctl daemon-reload
+# docker should not start on boot: we restart it inside /usr/local/bin/ensure-encrypted-partitions.sh,
+# and the BootProbeCommand defaults to "docker ps -q"
+$SUDO systemctl disable docker
+
# Make sure user_allow_other is set in fuse.conf
$SUDO sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf
##### About
This directory holds a small script to help you get Arvados up and running, using the
-[Saltstack arvados-formula](https://github.com/arvados/arvados-formula.git)
+[Saltstack arvados-formula](https://git.arvados.org/arvados-formula.git)
in master-less mode.
There are a few preset examples that you can use:
* CLUSTER.DOMAIN
* collections.CLUSTER.DOMAIN
-* \*\-\-collections.CLUSTER.DOMAIN
+* \*.collections.CLUSTER.DOMAIN
* download.CLUSTER.DOMAIN
* keep.CLUSTER.DOMAIN
* workbench.CLUSTER.DOMAIN
InternalURLs:
'http://localhost:8004': {}
WebDAV:
- ExternalURL: 'https://*--collections.__CLUSTER__.__DOMAIN__:__KEEPWEB_EXT_SSL_PORT__/'
+ ExternalURL: 'https://*.collections.__CLUSTER__.__DOMAIN__:__KEEPWEB_EXT_SSL_PORT__/'
InternalURLs:
'http://localhost:9002': {}
WebDAVDownload:
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+aws_credentials:
+ region: __LE_AWS_REGION__
+ access_key_id: __LE_AWS_ACCESS_KEY_ID__
+ secret_access_key: __LE_AWS_SECRET_ACCESS_KEY__
use_package: true
pkgs:
- certbot: latest
- - python3-certbot-nginx
+ - python3-certbot-dns-route53
config:
server: https://acme-v02.api.letsencrypt.org/directory
email: __INITIAL_USER_EMAIL__
- authenticator: nginx
- webroot-path: /var/www
+ authenticator: dns-route53
agree-tos: true
keep-until-expiring: true
expand: true
max-log-backups: 0
deploy-hook: systemctl reload nginx
-
-### NGINX
-nginx:
- ### SNIPPETS
- snippets:
- ### LETSENCRYPT DEFAULT PATH
- letsencrypt_well_known.conf:
- - location /.well-known:
- - root: /var/www
- download.__CLUSTER__.__DOMAIN__
collections.__CLUSTER__.__DOMAIN__:
- collections.__CLUSTER__.__DOMAIN__
+ - '*.collections.__CLUSTER__.__DOMAIN__'
### NGINX
nginx:
- server_name: __CLUSTER__.__DOMAIN__
- listen:
- 80 default
- - include: snippets/letsencrypt_well_known.conf
- location /:
- return: '301 https://$host$request_uri'
- server_name: keep.__CLUSTER__.__DOMAIN__
- listen:
- 80
- - include: snippets/letsencrypt_well_known.conf
- location /:
- return: '301 https://$host$request_uri'
overwrite: true
config:
- server:
- - server_name: '~^((.*--)?collections|download)\.__CLUSTER__\.__DOMAIN__'
+ - server_name: '~^((.*\.)?collections|download)\.__CLUSTER__\.__DOMAIN__'
- listen:
- 80
- - include: snippets/letsencrypt_well_known.conf
- location /:
- return: '301 https://$host$request_uri'
enabled: true
overwrite: true
requires:
- cmd: create-initial-cert-collections.__CLUSTER__.__DOMAIN__-collections.__CLUSTER__.__DOMAIN__
+ cmd: 'create-initial-cert-collections.__CLUSTER__.__DOMAIN__-collections.__CLUSTER__.__DOMAIN__+*.__CLUSTER__.__DOMAIN__'
config:
- server:
- - server_name: '~^(.*--)?collections\.__CLUSTER__\.__DOMAIN__'
+ - server_name: '*.collections.__CLUSTER__.__DOMAIN__'
- listen:
- __CONTROLLER_EXT_SSL_PORT__ http2 ssl
- index: index.html index.htm
- server_name: webshell.__CLUSTER__.__DOMAIN__
- listen:
- 80
- - include: snippets/letsencrypt_well_known.conf
- location /:
- return: '301 https://$host$request_uri'
- server_name: ws.__CLUSTER__.__DOMAIN__
- listen:
- 80
- - include: snippets/letsencrypt_well_known.conf
- location /:
- return: '301 https://$host$request_uri'
- server_name: workbench2.__CLUSTER__.__DOMAIN__
- listen:
- 80
- - include: snippets/letsencrypt_well_known.conf
- location /:
- return: '301 https://$host$request_uri'
- server_name: workbench.__CLUSTER__.__DOMAIN__
- listen:
- 80
- - include: snippets/letsencrypt_well_known.conf
- location /:
- return: '301 https://$host$request_uri'
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+{%- set aws_credentials = pillar.get('aws_credentials', {}) %}
+
+{%- if aws_credentials %}
+extra_extra_aws_credentials_root_aws_config_file_managed:
+ file.managed:
+ - name: /root/.aws/config
+ - makedirs: true
+ - user: root
+ - group: root
+ - mode: '0600'
+ - replace: false
+ - contents: |
+ [default]
+ region= {{ aws_credentials.region }}
+
+extra_extra_aws_credentials_root_aws_credentials_file_managed:
+ file.managed:
+ - name: /root/.aws/credentials
+ - makedirs: true
+ - user: root
+ - group: root
+ - mode: '0600'
+ - replace: false
+ - contents: |
+ [default]
+ aws_access_key_id = {{ aws_credentials.access_key_id }}
+ aws_secret_access_key = {{ aws_credentials.secret_access_key }}
+{%- endif %}
- download.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
- collections.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
-extra_extra_hosts_entries_etc_hosts_shell_host_present:
+extra_extra_hosts_entries_etc_hosts_webshell_host_present:
host.present:
- ip: __WEBSHELL_INT_IP__
+ - names:
+ - webshell.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
+
+extra_extra_hosts_entries_etc_hosts_shell_host_present:
+ host.present:
+ - ip: __SHELL_INT_IP__
- names:
- shell.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
# Internal IPs for the configuration
CLUSTER_INT_CIDR=10.0.0.0/16
+
+# Note the IPs in this example are shared between roles, as suggested in
+# https://doc.arvados.org/main/install/salt-multi-host.html
CONTROLLER_INT_IP=10.0.0.1
WEBSOCKET_INT_IP=10.0.0.1
KEEP_INT_IP=10.0.0.2
+# Both for collections and downloads
+KEEPWEB_INT_IP=10.0.0.2
KEEPSTORE0_INT_IP=10.0.0.3
KEEPSTORE1_INT_IP=10.0.0.4
-# Both for collections and downloads
-KEEPWEB_INT_IP=10.0.0.5
-WEBSHELL_INT_IP=10.0.0.6
-WORKBENCH1_INT_IP=10.0.0.7
-WORKBENCH2_INT_IP=10.0.0.7
-DATABASE_INT_IP=10.0.0.8
+WORKBENCH1_INT_IP=10.0.0.5
+WORKBENCH2_INT_IP=10.0.0.5
+WEBSHELL_INT_IP=10.0.0.5
+DATABASE_INT_IP=10.0.0.6
+SHELL_INT_IP=10.0.0.7
INITIAL_USER="admin"
INITIAL_USER_PASSWORD="password"
# variable to "no", provide and upload your own certificates to the instances and
# modify the 'nginx_*' salt pillars accordingly
USE_LETSENCRYPT="yes"
+USE_LETSENCRYPT_IAM_USER="yes"
+# For collections, we need to obtain a wildcard certificate for
+# '*.collections.<cluster>.<domain>'. This is only possible through a DNS-01 challenge.
+# For that reason, you'll need to provide AWS credentials with permissions to manage
+# RRs in the route53 zone for the cluster.
+# WARNING!: If AWS credentials files already exist in the hosts, they won't be replaced.
+LE_AWS_REGION="us-east-1"
+LE_AWS_ACCESS_KEY_ID="AKIABCDEFGHIJKLMNOPQ"
+LE_AWS_SECRET_ACCESS_KEY="thisistherandomstringthatisyoursecretkey"
# The directory to check for the config files (pillars, states) you want to use.
# There are a few examples under 'config_examples'.
echo >&2 " Possible values are:"
echo >&2 " api"
echo >&2 " controller"
+ echo >&2 " dispatcher"
+ echo >&2 " keepproxy"
echo >&2 " keepstore"
- echo >&2 " websocket"
echo >&2 " keepweb"
- echo >&2 " workbench2"
- echo >&2 " keepproxy"
echo >&2 " shell"
+ echo >&2 " webshell"
+ echo >&2 " websocket"
echo >&2 " workbench"
- echo >&2 " dispatcher"
+ echo >&2 " workbench2"
echo >&2 " Defaults to applying them all"
echo >&2 " -h, --help Display this help and exit"
echo >&2 " -v, --vagrant Run in vagrant and use the /vagrant shared dir"
for i in ${2//,/ }
do
# Verify the role exists
- if [[ ! "database,api,controller,keepstore,websocket,keepweb,workbench2,keepproxy,shell,workbench,dispatcher" == *"$i"* ]]; then
+ if [[ ! "database,api,controller,keepstore,websocket,keepweb,workbench2,webshell,keepproxy,shell,workbench,dispatcher" == *"$i"* ]]; then
echo "The role '${i}' is not a valid role"
usage
exit 1
WORKBENCH1_EXT_SSL_PORT=443
WORKBENCH2_EXT_SSL_PORT=3001
-RELEASE="production"
-VERSION="2.1.2-1"
+# For a stable release, change RELEASE "production" and VERSION to the
+# package version (including the iteration, e.g. X.Y.Z-1) of the
+# release.
+RELEASE="development"
+VERSION="latest"
-# Formulas versions
+# The arvados-formula version. For a stable release, this should be a
+# branch name (e.g. X.Y-dev) or tag for the release.
ARVADOS_TAG="master"
+
+# Other formula versions we depend on
POSTGRES_TAG="v0.41.6"
NGINX_TAG="temp-fix-missing-statements-in-pillar"
DOCKER_TAG="v1.0.0"
# Get the formula and dependencies
cd ${F_DIR} || exit 1
-git clone --branch "${ARVADOS_TAG}" https://github.com/arvados/arvados-formula.git
+git clone --branch "${ARVADOS_TAG}" https://git.arvados.org/arvados-formula.git
git clone --branch "${DOCKER_TAG}" https://github.com/saltstack-formulas/docker-formula.git
git clone --branch "${LOCALE_TAG}" https://github.com/saltstack-formulas/locale-formula.git
# git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git
echo "${SOURCE_PILLARS_DIR} does not exist or is not a directory. Exiting."
exit 1
fi
-for f in "${SOURCE_PILLARS_DIR}"/*; do
+for f in $(ls "${SOURCE_PILLARS_DIR}"/*); do
sed "s#__ANONYMOUS_USER_TOKEN__#${ANONYMOUS_USER_TOKEN}#g;
s#__BLOB_SIGNING_KEY__#${BLOB_SIGNING_KEY}#g;
s#__CONTROLLER_EXT_SSL_PORT__#${CONTROLLER_EXT_SSL_PORT}#g;
s#__INITIAL_USER_EMAIL__#${INITIAL_USER_EMAIL}#g;
s#__INITIAL_USER_PASSWORD__#${INITIAL_USER_PASSWORD}#g;
s#__INITIAL_USER__#${INITIAL_USER}#g;
+ s#__LE_AWS_REGION__#${LE_AWS_REGION}#g;
+ s#__LE_AWS_SECRET_ACCESS_KEY__#${LE_AWS_SECRET_ACCESS_KEY}#g;
+ s#__LE_AWS_ACCESS_KEY_ID__#${LE_AWS_ACCESS_KEY_ID}#g;
s#__DATABASE_PASSWORD__#${DATABASE_PASSWORD}#g;
s#__KEEPWEB_EXT_SSL_PORT__#${KEEPWEB_EXT_SSL_PORT}#g;
s#__KEEP_EXT_SSL_PORT__#${KEEP_EXT_SSL_PORT}#g;
s#__KEEPSTORE1_INT_IP__#${KEEPSTORE1_INT_IP}#g;
s#__KEEPWEB_INT_IP__#${KEEPWEB_INT_IP}#g;
s#__WEBSHELL_INT_IP__#${WEBSHELL_INT_IP}#g;
+ s#__SHELL_INT_IP__#${SHELL_INT_IP}#g;
s#__WORKBENCH1_INT_IP__#${WORKBENCH1_INT_IP}#g;
s#__WORKBENCH2_INT_IP__#${WORKBENCH2_INT_IP}#g;
s#__DATABASE_INT_IP__#${DATABASE_INT_IP}#g;
fi
mkdir -p /tmp/cluster_tests
# Replace cluster and domain name in the test files
-for f in "${SOURCE_TESTS_DIR}"/*; do
+for f in $(ls "${SOURCE_TESTS_DIR}"/*); do
sed "s#__CLUSTER__#${CLUSTER}#g;
s#__CONTROLLER_EXT_SSL_PORT__#${CONTROLLER_EXT_SSL_PORT}#g;
s#__DOMAIN__#${DOMAIN}#g;
if [ -d "${SOURCE_STATES_DIR}" ]; then
mkdir -p "${F_DIR}"/extra/extra
- for f in "${SOURCE_STATES_DIR}"/*; do
+ for f in $(ls "${SOURCE_STATES_DIR}"/*); do
sed "s#__ANONYMOUS_USER_TOKEN__#${ANONYMOUS_USER_TOKEN}#g;
s#__CLUSTER__#${CLUSTER}#g;
s#__BLOB_SIGNING_KEY__#${BLOB_SIGNING_KEY}#g;
# States, extra states
if [ -d "${F_DIR}"/extra/extra ]; then
- for f in "${F_DIR}"/extra/extra/*.sls; do
+ for f in $(ls "${F_DIR}"/extra/extra/*.sls); do
echo " - extra.$(basename ${f} | sed 's/.sls$//g')" >> ${S_DIR}/top.sls
done
fi
if [ -z "${ROLES}" ]; then
# States
echo " - nginx.passenger" >> ${S_DIR}/top.sls
+ # Currently, only available on config_examples/multi_host/aws
if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
- grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
+ if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
+ grep -q "aws_credentials" ${S_DIR}/top.sls || echo " - aws_credentials" >> ${S_DIR}/top.sls
+ fi
+ grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
fi
echo " - postgres" >> ${S_DIR}/top.sls
- echo " - docker" >> ${S_DIR}/top.sls
+ echo " - docker.software" >> ${S_DIR}/top.sls
echo " - arvados" >> ${S_DIR}/top.sls
# Pillars
echo " - nginx_workbench2_configuration" >> ${P_DIR}/top.sls
echo " - nginx_workbench_configuration" >> ${P_DIR}/top.sls
echo " - postgresql" >> ${P_DIR}/top.sls
+ # Currently, only available on config_examples/multi_host/aws
if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
- grep -q "letsencrypt" ${P_DIR}/top.sls || echo " - letsencrypt" >> ${P_DIR}/top.sls
+ if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
+ grep -q "aws_credentials" ${P_DIR}/top.sls || echo " - aws_credentials" >> ${P_DIR}/top.sls
+ fi
+ grep -q "letsencrypt" ${P_DIR}/top.sls || echo " - letsencrypt" >> ${P_DIR}/top.sls
fi
else
# If we add individual roles, make sure we add the repo first
grep -q "nginx.passenger" ${S_DIR}/top.sls || echo " - nginx.passenger" >> ${S_DIR}/top.sls
### If we don't install and run LE before arvados-api-server, it fails and breaks everything
### after it so we add this here, as we are, after all, sharing the host for api and controller
+ # Currently, only available on config_examples/multi_host/aws
if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
- grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
+ if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
+ grep -q "aws_credentials" ${S_DIR}/top.sls || echo " - aws_credentials" >> ${S_DIR}/top.sls
+ fi
+ grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
fi
grep -q "arvados.${R}" ${S_DIR}/top.sls || echo " - arvados.${R}" >> ${S_DIR}/top.sls
# Pillars
+ grep -q "aws_credentials" ${P_DIR}/top.sls || echo " - aws_credentials" >> ${P_DIR}/top.sls
grep -q "docker" ${P_DIR}/top.sls || echo " - docker" >> ${P_DIR}/top.sls
grep -q "postgresql" ${P_DIR}/top.sls || echo " - postgresql" >> ${P_DIR}/top.sls
grep -q "nginx_passenger" ${P_DIR}/top.sls || echo " - nginx_passenger" >> ${P_DIR}/top.sls
grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo " - nginx_${R}_configuration" >> ${P_DIR}/top.sls
;;
- "controller" | "websocket" | "workbench" | "workbench2" | "keepweb" | "keepproxy")
+ "controller" | "websocket" | "workbench" | "workbench2" | "webshell" | "keepweb" | "keepproxy")
# States
grep -q "nginx.passenger" ${S_DIR}/top.sls || echo " - nginx.passenger" >> ${S_DIR}/top.sls
+ # Currently, only available on config_examples/multi_host/aws
if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
- grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
+ if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
+ grep -q "aws_credentials" ${S_DIR}/top.sls || echo " - aws_credentials" >> ${S_DIR}/top.sls
+ fi
+ grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
+ fi
+ # webshell role is just a nginx vhost, so it has no state
+ if [ "${R}" != "webshell" ]; then
+ grep -q "arvados.${R}" ${S_DIR}/top.sls || echo " - arvados.${R}" >> ${S_DIR}/top.sls
fi
- grep -q "arvados.${R}" ${S_DIR}/top.sls || echo " - arvados.${R}" >> ${S_DIR}/top.sls
# Pillars
grep -q "nginx_passenger" ${P_DIR}/top.sls || echo " - nginx_passenger" >> ${P_DIR}/top.sls
grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo " - nginx_${R}_configuration" >> ${P_DIR}/top.sls
+ # Currently, only available on config_examples/multi_host/aws
if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
- grep -q "letsencrypt" ${P_DIR}/top.sls || echo " - letsencrypt" >> ${P_DIR}/top.sls
+ if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
+ grep -q "aws_credentials" ${P_DIR}/top.sls || echo " - aws_credentials" >> ${P_DIR}/top.sls
+ fi
+ grep -q "letsencrypt" ${P_DIR}/top.sls || echo " - letsencrypt" >> ${P_DIR}/top.sls
grep -q "letsencrypt_${R}_configuration" ${P_DIR}/top.sls || echo " - letsencrypt_${R}_configuration" >> ${P_DIR}/top.sls
fi
;;
"shell")
# States
- grep -q "docker" ${S_DIR}/top.sls || echo " - docker" >> ${S_DIR}/top.sls
+ grep -q "docker" ${S_DIR}/top.sls || echo " - docker.software" >> ${S_DIR}/top.sls
grep -q "arvados.${R}" ${S_DIR}/top.sls || echo " - arvados.${R}" >> ${S_DIR}/top.sls
# Pillars
grep -q "" ${P_DIR}/top.sls || echo " - docker" >> ${P_DIR}/top.sls
- grep -q "nginx_webshell_configuration" ${P_DIR}/top.sls || echo " - nginx_webshell_configuration" >> ${P_DIR}/top.sls
;;
"dispatcher")
# States
- grep -q "docker" ${S_DIR}/top.sls || echo " - docker" >> ${S_DIR}/top.sls
+ grep -q "docker" ${S_DIR}/top.sls || echo " - docker.software" >> ${S_DIR}/top.sls
grep -q "arvados.${R}" ${S_DIR}/top.sls || echo " - arvados.${R}" >> ${S_DIR}/top.sls
# Pillars
# ATM, no specific pillar needed
projects / arvados.git / commitdiff
