mimemagic (0.3.8)
nokogiri (~> 1)
mini_mime (1.0.2)
- mini_portile2 (2.5.0)
+ mini_portile2 (2.5.1)
minitest (5.10.3)
mocha (1.8.0)
metaclass (~> 0.0.1)
net-ssh-gateway (2.0.0)
net-ssh (>= 4.0.0)
nio4r (2.5.7)
- nokogiri (1.11.2)
+ nokogiri (1.11.5)
mini_portile2 (~> 2.5.0)
racc (~> 1.4)
npm-rails (0.2.1)
api_params[:filters] = @filters if @filters
api_params[:distinct] = @distinct if @distinct
api_params[:include_trash] = @include_trash if @include_trash
+ api_params[:cluster_id] = Rails.configuration.ClusterID
if @fetch_multiple_pages
# Default limit to (effectively) api server's MAX_LIMIT
api_params[:limit] = 2**(0.size*8 - 1) - 1
<%= raw(Rails.configuration.Workbench.WelcomePageHTML) %>
<% case %>
- <% when Rails.configuration.Login.Google.Enable %>
- <% when Rails.configuration.Login.OpenIDConnect.Enable %>
- <% when Rails.configuration.Login.SSO.Enable %>
- <div class="pull-right">
- <%= link_to arvados_api_client.arvados_login_url(return_to: request.url), class: "btn btn-primary" do %>
- Log in to <%= Rails.configuration.Workbench.SiteName %>
- <i class="fa fa-fw fa-arrow-circle-right"></i>
- <% end %>
- </div>
- <% when Rails.configuration.Login.PAM.Enable %>
- <% when Rails.configuration.Login.LDAP.Enable %>
- <% when Rails.configuration.Login.Test.Enable %>
+ <% when Rails.configuration.Login.PAM.Enable,
+ Rails.configuration.Login.LDAP.Enable,
+ Rails.configuration.Login.Test.Enable %>
<form id="login-form-tag" onsubmit="controller_password_authenticate(event)">
<p>username <input type="text" class="form-control" name="login-username"
value="" id="login-username" style="width: 50%"
<span style="color: red"><p id="login-authenticate-error"></p></span>
<button type="submit" class="btn btn-primary">Log in</button>
</form>
+ <% else %>
+ <div class="pull-right">
+ <%= link_to arvados_api_client.arvados_login_url(return_to: request.url), class: "btn btn-primary" do %>
+ Log in to <%= Rails.configuration.Workbench.SiteName %>
+ <i class="fa fa-fw fa-arrow-circle-right"></i>
+ <% end %>
+ </div>
<% end %>
</div>
// kex_exchange_identification: Connection closed by remote host
// Connection closed by UNKNOWN port 65535
// exit status 255
+ //
+ // In case our target is a container request, the probe also
+ // resolves it to a container, so we don't connect to two
+ // different containers in a race.
+ var probetarget bytes.Buffer
exitcode := connectSSHCommand{}.RunCommand(
"arvados-client connect-ssh",
[]string{"-detach-keys=" + *detachKeys, "-probe-only=true", target},
- &bytes.Buffer{}, &bytes.Buffer{}, stderr)
+ &bytes.Buffer{}, &probetarget, stderr)
if exitcode != 0 {
return exitcode
}
+ target = strings.Trim(probetarget.String(), "\n")
selfbin, err := os.Readlink("/proc/self/exe")
if err != nil {
`)
f.PrintDefaults()
}
- probeOnly := f.Bool("probe-only", false, "do not transfer IO, just exit 0 immediately if tunnel setup succeeds")
+ probeOnly := f.Bool("probe-only", false, "do not transfer IO, just setup tunnel, print target UUID, and exit")
detachKeys := f.String("detach-keys", "", "set detach key sequence, as in docker-attach(1)")
if err := f.Parse(args); err != nil {
fmt.Fprintln(stderr, err)
defer sshconn.Conn.Close()
if *probeOnly {
+ fmt.Fprintln(stdout, targetUUID)
return 0
}
This is independent of @Workbench.IdleTimeout@. Even if Workbench auto-logout is disabled, this option will ensure that the user is always required to log in again after the configured amount of time.
-When this configuration is active (has a nonzero value), the Workbench client will also be "untrusted" by default. This means tokens issued to Workbench cannot be used to list other tokens issued to the user, and cannot be used to grant new tokens. This stops an attacker from leveraging a leaked token to aquire other tokens, but also interferes with some Workbench features that create new tokens on behalf of the user.
+h2. Untrusted login tokens
+
+<pre>
+Clusters:
+ zzzzz:
+ ...
+ Login:
+ TrustLoginTokens: false
+ ...
+</pre>
+
+When `TrustLoginTokens` is `false`, tokens issued through login will be "untrusted" by default. Untrusted tokens cannot be used to list other tokens issued to the user, and cannot be used to grant new tokens. This stops an attacker from leveraging a leaked token to aquire other tokens, but also interferes with some Workbench features that create new tokens on behalf of the user.
The default value @Login.TokenLifetime@ is zero, meaning login tokens do not expire (unless @API.MaxTokenLifetime@ is set).
...
</pre>
-Tokens created without an explicit expiration time, or that exceed maximum lifetime, will be clamped to @API.MaxTokenLifetime@.
+Tokens created without an explicit expiration time, or that exceed maximum lifetime, will be set to @API.MaxTokenLifetime@.
Similar to @Login.TokenLifetime@, this option ensures that the user is always required to log in again after the configured amount of time.
-Unlike @Login.TokenLifetime@, this applies to all API operations that manipulate tokens, regardless of whether the token was created by logging in, or by using the API. Also unlike @Login.TokenLifetime@, this setting does not imply any additional restrictions on token capabilities (it does not interfere with Workbench features that create new tokens on behalf of the user). If @Login.TokenLifetime@ is greater than @API.MaxTokenLifetime@, MaxTokenLifetime takes precedence.
+Unlike @Login.TokenLifetime@, this applies to all API operations that manipulate tokens, regardless of whether the token was created by logging in, or by using the API. If @Login.TokenLifetime@ is greater than @API.MaxTokenLifetime@, MaxTokenLifetime takes precedence.
-Admin users are permitted to create tokens with expiration times further in the future than MaxTokenLifetime, or with no expiration time at all.
+Admin users are permitted to create tokens with expiration times further in the future than @MaxTokenLifetime@.
The default value @MaxTokenLifetime@ is zero, which means there is no maximum token lifetime.
h2. Choosing a policy
-@Workbench.IdleTimeout@ only affects browser behavior. It is strongly recommended that automatic browser logout be used together with one or both token lifetime options, which are enforced on API side.
+@Workbench.IdleTimeout@ only affects browser behavior. It is strongly recommended that automatic browser logout be used together with @Login.TokenLifetime@, which is enforced on API side.
-@Login.TokenLifetime@ is more restrictive. A token obtained by logging into Workbench cannot be "refreshed" to gain access for an indefinite period. However, it interferes with some Workbench features, as well as ease of use in other contexts, such as the Arvados command line. This option is recommended only if most users will only ever interact with the system through Workbench or WebShell. For users or service accounts that need to tokens with fewer restrictions, the admin can "create a token at the command line":user-management-cli.html#create-token .
+@TrustLoginTokens: true@ (default value) is less restrictive. Be aware that an unrestricted token can be "refreshed" to gain access for an indefinite period. This means, during the window that the token is valid, the user is permitted to create a new token, which will have a new expiration further in the future (of course, once the token has expired, this is no longer possible). Unrestricted tokens are required for some Workbench features, as well as ease of use in other contexts, such as the Arvados command line. This option is recommended if many users will interact with the system through the command line.
-@API.MaxTokenLifetime@ is less restrictive. Be aware that an unrestricted token can be "refreshed" to gain access for an indefinite period. This means, during the window that the token is valid, the user is permitted to create a new token, which will have a new expiration further in the future (of course, once the token has expired, this is no longer possible). Unrestricted tokens are required for some Workbench features, as well as ease of use in other contexts, such as the Arvados command line. This option is recommended if many users will interact with the system through the command line.
+@TrustLoginTokens: false@ is more restrictive. A token obtained by logging into Workbench cannot be "refreshed" to gain access for an indefinite period. However, it interferes with some Workbench features, as well as ease of use in other contexts, such as the Arvados command line. This option is recommended only if most users will only ever interact with the system through Workbench or WebShell. For users or service accounts that need to tokens with fewer restrictions, the admin can "create a token at the command line":user-management-cli.html#create-token using the @SystemRootToken@.
-In every case, admin users may always create tokens with no expiration date.
+In every case, admin users may always create tokens with expiration dates far in the future.
These policies do not apply to tokens created by the API server for the purposes of authorizing a container to run, as those tokens are automatically expired when the container is finished.
|old_user_uuid|uuid|The uuid of the "old" account|query||
|new_owner_uuid|uuid|The uuid of a project to which objects owned by the "old" user will be reassigned.|query||
|redirect_to_new_user|boolean|If true, also redirect login and reassign authorization credentials from "old" user to the "new" user|query||
+
+h3. authenticate
+
+Create a new API token based on username/password credentials. Returns an "API client authorization":api_client_authorizations.html object containing the API token, or an "error object.":../requests.html#errors
+
+Valid credentials are determined by the choice of "configured login backend.":{{site.baseurl}}/install/setup-login.html
+
+Note: this endpoint cannot be used with login backends that use web-based third party authentication, such as Google or OpenID Connect.
+
+Arguments:
+
+table(table table-bordered table-condensed).
+|_. Argument |_. Type |_. Description |_. Location |_. Example |
+{background:#ccffcc}.|username|string|The username.|body||
+{background:#ccffcc}.|password|string|The password.|body||
API requests must provide the API token using the @Authorization@ header in the following format:
<pre>
-$ curl -v -H "Authorization: OAuth2 xxxxapitokenxxxx" https://192.168.5.2:8000/arvados/v1/collections
+$ curl -v -H "Authorization: Bearer xxxxapitokenxxxx" https://192.168.5.2:8000/arvados/v1/collections
> GET /arvados/v1/collections HTTP/1.1
> ...
-> Authorization: OAuth2 xxxxapitokenxxxx
+> Authorization: Bearer xxxxapitokenxxxx
> ...
</pre>
+On a cluster configured to use an OpenID Connect provider (other than Google) as a login backend, Arvados can be configured to accept an OpenID Connect access token in place of an Arvados API token. OIDC access tokens are also accepted by a cluster that delegates login to another cluster (LoginCluster) which in turn has this feature configured. See @Login.OpenIDConnect.AcceptAccessTokenScope@ in the "default config.yml file":{{site.baseurl}}/admin/config.html for details.
+
+<pre>
+$ curl -v -H "Authorization: Bearer xxxx-openid-connect-access-token-xxxx" https://192.168.5.2:8000/arvados/v1/collections
+</pre>
+
h3. Parameters
Request parameters may be provided in one of two ways. They may be provided in the "query" section of request URI, or they may be provided in the body of the request with application/x-www-form-urlencoded encoding. If parameters are provided in both places, their values will be merged. Parameter names must be unique. If a parameter appears multiple times, the behavior is undefined.
Results are returned JSON-encoded in the response body.
-h3. Errors
+h3(#errors). Errors
If a request cannot be fulfilled, the API will return 4xx or 5xx HTTP status code. Be aware that the API server may return a 404 (Not Found) status for resources that exist but for which the client does not have read access. The API will also return an error record:
h3. Create a new record
<pre>
-$ curl -v -X POST --data-urlencode 'collection={"name":"empty collection"}' -H "Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections | jq .
+$ curl -v -X POST --data-urlencode 'collection={"name":"empty collection"}' -H "Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections | jq .
> POST /arvados/v1/collections HTTP/1.1
> User-Agent: curl/7.38.0
> Host: 192.168.5.2:8000
> Accept: */*
-> Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
+> Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
> Content-Length: 54
> Content-Type: application/x-www-form-urlencoded
>
h3. Delete a record
<pre>
-$ curl -X DELETE -v -H "Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections/962eh-4zz18-m1ma0mxxfg3mbcc | jq .
+$ curl -X DELETE -v -H "Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections/962eh-4zz18-m1ma0mxxfg3mbcc | jq .
> DELETE /arvados/v1/collections/962eh-4zz18-m1ma0mxxfg3mbcc HTTP/1.1
> User-Agent: curl/7.38.0
> Host: 192.168.5.2:8000
> Accept: */*
-> Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
+> Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
>
< HTTP/1.1 200 OK
< Content-Type: application/json; charset=utf-8
h3. Get a specific record
<pre>
-$ curl -v -H "Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km | jq .
+$ curl -v -H "Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km | jq .
> GET /arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km HTTP/1.1
> User-Agent: curl/7.38.0
> Host: 192.168.5.2:8000
> Accept: */*
-> Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
+> Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
>
< HTTP/1.1 200 OK
< Content-Type: application/json; charset=utf-8
(Note, return result is truncated).
<pre>
-$ curl -v -G --data-urlencode 'filters=[["created_at",">","2016-11-08T21:38:24.124834000Z"]]' -H "Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections | jq .
+$ curl -v -G --data-urlencode 'filters=[["created_at",">","2016-11-08T21:38:24.124834000Z"]]' -H "Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections | jq .
> GET /arvados/v1/collections?filters=%5B%5B%22uuid%22%2C%20%22%3D%22%2C%20%22962eh-4zz18-xi32mpz2621o8km%22%5D%5D HTTP/1.1
> User-Agent: curl/7.38.0
> Host: 192.168.5.2:8000
> Accept: */*
-> Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
+> Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
>
< HTTP/1.1 200 OK
< Content-Type: application/json; charset=utf-8
h3. Update a field
<pre>
-$ curl -v -X PUT --data-urlencode 'collection={"name":"rna.SRR948778.bam"}' -H "Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km | jq .
+$ curl -v -X PUT --data-urlencode 'collection={"name":"rna.SRR948778.bam"}' -H "Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km | jq .
> PUT /arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km HTTP/1.1
> User-Agent: curl/7.38.0
> Host: 192.168.5.2:8000
> Accept: */*
-> Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
+> Authorization: Bearer oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
> Content-Length: 53
> Content-Type: application/x-www-form-urlencoded
>
All requests to the API server must have an API token. API tokens can be issued by going though the login flow, or created via the API. At this time, only browser based applications can perform login from email/password. Command line applications and services must use an API token provided via the @ARVADOS_API_TOKEN@ environment variable or configuration file.
-h2. Browser login
+h2. Login
-Browser based applications can perform log in via the following highlevel flow:
+Browser based applications can log in using one of the two possible flows:
-# The web application presents a "login" link to @/login@ on the API server with a @return_to@ parameter provided in the query portion of the URL. For example @https://{{ site.arvados_api_host }}/login?return_to=XXX@ , where @return_to=XXX@ is the URL of the login page for the web application.
-# The "login" link takes the browser to the login page (this may involve several redirects)
-# The user logs in. API server authenticates the user and issues a new API token.
-# The browser is redirected to the login page URL provided in @return_to=XXX@ with the addition of @?api_token=xxxxapitokenxxxx@.
-# The web application gets the login request with the included authorization token.
+h3. Authenticate via a third party
-!{{site.baseurl}}/images/Session_Establishment.svg!
+# The web application instructs the user to click on a link to the @/login@ endpoint on the API server. This link should include the @return_to@ parameter in the query portion of the URL. For example @https://{{ site.arvados_api_host }}/login?return_to=XXX@ , where @return_to=XXX@ is a page in the web application.
+# The @/login@ endpoint redirects the user to the configured third party authentication provider (e.g. Google or other OpenID Connect provider).
+# The user logs in to the third party provider, then they are redirected back to the API server.
+# The API server authenticates the user, issues a new API token, and redirects the browser to the URL provided in @return_to=XXX@ with the addition of @?api_token=xxxxapitokenxxxx@.
+# The web application gets the authorization token from the query and uses it to access the API server on the user's behalf.
+
+h3. Direct username/password authentication
+
+# The web application presents username and password fields.
+# When the submit button is pressed, using Javascript, the browser sends a POST request to @/arvados/v1/users/authenticate@
+** The request payload type is @application/javascript@
+** The request body is a JSON object with @username@ and @password@ fields.
+# The API server receives the username and password, authenticates them with the upstream provider (such as LDAP or PAM), and responds with the @api_client_authorization@ object for the new API token.
+# The web application receives the authorization token in the response and uses it to access the API server on the user's behalf.
+
+h3. Using an OpenID Connect access token
-The "browser authentication process is documented in detail on the Arvados wiki.":https://dev.arvados.org/projects/arvados/wiki/Workbench_authentication_process
+A cluster that uses OpenID Connect as a login provider can be configured to accept OIDC access tokens as well as Arvados API tokens (this is disabled by default; see @Login.OpenIDConnect.AcceptAccessToken@ in the "default config.yml file":{{site.baseurl}}/admin/config.html).
+# The client obtains an access token from the OpenID Connect provider via some method outside of Arvados.
+# The client presents the access token with an Arvados API request (e.g., request header @Authorization: Bearer xxxxaccesstokenxxxx@).
+# Depending on configuration, the API server decodes the access token (which must be a signed JWT) and confirms that it includes the required scope (see @Login.OpenIDConnect.AcceptAccessTokenScope@ in the "default config.yml file":{{site.baseurl}}/admin/config.html).
+# The API server uses the provider's UserInfo endpoint to validate the presented token.
+# If the token is valid, it is cached in the Arvados database and accepted in subsequent API calls for the next 10 minutes.
+
+h3. Diagram
+
+!{{site.baseurl}}/images/Session_Establishment.svg!
h2. User activation
|_. Distribution|_. State|_. Last supported version|
|CentOS 7|Supported|Latest|
|Debian 10 ("buster")|Supported|Latest|
+|Ubuntu 20.04 ("focal")|Supported|Latest|
|Ubuntu 18.04 ("bionic")|Supported|Latest|
-|Ubuntu 16.04 ("xenial")|Supported|Latest|
-|Debian 9 ("stretch")|EOL|Latest 2.1.X release|
+|Ubuntu 16.04 ("xenial")|EOL|2.1.2|
+|Debian 9 ("stretch")|EOL|2.1.2|
|Debian 8 ("jessie")|EOL|1.4.3|
|Ubuntu 14.04 ("trusty")|EOL|1.4.3|
|Ubuntu 12.04 ("precise")|EOL|8ed7b6dd5d4df93a3f37096afe6d6f81c2a7ef6e (2017-05-03)|
h2(#multi_host). Multi host install using the provision.sh script
-This is a package-based installation method. The Salt scripts are available from the "tools/salt-install":https://github.com/arvados/arvados/tree/master/tools/salt-install directory in the Arvados git repository.
+{% if site.current_version %}
+{% assign branchname = site.current_version | slice: 1, 5 | append: '-dev' %}
+{% else %}
+{% assign branchname = 'master' %}
+{% endif %}
-This procedure will install all the main Arvados components to get you up and running in a multi host environment.
+This is a package-based installation method. Start with the @provision.sh@ script which is available by cloning the @{{ branchname }}@ branch from "https://git.arvados.org/arvados.git":https://git.arvados.org/arvados.git . The @provision.sh@ script and its supporting files can be found in the "arvados/tools/salt-install":https://git.arvados.org/arvados.git/tree/refs/heads/{{ branchname }}:/tools/salt-install directory in the Arvados git repository.
-We suggest you to use the @provision.sh@ script to deploy Arvados, which is implemented with the @arvados-formula@ in a Saltstack master-less setup. After setting up a few variables in a config file (next step), you'll be ready to run it and get Arvados deployed.
+This procedure will install all the main Arvados components to get you up and running in a multi-host environment.
+
+The @provision.sh@ script will help you deploy Arvados by preparing your environment to be able to run the installer, then running it. The actual installer is located at "arvados-formula":https://git.arvados.org/arvados-formula.git/tree/refs/heads/{{ branchname }} and will be cloned during the running of the @provision.sh@ script. The installer is built using "Saltstack":https://saltproject.io/ and @provision.sh@ performs the install using master-less mode.
+
+After setting up a few variables in a config file (next step), you'll be ready to run it and get Arvados deployed.
h3(#create_a_compute_image). Create a compute image
</code></pre>
</notextile>
-h2(#initial_user). Initial user and login
+h2(#initial_user). Initial user and login
At this point you should be able to log into the Arvados cluster. The initial URL will be:
<b>NOTE: The single host installation is not recommended for production use.</b>
-This is a package-based installation method. The Salt scripts are available from the "tools/salt-install":https://github.com/arvados/arvados/tree/master/tools/salt-install directory in the Arvados git repository.
+{% if site.current_version %}
+{% assign branchname = site.current_version | slice: 1, 5 | append: '-dev' %}
+{% else %}
+{% assign branchname = 'master' %}
+{% endif %}
+
+This is a package-based installation method. Start with the @provision.sh@ script which is available by cloning the @{{ branchname }}@ branch from "https://git.arvados.org/arvados.git":https://git.arvados.org/arvados.git . The @provision.sh@ script and its supporting files can be found in the "arvados/tools/salt-install":https://git.arvados.org/arvados.git/tree/refs/heads/{{ branchname }}:/tools/salt-install directory in the Arvados git repository.
This procedure will install all the main Arvados components to get you up and running in a single host. The whole installation procedure takes somewhere between 15 to 60 minutes, depending on the host resources and its network bandwidth. As a reference, on a virtual machine with 1 core and 1 GB RAM, it takes ~25 minutes to do the initial install.
-We suggest you to use the @provision.sh@ script to deploy Arvados, which is implemented with the @arvados-formula@ in a Saltstack master-less setup. After setting up a few variables in a config file (next step), you'll be ready to run it and get Arvados deployed.
+The @provision.sh@ script will help you deploy Arvados by preparing your environment to be able to run the installer, then running it. The actual installer is located at "arvados-formula":https://git.arvados.org/arvados-formula.git/tree/refs/heads/{{ branchname }} and will be cloned during the running of the @provision.sh@ script. The installer is built using "Saltstack":https://saltproject.io/ and @provision.sh@ performs the install using master-less mode.
+
+After setting up a few variables in a config file (next step), you'll be ready to run it and get Arvados deployed.
h2(#choose_configuration). Choose the desired configuration
</code></pre>
</notextile>
-h2(#initial_user). Initial user and login
+h2(#initial_user). Initial user and login
At this point you should be able to log into the Arvados cluster. The initial URL will be:
Check the LDAP section in the "default config file":{{site.baseurl}}/admin/config.html for more details and configuration options.
-h2(#pam). PAM (experimental)
+h2(#pam). PAM
With this configuration, authentication is done according to the Linux PAM ("Pluggable Authentication Modules") configuration on your controller host.
Check the "default config file":{{site.baseurl}}/admin/config.html for more PAM configuration options.
-The default PAM configuration on most Linux systems uses the local password database in @/etc/shadow@ for all logins. In this case, in order to log in to Arvados, users must have a UNIX account and password on the controller host itself. This can be convenient for a single-user or test cluster. User accounts can have @/dev/false@ as the shell in order to allow the user to log into Arvados but not log into a shell on the controller host.
+The default PAM configuration on most Linux systems uses the local user/password database in @/etc/passwd@ and @/etc/shadow@ for all logins. In this case, in order to log in to Arvados, users must have a UNIX account and password on the controller host itself. This can be convenient for a single-user or test cluster. Configuring a user account with a shell of @/bin/false@ will enable the user to log into Arvados but not log into shell login on the controller host.
-PAM can also be configured to use different backends like LDAP. In a production environment, PAM configuration should use the service name ("arvados" by default) to set a separate policy for Arvados logins: generally, Arvados users should not have shell accounts on the controller node.
+PAM can also be configured to use other authentication systems such such as NIS or Kerberos. In a production environment, PAM configuration should use the service name ("arvados" by default) and set a separate policy for Arvados login. In this case, Arvados users should not have shell accounts on the controller node.
For information about configuring PAM, refer to the "PAM System Administrator's Guide":http://www.linux-pam.org/Linux-PAM-html/Linux-PAM_SAG.html.
UUIDTTL: 5s
# Block cache entries. Each block consumes up to 64 MiB RAM.
- MaxBlockEntries: 4
+ MaxBlockEntries: 20
# Collection cache entries.
MaxCollectionEntries: 1000
AuthenticationRequestParameters:
SAMPLE: ""
+ # Accept an OIDC access token as an API token if the OIDC
+ # provider's UserInfo endpoint accepts it.
+ #
+ # AcceptAccessTokenScope should also be used when enabling
+ # this feature.
+ AcceptAccessToken: false
+
+ # Before accepting an OIDC access token as an API token, first
+ # check that it is a JWT whose "scope" value includes this
+ # value. Example: "https://zzzzz.example.com/" (your Arvados
+ # API endpoint).
+ #
+ # If this value is empty and AcceptAccessToken is true, all
+ # access tokens will be accepted regardless of scope,
+ # including non-JWT tokens. This is not recommended.
+ AcceptAccessTokenScope: ""
+
PAM:
# (Experimental) Use PAM to authenticate users.
Enable: false
# Default value zero means tokens don't have expiration.
TokenLifetime: 0s
+ # If true (default) tokens issued through login are allowed to create
+ # new tokens.
+ # If false, tokens issued through login are not allowed to
+ # viewing/creating other tokens. New tokens can only be created
+ # by going through login again.
+ IssueTrustedTokens: true
+
# When the token is returned to a client, the token itself may
- # be restricted from manipulating other tokens based on whether
+ # be restricted from viewing/creating other tokens based on whether
# the client is "trusted" or not. The local Workbench1 and
# Workbench2 are trusted by default, but if this is a
# LoginCluster, you probably want to include the other Workbench
"Login.LDAP.UsernameAttribute": false,
"Login.LoginCluster": true,
"Login.OpenIDConnect": true,
+ "Login.OpenIDConnect.AcceptAccessToken": false,
+ "Login.OpenIDConnect.AcceptAccessTokenScope": false,
"Login.OpenIDConnect.AuthenticationRequestParameters": false,
"Login.OpenIDConnect.ClientID": false,
"Login.OpenIDConnect.ClientSecret": false,
"Login.Test.Enable": true,
"Login.Test.Users": false,
"Login.TokenLifetime": false,
+ "Login.IssueTrustedTokens": false,
"Login.TrustedClients": false,
"Mail": true,
"Mail.EmailFrom": false,
UUIDTTL: 5s
# Block cache entries. Each block consumes up to 64 MiB RAM.
- MaxBlockEntries: 4
+ MaxBlockEntries: 20
# Collection cache entries.
MaxCollectionEntries: 1000
AuthenticationRequestParameters:
SAMPLE: ""
+ # Accept an OIDC access token as an API token if the OIDC
+ # provider's UserInfo endpoint accepts it.
+ #
+ # AcceptAccessTokenScope should also be used when enabling
+ # this feature.
+ AcceptAccessToken: false
+
+ # Before accepting an OIDC access token as an API token, first
+ # check that it is a JWT whose "scope" value includes this
+ # value. Example: "https://zzzzz.example.com/" (your Arvados
+ # API endpoint).
+ #
+ # If this value is empty and AcceptAccessToken is true, all
+ # access tokens will be accepted regardless of scope,
+ # including non-JWT tokens. This is not recommended.
+ AcceptAccessTokenScope: ""
+
PAM:
# (Experimental) Use PAM to authenticate users.
Enable: false
# Default value zero means tokens don't have expiration.
TokenLifetime: 0s
+ # If true (default) tokens issued through login are allowed to create
+ # new tokens.
+ # If false, tokens issued through login are not allowed to
+ # viewing/creating other tokens. New tokens can only be created
+ # by going through login again.
+ IssueTrustedTokens: true
+
# When the token is returned to a client, the token itself may
- # be restricted from manipulating other tokens based on whether
+ # be restricted from viewing/creating other tokens based on whether
# the client is "trusted" or not. The local Workbench1 and
# Workbench2 are trusted by default, but if this is a
# LoginCluster, you probably want to include the other Workbench
cluster.Login.OpenIDConnect.ClientSecret = s.fakeProvider.ValidClientSecret
cluster.Login.OpenIDConnect.EmailClaim = "email"
cluster.Login.OpenIDConnect.EmailVerifiedClaim = "email_verified"
+ cluster.Login.OpenIDConnect.AcceptAccessToken = true
+ cluster.Login.OpenIDConnect.AcceptAccessTokenScope = ""
s.testHandler = &Handler{Cluster: cluster}
s.testServer = newServerFromIntegrationTestEnv(c)
_, err := fn(ctx, conn.cluster.ClusterID, conn.local, opts)
return err
}
+ if opts.ClusterID != "" {
+ // Client explicitly selected cluster
+ _, err := fn(ctx, conn.cluster.ClusterID, conn.chooseBackend(opts.ClusterID), opts)
+ return err
+ }
cannotSplit := false
var matchAllFilters map[string]bool
ClientSecret: ` + s.oidcprovider.ValidClientSecret + `
EmailClaim: email
EmailVerifiedClaim: email_verified
+ AcceptAccessToken: true
+ AcceptAccessTokenScope: ""
`
} else {
yaml += `
}
case wantOpenIDConnect:
return &oidcLoginController{
- Cluster: cluster,
- Parent: parent,
- Issuer: cluster.Login.OpenIDConnect.Issuer,
- ClientID: cluster.Login.OpenIDConnect.ClientID,
- ClientSecret: cluster.Login.OpenIDConnect.ClientSecret,
- AuthParams: cluster.Login.OpenIDConnect.AuthenticationRequestParameters,
- EmailClaim: cluster.Login.OpenIDConnect.EmailClaim,
- EmailVerifiedClaim: cluster.Login.OpenIDConnect.EmailVerifiedClaim,
- UsernameClaim: cluster.Login.OpenIDConnect.UsernameClaim,
+ Cluster: cluster,
+ Parent: parent,
+ Issuer: cluster.Login.OpenIDConnect.Issuer,
+ ClientID: cluster.Login.OpenIDConnect.ClientID,
+ ClientSecret: cluster.Login.OpenIDConnect.ClientSecret,
+ AuthParams: cluster.Login.OpenIDConnect.AuthenticationRequestParameters,
+ EmailClaim: cluster.Login.OpenIDConnect.EmailClaim,
+ EmailVerifiedClaim: cluster.Login.OpenIDConnect.EmailVerifiedClaim,
+ UsernameClaim: cluster.Login.OpenIDConnect.UsernameClaim,
+ AcceptAccessToken: cluster.Login.OpenIDConnect.AcceptAccessToken,
+ AcceptAccessTokenScope: cluster.Login.OpenIDConnect.AcceptAccessTokenScope,
}
case wantSSO:
return &ssoLoginController{Parent: parent}
"golang.org/x/oauth2"
"google.golang.org/api/option"
"google.golang.org/api/people/v1"
+ "gopkg.in/square/go-jose.v2/jwt"
)
var (
)
type oidcLoginController struct {
- Cluster *arvados.Cluster
- Parent *Conn
- Issuer string // OIDC issuer URL, e.g., "https://accounts.google.com"
- ClientID string
- ClientSecret string
- UseGooglePeopleAPI bool // Use Google People API to look up alternate email addresses
- EmailClaim string // OpenID claim to use as email address; typically "email"
- EmailVerifiedClaim string // If non-empty, ensure claim value is true before accepting EmailClaim; typically "email_verified"
- UsernameClaim string // If non-empty, use as preferred username
- AuthParams map[string]string // Additional parameters to pass with authentication request
+ Cluster *arvados.Cluster
+ Parent *Conn
+ Issuer string // OIDC issuer URL, e.g., "https://accounts.google.com"
+ ClientID string
+ ClientSecret string
+ UseGooglePeopleAPI bool // Use Google People API to look up alternate email addresses
+ EmailClaim string // OpenID claim to use as email address; typically "email"
+ EmailVerifiedClaim string // If non-empty, ensure claim value is true before accepting EmailClaim; typically "email_verified"
+ UsernameClaim string // If non-empty, use as preferred username
+ AcceptAccessToken bool // Accept access tokens as API tokens
+ AcceptAccessTokenScope string // If non-empty, don't accept access tokens as API tokens unless they contain this scope
+ AuthParams map[string]string // Additional parameters to pass with authentication request
// override Google People API base URL for testing purposes
// (normally empty, set by google pkg to
if !ok {
return loginError(errors.New("error in OAuth2 exchange: no ID token in OAuth2 token"))
}
+ ctxlog.FromContext(ctx).WithField("rawIDToken", rawIDToken).Debug("oauth2Token provided ID token")
idToken, err := ctrl.verifier.Verify(ctx, rawIDToken)
if err != nil {
return loginError(fmt.Errorf("error verifying ID token: %s", err))
if err != nil {
return fmt.Errorf("error setting up OpenID Connect provider: %s", err)
}
+ if ok, err := ta.checkAccessTokenScope(ctx, tok); err != nil || !ok {
+ ta.cache.Add(tok, time.Now().Add(tokenCacheNegativeTTL))
+ return err
+ }
oauth2Token := &oauth2.Token{
AccessToken: tok,
}
ta.cache.Add(tok, aca)
return nil
}
+
+// Check that the provided access token is a JWT with the required
+// scope. If it is a valid JWT but missing the required scope, we
+// return a 403 error, otherwise true (acceptable as an API token) or
+// false (pass through unmodified).
+//
+// Return false if configured not to accept access tokens at all.
+//
+// Note we don't check signature or expiry here. We are relying on the
+// caller to verify those separately (e.g., by calling the UserInfo
+// endpoint).
+func (ta *oidcTokenAuthorizer) checkAccessTokenScope(ctx context.Context, tok string) (bool, error) {
+ if !ta.ctrl.AcceptAccessToken {
+ return false, nil
+ } else if ta.ctrl.AcceptAccessTokenScope == "" {
+ return true, nil
+ }
+ var claims struct {
+ Scope string `json:"scope"`
+ }
+ if t, err := jwt.ParseSigned(tok); err != nil {
+ ctxlog.FromContext(ctx).WithError(err).Debug("error parsing jwt")
+ return false, nil
+ } else if err = t.UnsafeClaimsWithoutVerification(&claims); err != nil {
+ ctxlog.FromContext(ctx).WithError(err).Debug("error extracting jwt claims")
+ return false, nil
+ }
+ for _, s := range strings.Split(claims.Scope, " ") {
+ if s == ta.ctrl.AcceptAccessTokenScope {
+ return true, nil
+ }
+ }
+ ctxlog.FromContext(ctx).WithFields(logrus.Fields{"have": claims.Scope, "need": ta.ctrl.AcceptAccessTokenScope}).Infof("unacceptable access token scope")
+ return false, httpserver.ErrorWithStatus(errors.New("unacceptable access token scope"), http.StatusUnauthorized)
+}
json.Unmarshal([]byte(fmt.Sprintf("%q", s.fakeProvider.Issuer.URL)), &s.cluster.Login.OpenIDConnect.Issuer)
s.cluster.Login.OpenIDConnect.ClientID = "oidc#client#id"
s.cluster.Login.OpenIDConnect.ClientSecret = "oidc#client#secret"
+ s.cluster.Login.OpenIDConnect.AcceptAccessToken = true
+ s.cluster.Login.OpenIDConnect.AcceptAccessTokenScope = ""
s.fakeProvider.ValidClientID = "oidc#client#id"
s.fakeProvider.ValidClientSecret = "oidc#client#secret"
db := arvadostest.DB(c, s.cluster)
tokenCacheTTL = time.Millisecond
tokenCacheRaceWindow = time.Millisecond
+ tokenCacheNegativeTTL = time.Millisecond
oidcAuthorizer := OIDCAccessTokenAuthorizer(s.cluster, func(context.Context) (*sqlx.DB, error) { return db, nil })
accessToken := s.fakeProvider.ValidAccessToken()
mac := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
io.WriteString(mac, accessToken)
- hmac := fmt.Sprintf("%x", mac.Sum(nil))
+ apiToken := fmt.Sprintf("%x", mac.Sum(nil))
cleanup := func() {
- _, err := db.Exec(`delete from api_client_authorizations where api_token=$1`, hmac)
+ _, err := db.Exec(`delete from api_client_authorizations where api_token=$1`, apiToken)
c.Check(err, check.IsNil)
}
cleanup()
c.Assert(creds.Tokens, check.HasLen, 1)
c.Check(creds.Tokens[0], check.Equals, accessToken)
- err := db.QueryRowContext(ctx, `select expires_at at time zone 'UTC' from api_client_authorizations where api_token=$1`, hmac).Scan(&exp1)
+ err := db.QueryRowContext(ctx, `select expires_at at time zone 'UTC' from api_client_authorizations where api_token=$1`, apiToken).Scan(&exp1)
c.Check(err, check.IsNil)
c.Check(exp1.Sub(time.Now()) > -time.Second, check.Equals, true)
c.Check(exp1.Sub(time.Now()) < time.Second, check.Equals, true)
})(ctx, nil)
// If the token is used again after the in-memory cache
- // expires, oidcAuthorizer must re-checks the token and update
+ // expires, oidcAuthorizer must re-check the token and update
// the expires_at value in the database.
time.Sleep(3 * time.Millisecond)
oidcAuthorizer.WrapCalls(func(ctx context.Context, opts interface{}) (interface{}, error) {
var exp time.Time
- err := db.QueryRowContext(ctx, `select expires_at at time zone 'UTC' from api_client_authorizations where api_token=$1`, hmac).Scan(&exp)
+ err := db.QueryRowContext(ctx, `select expires_at at time zone 'UTC' from api_client_authorizations where api_token=$1`, apiToken).Scan(&exp)
c.Check(err, check.IsNil)
c.Check(exp.Sub(exp1) > 0, check.Equals, true)
c.Check(exp.Sub(exp1) < time.Second, check.Equals, true)
return nil, nil
})(ctx, nil)
+
+ s.fakeProvider.AccessTokenPayload = map[string]interface{}{"scope": "openid profile foobar"}
+ accessToken = s.fakeProvider.ValidAccessToken()
+ ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{accessToken}})
+
+ mac = hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
+ io.WriteString(mac, accessToken)
+ apiToken = fmt.Sprintf("%x", mac.Sum(nil))
+
+ for _, trial := range []struct {
+ configEnable bool
+ configScope string
+ acceptable bool
+ shouldRun bool
+ }{
+ {true, "foobar", true, true},
+ {true, "foo", false, false},
+ {true, "", true, true},
+ {false, "", false, true},
+ {false, "foobar", false, true},
+ } {
+ c.Logf("trial = %+v", trial)
+ cleanup()
+ s.cluster.Login.OpenIDConnect.AcceptAccessToken = trial.configEnable
+ s.cluster.Login.OpenIDConnect.AcceptAccessTokenScope = trial.configScope
+ oidcAuthorizer = OIDCAccessTokenAuthorizer(s.cluster, func(context.Context) (*sqlx.DB, error) { return db, nil })
+ checked := false
+ oidcAuthorizer.WrapCalls(func(ctx context.Context, opts interface{}) (interface{}, error) {
+ var n int
+ err := db.QueryRowContext(ctx, `select count(*) from api_client_authorizations where api_token=$1`, apiToken).Scan(&n)
+ c.Check(err, check.IsNil)
+ if trial.acceptable {
+ c.Check(n, check.Equals, 1)
+ } else {
+ c.Check(n, check.Equals, 0)
+ }
+ checked = true
+ return nil, nil
+ })(ctx, nil)
+ c.Check(checked, check.Equals, trial.shouldRun)
+ }
}
func (s *OIDCLoginSuite) TestGenericOIDCLogin(c *check.C) {
})
return nil
}
-
- return fmt.Errorf("Unsupported file type (mode %o) in output dir: %q", fi.Mode(), src)
+ cp.logger.Printf("Skipping unsupported file type (mode %o) in output dir: %q", fi.Mode(), src)
+ return nil
}
// Return the host path that was mounted at the given path in the
package crunchrun
import (
+ "bytes"
"io"
"io/ioutil"
"os"
+ "syscall"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "github.com/sirupsen/logrus"
check "gopkg.in/check.v1"
)
var _ = check.Suite(&copierSuite{})
type copierSuite struct {
- cp copier
+ cp copier
+ log bytes.Buffer
}
func (s *copierSuite) SetUpTest(c *check.C) {
- tmpdir, err := ioutil.TempDir("", "crunch-run.test.")
- c.Assert(err, check.IsNil)
+ tmpdir := c.MkDir()
api, err := arvadosclient.MakeArvadosClient()
c.Assert(err, check.IsNil)
+ s.log = bytes.Buffer{}
s.cp = copier{
client: arvados.NewClientFromEnv(),
arvClient: api,
secretMounts: map[string]arvados.Mount{
"/secret_text": {Kind: "text", Content: "xyzzy"},
},
+ logger: &logrus.Logger{Out: &s.log, Formatter: &logrus.TextFormatter{}, Level: logrus.InfoLevel},
}
}
-func (s *copierSuite) TearDownTest(c *check.C) {
- os.RemoveAll(s.cp.hostOutputDir)
-}
-
func (s *copierSuite) TestEmptyOutput(c *check.C) {
err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
c.Check(err, check.IsNil)
_, err = io.WriteString(f, "foo")
c.Assert(err, check.IsNil)
c.Assert(f.Close(), check.IsNil)
+ err = syscall.Mkfifo(s.cp.hostOutputDir+"/dir1/fifo", 0644)
+ c.Assert(err, check.IsNil)
err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
c.Check(err, check.IsNil)
{src: os.DevNull, dst: "/dir1/dir2/dir3/.keep"},
{src: s.cp.hostOutputDir + "/dir1/foo", dst: "/dir1/foo", size: 3},
})
+ c.Check(s.log.String(), check.Matches, `.* msg="Skipping unsupported file type \(mode 200000644\) in output dir: \\"/ctr/outdir/dir1/fifo\\""\n`)
}
func (s *copierSuite) TestSymlinkCycle(c *check.C) {
} else {
err = inst.runBash(`
cd /tmp
+rm -rf /var/lib/arvados/go/
wget --progress=dot:giga -O- https://storage.googleapis.com/golang/go`+goversion+`.linux-amd64.tar.gz | tar -C /var/lib/arvados -xzf -
ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
`, stdout, stderr)
EmailClaim string
EmailVerifiedClaim string
UsernameClaim string
+ AcceptAccessToken bool
+ AcceptAccessTokenScope string
AuthenticationRequestParameters map[string]string
}
PAM struct {
RemoteTokenRefresh Duration
TokenLifetime Duration
TrustedClients map[string]struct{}
+ IssueTrustedTokens bool
}
Mail struct {
MailchimpAPIKey string
"gopkg.in/check.v1"
"gopkg.in/square/go-jose.v2"
+ "gopkg.in/square/go-jose.v2/jwt"
)
type OIDCProvider struct {
ValidClientID string
ValidClientSecret string
// desired response from token endpoint
- AuthEmail string
- AuthEmailVerified bool
- AuthName string
+ AuthEmail string
+ AuthEmailVerified bool
+ AuthName string
+ AccessTokenPayload map[string]interface{}
PeopleAPIResponse map[string]interface{}
c.Assert(err, check.IsNil)
p.Issuer = httptest.NewServer(http.HandlerFunc(p.serveOIDC))
p.PeopleAPI = httptest.NewServer(http.HandlerFunc(p.servePeopleAPI))
+ p.AccessTokenPayload = map[string]interface{}{"sub": "example"}
return p
}
func (p *OIDCProvider) ValidAccessToken() string {
- return p.fakeToken([]byte("fake access token"))
+ buf, _ := json.Marshal(p.AccessTokenPayload)
+ return p.fakeToken(buf)
}
func (p *OIDCProvider) serveOIDC(w http.ResponseWriter, req *http.Request) {
case "/auth":
w.WriteHeader(http.StatusInternalServerError)
case "/userinfo":
- if authhdr := req.Header.Get("Authorization"); strings.TrimPrefix(authhdr, "Bearer ") != p.ValidAccessToken() {
+ authhdr := req.Header.Get("Authorization")
+ if _, err := jwt.ParseSigned(strings.TrimPrefix(authhdr, "Bearer ")); err != nil {
p.c.Logf("OIDCProvider: bad auth %q", authhdr)
w.WriteHeader(http.StatusUnauthorized)
return
# Monkeypatch google-api-client gem to avoid sending newline characters
# on headers to make ruby-2.3.7+ happy.
# See: https://dev.arvados.org/issues/13920
-Google::APIClient::ENV::OS_VERSION.strip!
+# Addendum: OS_VERSION sometimes is not mutable, that's why we duplicate it (See: #17738)
+Google::APIClient::ENV::OS_VERSION = +Google::APIClient::ENV::OS_VERSION.strip!
require 'json'
require 'tempfile'
mimemagic (0.3.8)
nokogiri (~> 1)
mini_mime (1.0.2)
- mini_portile2 (2.5.0)
+ mini_portile2 (2.5.1)
minitest (5.10.3)
mocha (1.8.0)
metaclass (~> 0.0.1)
net-ssh-gateway (2.0.0)
net-ssh (>= 4.0.0)
nio4r (2.5.7)
- nokogiri (1.11.2)
+ nokogiri (1.11.5)
mini_portile2 (~> 2.5.0)
racc (~> 1.4)
oauth2 (1.4.1)
end
def is_trusted
- (from_trusted_url && Rails.configuration.Login.TokenLifetime == 0) || super
+ (from_trusted_url && Rails.configuration.Login.IssueTrustedTokens) || super
end
protected
protected
def clamp_token_expiration
- if !current_user.andand.is_admin && Rails.configuration.API.MaxTokenLifetime > 0
+ if Rails.configuration.API.MaxTokenLifetime > 0
max_token_expiration = db_current_time + Rails.configuration.API.MaxTokenLifetime
- if (self.new_record? || self.expires_at_changed?) && (self.expires_at.nil? || self.expires_at > max_token_expiration)
+ if (self.new_record? || self.expires_at_changed?) && (self.expires_at.nil? || (self.expires_at > max_token_expiration && !current_user.andand.is_admin))
self.expires_at = max_token_expiration
end
end
end
end
- test "expires_at can be set to #{desired_expiration.nil? ? 'nil' : 'exceed the limit'} by admins when API.MaxTokenLifetime is set" do
+ test "behavior when expires_at is set to #{desired_expiration.nil? ? 'nil' : 'exceed the limit'} by admins when API.MaxTokenLifetime is set" do
Rails.configuration.API.MaxTokenLifetime = 1.hour
# Test token creation
headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin_trustedclient).api_token}"}
assert_response 200
if desired_expiration.nil?
- assert json_response['expires_at'].nil?
+ # When expires_at is nil, default to MaxTokenLifetime
+ assert_operator (json_response['expires_at'].to_time.to_i - (db_current_time + Rails.configuration.API.MaxTokenLifetime).to_i).abs, :<, 2
else
assert_equal json_response['expires_at'].to_time.to_i, desired_expiration.to_i
end
# Test token update (reverse the above behavior)
- previous_expiration = json_response['expires_at']
token_uuid = json_response['uuid']
- if previous_expiration.nil?
- desired_updated_expiration = db_current_time + Rails.configuration.API.MaxTokenLifetime + 1.hour
+ if desired_expiration.nil?
+ submitted_updated_expiration = db_current_time + Rails.configuration.API.MaxTokenLifetime + 1.hour
else
- desired_updated_expiration = nil
+ submitted_updated_expiration = nil
end
put "/arvados/v1/api_client_authorizations/#{token_uuid}",
params: {
:api_client_authorization => {
- :expires_at => desired_updated_expiration,
+ :expires_at => submitted_updated_expiration,
}
},
headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin_trustedclient).api_token}"}
assert_response 200
- if desired_updated_expiration.nil?
- assert json_response['expires_at'].nil?
+ if submitted_updated_expiration.nil?
+ assert_operator (json_response['expires_at'].to_time.to_i - (db_current_time + Rails.configuration.API.MaxTokenLifetime).to_i).abs, :<, 2
else
- assert_equal json_response['expires_at'].to_time.to_i, desired_updated_expiration.to_i
+ assert_equal json_response['expires_at'].to_time.to_i, submitted_updated_expiration.to_i
end
end
end
[true, false].each do |token_lifetime_enabled|
test "configured workbench is trusted when token lifetime is#{token_lifetime_enabled ? '': ' not'} enabled" do
Rails.configuration.Login.TokenLifetime = token_lifetime_enabled ? 8.hours : 0
+ Rails.configuration.Login.IssueTrustedTokens = !token_lifetime_enabled;
Rails.configuration.Services.Workbench1.ExternalURL = URI("http://wb1.example.com")
Rails.configuration.Services.Workbench2.ExternalURL = URI("https://wb2.example.com:443")
Rails.configuration.Login.TrustedClients = ActiveSupport::OrderedOptions.new
},
})
if err == nil {
- c.collections.Add(client.AuthToken+"\000"+coll.PortableDataHash, &cachedCollection{
+ c.collections.Add(client.AuthToken+"\000"+updated.PortableDataHash, &cachedCollection{
expire: time.Now().Add(time.Duration(c.config.TTL)),
collection: &updated,
})
c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
}
+// Writing to a collection shouldn't affect its entry in the
+// PDH-to-manifest cache.
+func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.Equals, nil)
+ arv.ApiToken = arvadostest.ActiveToken
+
+ u := mustParseURL("http://x.example/testfile")
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
+ }
+
+ checkWithID := func(id string, status int) {
+ req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
+ req.Host = req.URL.Host
+ resp := httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, status)
+ }
+
+ var colls [2]arvados.Collection
+ for i := range colls {
+ err := arv.Create("collections",
+ map[string]interface{}{
+ "ensure_unique_name": true,
+ "collection": map[string]interface{}{
+ "name": "test collection",
+ },
+ }, &colls[i])
+ c.Assert(err, check.Equals, nil)
+ }
+
+ // Populate cache with empty collection
+ checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
+
+ // write a file to colls[0]
+ reqPut := *req
+ reqPut.Method = "PUT"
+ reqPut.URL.Host = colls[0].UUID + ".example"
+ reqPut.Host = req.URL.Host
+ reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
+ resp := httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, &reqPut)
+ c.Check(resp.Code, check.Equals, http.StatusCreated)
+
+ // new file should not appear in colls[1]
+ checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
+ checkWithID(colls[1].UUID, http.StatusNotFound)
+
+ checkWithID(colls[0].UUID, http.StatusOK)
+}
+
func copyHeader(h http.Header) http.Header {
hc := http.Header{}
for k, v := range h {
EXTRA2+=" -var public_key_file=$PUBLIC_KEY_FILE"
fi
+echo
+packer version
+echo
echo packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE
packer build$EXTRA -var "arvados_cluster=$ARVADOS_CLUSTER_ID"$EXTRA2 $JSON_FILE
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+aws_credentials:
+ region: __LE_AWS_REGION__
+ access_key_id: __LE_AWS_ACCESS_KEY_ID__
+ secret_access_key: __LE_AWS_SECRET_ACCESS_KEY__
- download.__CLUSTER__.__DOMAIN__
collections.__CLUSTER__.__DOMAIN__:
- collections.__CLUSTER__.__DOMAIN__
- - *.collections.__CLUSTER__.__DOMAIN__
+ - '*.collections.__CLUSTER__.__DOMAIN__'
### NGINX
nginx:
enabled: true
overwrite: true
requires:
- cmd: create-initial-cert-collections.__CLUSTER__.__DOMAIN__-collections.__CLUSTER__.__DOMAIN__
+ cmd: 'create-initial-cert-collections.__CLUSTER__.__DOMAIN__-collections.__CLUSTER__.__DOMAIN__+*.__CLUSTER__.__DOMAIN__'
config:
- server:
- server_name: '*.collections.__CLUSTER__.__DOMAIN__'
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+{%- set aws_credentials = pillar.get('aws_credentials', {}) %}
+
+{%- if aws_credentials %}
+extra_extra_aws_credentials_root_aws_config_file_managed:
+ file.managed:
+ - name: /root/.aws/config
+ - makedirs: true
+ - user: root
+ - group: root
+ - mode: '0600'
+ - replace: false
+ - contents: |
+ [default]
+ region= {{ aws_credentials.region }}
+
+extra_extra_aws_credentials_root_aws_credentials_file_managed:
+ file.managed:
+ - name: /root/.aws/credentials
+ - makedirs: true
+ - user: root
+ - group: root
+ - mode: '0600'
+ - replace: false
+ - contents: |
+ [default]
+ aws_access_key_id = {{ aws_credentials.access_key_id }}
+ aws_secret_access_key = {{ aws_credentials.secret_access_key }}
+{%- endif %}
# variable to "no", provide and upload your own certificates to the instances and
# modify the 'nginx_*' salt pillars accordingly
USE_LETSENCRYPT="yes"
+USE_LETSENCRYPT_IAM_USER="yes"
+# For collections, we need to obtain a wildcard certificate for
+# '*.collections.<cluster>.<domain>'. This is only possible through a DNS-01 challenge.
+# For that reason, you'll need to provide AWS credentials with permissions to manage
+# RRs in the route53 zone for the cluster.
+# WARNING!: If AWS credentials files already exist in the hosts, they won't be replaced.
+LE_AWS_REGION="us-east-1"
+LE_AWS_ACCESS_KEY_ID="AKIABCDEFGHIJKLMNOPQ"
+LE_AWS_SECRET_ACCESS_KEY="thisistherandomstringthatisyoursecretkey"
# The directory to check for the config files (pillars, states) you want to use.
# There are a few examples under 'config_examples'.
WORKBENCH1_EXT_SSL_PORT=443
WORKBENCH2_EXT_SSL_PORT=3001
-RELEASE="production"
-VERSION="2.1.2-1"
-
-# Formulas versions
+# For a stable release, change RELEASE "production" and VERSION to the
+# package version (including the iteration, e.g. X.Y.Z-1) of the
+# release.
+RELEASE="development"
+VERSION="latest"
+
+# The arvados-formula version. For a stable release, this should be a
+# branch name (e.g. X.Y-dev) or tag for the release.
ARVADOS_TAG="master"
+
+# Other formula versions we depend on
POSTGRES_TAG="v0.41.6"
NGINX_TAG="temp-fix-missing-statements-in-pillar"
DOCKER_TAG="v1.0.0"
s#__INITIAL_USER_EMAIL__#${INITIAL_USER_EMAIL}#g;
s#__INITIAL_USER_PASSWORD__#${INITIAL_USER_PASSWORD}#g;
s#__INITIAL_USER__#${INITIAL_USER}#g;
+ s#__LE_AWS_REGION__#${LE_AWS_REGION}#g;
+ s#__LE_AWS_SECRET_ACCESS_KEY__#${LE_AWS_SECRET_ACCESS_KEY}#g;
+ s#__LE_AWS_ACCESS_KEY_ID__#${LE_AWS_ACCESS_KEY_ID}#g;
s#__DATABASE_PASSWORD__#${DATABASE_PASSWORD}#g;
s#__KEEPWEB_EXT_SSL_PORT__#${KEEPWEB_EXT_SSL_PORT}#g;
s#__KEEP_EXT_SSL_PORT__#${KEEP_EXT_SSL_PORT}#g;
if [ -z "${ROLES}" ]; then
# States
echo " - nginx.passenger" >> ${S_DIR}/top.sls
+ # Currently, only available on config_examples/multi_host/aws
if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
- grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
+ if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
+ grep -q "aws_credentials" ${S_DIR}/top.sls || echo " - aws_credentials" >> ${S_DIR}/top.sls
+ fi
+ grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
fi
echo " - postgres" >> ${S_DIR}/top.sls
echo " - docker.software" >> ${S_DIR}/top.sls
echo " - nginx_workbench2_configuration" >> ${P_DIR}/top.sls
echo " - nginx_workbench_configuration" >> ${P_DIR}/top.sls
echo " - postgresql" >> ${P_DIR}/top.sls
+ # Currently, only available on config_examples/multi_host/aws
if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
- grep -q "letsencrypt" ${P_DIR}/top.sls || echo " - letsencrypt" >> ${P_DIR}/top.sls
+ if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
+ grep -q "aws_credentials" ${P_DIR}/top.sls || echo " - aws_credentials" >> ${P_DIR}/top.sls
+ fi
+ grep -q "letsencrypt" ${P_DIR}/top.sls || echo " - letsencrypt" >> ${P_DIR}/top.sls
fi
else
# If we add individual roles, make sure we add the repo first
grep -q "nginx.passenger" ${S_DIR}/top.sls || echo " - nginx.passenger" >> ${S_DIR}/top.sls
### If we don't install and run LE before arvados-api-server, it fails and breaks everything
### after it so we add this here, as we are, after all, sharing the host for api and controller
+ # Currently, only available on config_examples/multi_host/aws
if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
- grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
+ if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
+ grep -q "aws_credentials" ${S_DIR}/top.sls || echo " - aws_credentials" >> ${S_DIR}/top.sls
+ fi
+ grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
fi
grep -q "arvados.${R}" ${S_DIR}/top.sls || echo " - arvados.${R}" >> ${S_DIR}/top.sls
# Pillars
+ grep -q "aws_credentials" ${P_DIR}/top.sls || echo " - aws_credentials" >> ${P_DIR}/top.sls
grep -q "docker" ${P_DIR}/top.sls || echo " - docker" >> ${P_DIR}/top.sls
grep -q "postgresql" ${P_DIR}/top.sls || echo " - postgresql" >> ${P_DIR}/top.sls
grep -q "nginx_passenger" ${P_DIR}/top.sls || echo " - nginx_passenger" >> ${P_DIR}/top.sls
"controller" | "websocket" | "workbench" | "workbench2" | "webshell" | "keepweb" | "keepproxy")
# States
grep -q "nginx.passenger" ${S_DIR}/top.sls || echo " - nginx.passenger" >> ${S_DIR}/top.sls
+ # Currently, only available on config_examples/multi_host/aws
if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
- grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
+ if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
+ grep -q "aws_credentials" ${S_DIR}/top.sls || echo " - aws_credentials" >> ${S_DIR}/top.sls
+ fi
+ grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
fi
# webshell role is just a nginx vhost, so it has no state
if [ "${R}" != "webshell" ]; then
# Pillars
grep -q "nginx_passenger" ${P_DIR}/top.sls || echo " - nginx_passenger" >> ${P_DIR}/top.sls
grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo " - nginx_${R}_configuration" >> ${P_DIR}/top.sls
+ # Currently, only available on config_examples/multi_host/aws
if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
- grep -q "letsencrypt" ${P_DIR}/top.sls || echo " - letsencrypt" >> ${P_DIR}/top.sls
+ if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
+ grep -q "aws_credentials" ${P_DIR}/top.sls || echo " - aws_credentials" >> ${P_DIR}/top.sls
+ fi
+ grep -q "letsencrypt" ${P_DIR}/top.sls || echo " - letsencrypt" >> ${P_DIR}/top.sls
grep -q "letsencrypt_${R}_configuration" ${P_DIR}/top.sls || echo " - letsencrypt_${R}_configuration" >> ${P_DIR}/top.sls
fi
;;
projects / arvados.git / commitdiff