# See: https://github.com/rails/sprockets-rails/issues/443
gem 'sprockets', '~> 3.0'
-# Fast app boot times
-gem 'bootsnap', require: false
-
# Note: keeping this out of the "group :assets" section "may" allow us
# to use Coffescript for UJS responses. It also prevents a
# warning/problem when running tests: "WARN: tilt autoloading
multi_json (>= 1.0.0)
autoprefixer-rails (9.5.1.1)
execjs
- bootsnap (1.4.7)
- msgpack (~> 1.0)
bootstrap-sass (3.4.1)
autoprefixer-rails (>= 5.2.1)
sassc (>= 2.0.0)
metaclass (~> 0.0.1)
morrisjs-rails (0.5.1.2)
railties (> 3.1, < 6)
- msgpack (1.3.3)
multi_json (1.15.0)
multipart-post (2.1.1)
net-scp (2.0.0)
andand
angularjs-rails (~> 1.3.8)
arvados (~> 2.1.5)
- bootsnap
bootstrap-sass (~> 3.4.1)
bootstrap-tab-history-rails
bootstrap-x-editable-rails
uglifier (~> 2.0)
BUNDLED WITH
- 1.17.3
+ 2.2.19
ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
require 'bundler/setup' if File.exists?(ENV['BUNDLE_GEMFILE'])
-require 'bootsnap/setup' # Speed up boot time by caching expensive operations.
# Use ARVADOS_API_TOKEN environment variable (if set) in console
require 'rails'
test "trying to use expired token redirects to login page" do
visit page_with_token('expired_trustedclient')
- buttons = all("a.btn", text: /Log in/)
+ buttons = all("button.btn", text: /Log in/)
assert_equal(1, buttons.size, "Failed to find one login button")
- login_link = buttons.first[:href]
- assert_match(%r{//[^/]+/login}, login_link)
- assert_no_match(/\bapi_token=/, login_link)
end
end
- api/requests.html.textile.liquid
- api/methods.html.textile.liquid
- api/resources.html.textile.liquid
- - api/tokens_sso.html.textile.liquid
- Permission and authentication:
- api/methods/api_client_authorizations.html.textile.liquid
- api/methods/api_clients.html.textile.liquid
+++ /dev/null
----
-layout: default
-navsection: api
-title: API Authorization with SSO (deprecated)
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-{% include 'notebox_begin_warning' %}
-This page describes the deprecated login flow via the SSO server. SSO server authentication will be removed in a future Arvados release and should not be used for new installations. See "Set up web based login":{{site.baseurl}}/install/setup-login.html for more information.
-{% include 'notebox_end' %}
-
-All requests to the API server must have an API token. API tokens can be issued by going though the login flow, or created via the API. At this time, only browser based applications can perform login from email/password. Command line applications and services must use an API token provided via the @ARVADOS_API_TOKEN@ environment variable or configuration file.
-
-h2. Browser login
-
-Browser based applications can perform log in via the following highlevel flow:
-
-# The web application presents a "login" link to @/login@ on the API server with a @return_to@ parameter provided in the query portion of the URL. For example @https://{{ site.arvados_api_host }}/login?return_to=XXX@ , where @return_to=XXX@ is the URL of the login page for the web application.
-# The "login" link takes the browser to the login page (this may involve several redirects)
-# The user logs in. API server authenticates the user and issues a new API token.
-# The browser is redirected to the login page URL provided in @return_to=XXX@ with the addition of @?api_token=xxxxapitokenxxxx@.
-# The web application gets the login request with the included authorization token.
-
-!{{site.baseurl}}/images/Session_Establishment_with_SSO.svg!
-
-The "browser authentication process is documented in detail on the Arvados wiki.":https://dev.arvados.org/projects/arvados/wiki/Workbench_authentication_process
-
-h2. User activation
-
-"Creation and activation of new users is described here.":{{site.baseurl}}/admin/user-management.html
-
-h2. Creating tokens via the API
-
-The browser login method above issues a new token. Using that token, it is possible to make API calls to create additional tokens. To do so, use the @create@ method of the "API client authorizations":{{site.baseurl}}/api/methods/api_client_authorizations.html resource.
-
-h2. Trusted API clients
-
-The "api_clients":{{site.baseurl}}/api/methods/api_clients.html resource determines if web applications that have gone through the browser login flow may create or list API tokens.
-
-After the user has authenticated, but before an authorization token is issued and browser redirect sent (sending the browser back to the @return_to@ login page bearing @api_token@), the server strips the path and query portion from @return_to@ to get @url_prefix@. The @url_prefix@ is used to find or create an ApiClient object. The newly issued API client authorization (API token) is associated with this ApiClient object.
-
-API clients may be marked as "trusted" by making an API call to create or update an "api_clients":{{site.baseurl}}/api/methods/api_clients.html resource and set the @is_trusted@ flag to @true@. An authorization token associated with a "trusted" client is permitted to list authorization tokens on "API client authorizations":{{site.baseurl}}/api/methods/api_client_authorizations.html .
-
-A authorization token which is not associated with a trusted client may only use the @current@ method to query its own api_client_authorization object. The "untrusted" token is forbidden performing any other operations on API client authorizations, such as listing other authorizations or creating new authorizations.
-
-Authorization tokens which are not issued via the browser login flow (created directly via the API) inherit the api client of the token used to create them. They will always be "trusted" because untrusted API clients cannot create tokens.
-
-h2(#scopes). Scopes
-
-Scopes can restrict a token so it may only access certain resources. This is in addition to normal permission checks for the user associated with the token.
-
-Each entry in scopes consists of a @request_method@ and @request_path@. The @request_method@ is a HTTP method (one of @GET@, @POST@, @PATCH@ or @DELETE@) and @request_path@ is the request URI. A given request is permitted if it matches a scopes exactly, or the scope ends with @/@ and the request string is a prefix of the scope.
-
-As a special case, a scope of @["all"]@ allows all resources. This is the default if no scope is given.
-
-Using scopes is also described on the "Securing API access with scoped tokens":{{site.baseurl}}/admin/scoped-tokens.html page of the admin documentation.
-
-h3. Scope examples
-
-A scope of @GET /arvados/v1/collections@ permits listing collections.
-
-* Requests with different methods, such as creating a new collection using @POST /arvados/v1/collections@, will be rejected.
-* Requests to access other resources, such as @GET /arvados/v1/groups@, will be rejected.
-* Be aware that requests for specific records, such as @GET /arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km@ will also be rejected. This is because the scope @GET /arvados/v1/collections@ does not end in @/@
-
-A scope of @GET /arvados/v1/collections/@ (with @/@ suffix) will permit access to individual collections.
-
-* The request @GET /arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km@ will succeed
-* Be aware that requests for listing @GET /arvados/v1/collections@ (no @/@ suffix) will be rejected, because it is not a match with the rule @GET /arvados/v1/collections/@
-* A listing request @GET /arvados/v1/collections/@ will have the trailing @/@ suffix trimmed before the scope check, as a result it will not match the rule @GET /arvados/v1/collections/@.
-
-To allow both listing objects and requesting individual objects, include both in the scope: @["GET /arvados/v1/collections", "GET /arvados/v1/collections/"]@
-
-A narrow scope such as @GET /arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km@ will disallow listing objects as well as disallow requesting any object other than those listed in the scope.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+# "Introduction":#introduction
# "Hosts preparation":#hosts_preparation
## "Hosts setup using terraform (experimental)":#hosts_setup_using_terraform
## "Create a compute image":#create_a_compute_image
# "Initial user and login":#initial_user
# "Test the installed cluster running a simple workflow":#test_install
+
+
+h2(#introduction). Introduction
+
+Arvados components can be installed in a distributed infrastructure, whether it is an "on-prem" with physical or virtual hosts, or a cloud environment.
+
+As infrastructures vary a great deal from site to site, these instructions should be considered more as 'guidelines' than fixed steps to follow.
+
+We provide an "installer script":salt.html that can help you deploy the different Arvados components. At the time of writing, the provided examples are suitable to install Arvados on AWS.
+
+
+
h2(#hosts_preparation). Hosts preparation
In order to run Arvados on a multi-host installation, there are a few requirements that your infrastructure has to fulfill.
Note that these hosts can be virtual machines in your infrastructure and they don't need to be physical machines.
-h3(#hosts_setup_using_terraform). Hosts setup using terraform (experimental)
+Again, if your infrastructure differs from the setup proposed above (ie, using RDS or an existing DB server), remember that you will need to edit the configuration files for the scripts so they work with your infrastructure.
+
+
+h3(#hosts_setup_using_terraform). Hosts setup using terraform (AWS, experimental)
+
+We added a few "terraform":https://terraform.io/ scripts (https://github.com/arvados/arvados/tree/master/tools/terraform) to let you create these instances easier in an AWS account. Check "the Arvados terraform documentation":/doc/install/terraform.html for more details.
+
+
-We added a few "terraform":https://terraform.io/ scripts (https://github.com/arvados/arvados/tree/master/tools/terraform) to let you create these instances easier.
-Check "the Arvados terraform documentation":/doc/install/terraform.html for more details.
h2(#multi_host). Multi host install using the provision.sh script
{% assign branchname = 'master' %}
{% endif %}
-This is a package-based installation method. Start with the @provision.sh@ script which is available by cloning the @{{ branchname }}@ branch from "https://git.arvados.org/arvados.git":https://git.arvados.org/arvados.git . The @provision.sh@ script and its supporting files can be found in the "arvados/tools/salt-install":https://git.arvados.org/arvados.git/tree/refs/heads/{{ branchname }}:/tools/salt-install directory in the Arvados git repository.
+This is a package-based installation method. Start with the @provision.sh@ script which is available by cloning the @{{ branchname }}@ branch from "https://git.arvados.org/arvados.git":https://git.arvados.org/arvados.git . The @provision.sh@ script and its supporting files can be found in the "arvados/tools/salt-install":https://git.arvados.org/arvados.git/tree/refs/heads/{{ branchname }}:/tools/salt-install directory in the Arvados git repository.
This procedure will install all the main Arvados components to get you up and running in a multi-host environment.
h3(#create_a_compute_image). Create a compute image
-In a multi-host installation, containers are dispatched in docker daemons running in the <i>compute instances</i>, which need some special setup. We provide a "compute image builder script":https://github.com/arvados/arvados/tree/master/tools/compute-images that you can use to build a template image following "these instructions":https://doc.arvados.org/main/install/crunch2-cloud/install-compute-node.html . Once you have that image created, you can use the image reference in the Arvados configuration in the next steps.
+In a multi-host installation, containers are dispatched in docker daemons running in the <i>compute instances</i>, which need some special setup. We provide a "compute image builder script":https://github.com/arvados/arvados/tree/master/tools/compute-images that you can use to build a template image following "these instructions":https://doc.arvados.org/main/install/crunch2-cloud/install-compute-node.html . Once you have that image created, you can use the image ID in the Arvados configuration in the next steps.
h2(#choose_configuration). Choose the desired configuration
Edit the variables in the <i>local.params</i> file. Pay attention to the <b>*_INT_IP, *_TOKEN</b> and <b>*KEY</b> variables. Those variables will be used to do a search and replace on the <i>pillars/*</i> in place of any matching __VARIABLE__.
-The <i>multi_host</i> include LetsEncrypt salt code to automatically request and install the certificates for the public-facing hosts (API, Workbench) so it will need the hostnames to be reachable from the Internet. If this cluster will not be the case, please set the variable <i>USE_LETSENCRYPT=no</i>.
+The <i>multi_host</i> include LetsEncrypt salt code to automatically request and install the certificates for the public-facing hosts (API/controller, Workbench, Keepproxy/Keepweb) using AWS' Route53. If you will provide custom certificates, please set the variable <i>USE_LETSENCRYPT=no</i>.
h3(#further_customization). Further customization of the installation (modifying the salt pillars and states)
---
layout: default
navsection: installguide
-title: Salt prerequisites
+title: Planning and prerequisites
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
{% endcomment %}
# "Introduction":#introduction
-# "Install Saltstack":#saltstack
-# "Choose an Arvados installation configuration":#installconfiguration
+# "Provisioning Arvados with Saltstack":#provisioning_arvados
+# "The provisioning tool files and directories":#provisioning_tool_files and directories
+# "Choose an Arvados installation configuration":#choose_configuration
+## "Further customization of the installation (modifying the salt pillars and states)":#further_customization
+# "Dump the configuration files created with the provision script":#dump_provision_config
+# "Add the Arvados formula to your Saltstack infrastructure":#add_formula_to_saltstack
h2(#introduction). Introduction
-To ease the installation of the various Arvados components, we have developed a "Saltstack":https://www.saltstack.com/ 's "arvados-formula":https://github.com/arvados/arvados-formula.git which can help you get an Arvados cluster up and running.
+To ease the installation of the various Arvados components, we have developed a "Saltstack":https://www.saltstack.com/ 's "arvados-formula":https://git.arvados.org/arvados-formula.git which can help you get an Arvados cluster up and running.
-Saltstack is a Python-based, open-source software for event-driven IT automation, remote task execution, and configuration management. It can be used in a master/minion setup or master-less.
+Saltstack is a Python-based, open-source software for event-driven IT automation, remote task execution, and configuration management. It can be used in a _master/minion_ setup (where a master node orchestrates and coordinates the configuration of nodes in an infrastructure) or <i>master-less</i>, where Saltstack is run locally in a node, with no communication with a master node.
-This is a package-based installation method. The Salt scripts to install and configure Arvados using this formula are available at the "tools/salt-install":https://github.com/arvados/arvados/tree/master/tools/salt-install directory in the Arvados git repository.
+Similar to other configuration management tools like Puppet, Ansible or Chef, Saltstack uses files named <i>states</i> to describe the tasks that will be performed on a node to take it to a desired state, and <i>pillars</i> to configure variables passed to the states, adding flexibility to the tool.
-h2(#saltstack). Install Saltstack
+You don't need to be running a Saltstack infrastructure to install Arvados: we wrote a provisioning script that will take care of setting up Saltstack in the node/s where you want to install Arvados and run a <i>master-less installer</i>. Once Arvados is installed, you can either uninstall Saltstack and its files or you can keep them, to modify/maintain your Arvados installation in the future.
-If you already have a Saltstack environment or you plan to use the @provision.sh@ script we provide, you can skip this section.
+This is a package-based installation method.
-The simplest way to get Salt up and running on a node is to use the bootstrap script they provide:
+
+h2(#provisioning_arvados). Provisioning Arvados with Saltstack
+
+The "tools/salt-install":https://git.arvados.org/arvados.git/tree/{{ branchname }}:/tools/salt-install directory in the Arvados git repository contains a script that you can run in the node/s where you want to install Arvados' components (the @provision.sh@ script) and a few configuration examples for different setups, that you can use to customize your installation.
+
+The @provision.sh@ script will help you deploy Arvados by preparing your environment to be able to run the installer, then running it. The actual installer is located at "arvados-formula":https://git.arvados.org/arvados-formula.git/tree/refs/heads/{{ branchname }} and will be cloned during the running of the @provision.sh@ script. The installer is built using "Saltstack":https://saltproject.io/ and @provision.sh@ performs the install using master-less mode.
+
+After setting up a few variables in a config file and copying a directory from the examples (see below), you'll be ready to run it and get Arvados deployed.
+
+
+
+h2(#provisioning_tool_files and directories). The provisioning tool files and directories
+
+The "tools/salt-install":https://git.arvados.org/arvados.git/tree/{{ branchname }}:/tools/salt-install directory contains the following elements:
+
+* The @provision.sh@ script itself. You don't need to modify it.
+* A few @local.params.*@ example files. You will need to copy one of these files to a file named @local.params@, which is the main configuration file for the @provision.sh@ script.
+* A few @config_examples/*@ directories, with pillars and states templates. You need to copy one of these to a @local_config_dir@ directory, which will be used by the @provision.sh@ script to setup your nodes.
+* A @tests@ directory, with a simple workflow and arvados CLI commands you can run to tests your cluster is capable of running a CWL workflow, upload files and create a user.
+
+Once you decide on an Arvados architecture you want to apply, you need to copy one of the example configuration files and directory, and edit them to suit your needs.
+
+Ie., for a multi-hosts / multi-hostnames in AWS, you need to do this:
<notextile>
-<pre><code>curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
-sudo sh /tmp/bootstrap_salt.sh -XUdfP -x python3
+<pre><code>cp local.params.example.multiple_hosts local.params
+cp -r config_examples/multi_host/aws local_config_dir
</code></pre>
</notextile>
-For more information check "Saltstack's documentation":https://docs.saltstack.com/en/latest/topics/installation/index.html
+These local files will be preserved if you upgrade the repository.
+
+
+
+h2(#choose_configuration). Choose an Arvados installation configuration
+
+The configuration examples provided with this installer are suitable to install Arvados with the following distribution of hosts/roles:
+
+* All roles on a single host, which can be done in two fashions:
+** Using a single hostname, assigning <i>a different port (other than 443) for each user-facing service</i>: This choice is easier to setup, but the user will need to know the port/s for the different services she wants to connect to. See "Single host install using the provision.sh script":salt-single-host.html for more details.
+** Using multiple hostnames on the same IP: this setup involves a few extra steps but each service will have a meaningful hostname so it will make easier to access them later. See "Single host install using the provision.sh script":salt-single-host.html for more details.
+* Roles distributed over multiple AWS instances, using multiple hostnames. This example can be adapted to use on-prem too. See "Multiple hosts installation":salt-multi-host.html for more details.
+
+Once you decide which of these choices you prefer, copy one of the example configuration files and directory, and edit them to suit your needs.
+
+Ie, if you decide to install Arvados on a single host using multiple hostnames:
+<notextile>
+<pre><code>cp local.params.example.single_host_multiple_hostnames local.params
+cp -r config_examples/single_host/multiple_hostnames local_config_dir
+</code></pre>
+</notextile>
+
+Edit the variables in the <i>local.params</i> file.
+
+
+
+h3(#further_customization). Further customization of the installation (modifying the salt pillars and states)
+
+If you want or need further customization, you can edit the Saltstack pillars and states files. Pay particular attention to the <i>pillars/arvados.sls</i> one. Any extra <i>state</i> file you add under <i>local_config_dir/states</i> will be added to the salt run and applied to the host.
+
+
+
+h2(#dump_provision_config). Dump the configuration files created with the provision script
+
+As mentioned above, the @provision.sh@ script helps you create a set of configuration files to be used by the Saltstack @arvados-formula@ and other helper formulas.
+
+Is it possible you want to inspect these files before deploying them or use them within your existing Saltstack environment. In order to get a rendered version of these files, the @provision.sh@ script has a option, @--dump-config@, which takes a directory as mandatory parameter. When this option it used, the script will create the specified directory and write the pillars, states and tests files so you can inspect them.
+
+Ie.
+<notextile>
+<pre><code>./provision.sh --dump-config ./config_dump --role workbench
+</code></pre>
+</notextile>
+
+will dump the configuration files used to install a workbench node under the @config_dump@ directory.
+
+These files are also suitable to be used in your existing Saltstack environment (see below).
+
+
-h2(#installconfiguration). Choose an Arvados installation configuration
+h2.(#add_formula_to_saltstack). Add the Arvados formula to your Saltstack infrastructure
-The salt formula can be used in a few different ways. Choose one of these three options to install Arvados:
+If you already have a Saltstack environment you can add the arvados-formula to your Saltstack master and apply the corresponding states and pillars to the nodes on your infrastructure that will be used to run Arvados.
-* "Arvados on a single host":salt-single-host.html
-* "Arvados across multiple hosts":salt-multi-host.html
-* "Use Vagrant to install Arvados in a virtual machine":salt-vagrant.html
+The @--dump-config@ option described above writes a @pillars/top.sls@ and @salt/top.sls@ files that you can use as a guide to configure your infrastructure.
h3. arv-put
-You may specify the desired storage class for a collection uploaded using @arv-put@:
+You may specify one or more desired storage classes for a collection uploaded using @arv-put@:
<pre>
-$ arv-put --storage-classes=hot myfile.txt
+$ arv-put --storage-classes=hot,archival myfile.txt
+</pre>
+
+h3. arv-mount
+
+You can ask @arv-mount@ to use specific storage classes when creating new collections:
+
+<pre>
+$ arv-mount --storage-classes=transient --mount-tmp=scratch keep
</pre>
h3. arvados-cwl-runner
Collection blocks will be in the "default" storage class if not otherwise specified.
-Currently, a collection may only have one desired storage class.
-
Any user with write access to a collection may set any storage class on that collection.
Names of storage classes are internal to the cluster and decided by the administrator. Aside from "default", Arvados currently does not define any standard storage class names.
"git.arvados.org/arvados.git/lib/cloud"
"git.arvados.org/arvados.git/sdk/go/arvados"
"github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
"github.com/aws/aws-sdk-go/aws/ec2metadata"
return err.earliestRetry
}
+var isCodeCapacity = map[string]bool{
+ "InsufficientInstanceCapacity": true,
+ "VcpuLimitExceeded": true,
+ "MaxSpotInstanceCountExceeded": true,
+}
+
+// isErrorCapacity returns whether the error is to be throttled based on its code.
+// Returns false if error is nil.
+func isErrorCapacity(err error) bool {
+ if aerr, ok := err.(awserr.Error); ok && aerr != nil {
+ if _, ok := isCodeCapacity[aerr.Code()]; ok {
+ return true
+ }
+ }
+ return false
+}
+
+type ec2QuotaError struct {
+ error
+}
+
+func (er *ec2QuotaError) IsQuotaError() bool {
+ return true
+}
+
func wrapError(err error, throttleValue *atomic.Value) error {
if request.IsErrorThrottle(err) {
// Back off exponentially until an upstream call
}
throttleValue.Store(d)
return rateLimitError{error: err, earliestRetry: time.Now().Add(d)}
+ } else if isErrorCapacity(err) {
+ return &ec2QuotaError{err}
} else if err != nil {
throttleValue.Store(time.Duration(0))
return err
import (
"encoding/json"
"flag"
+ "sync/atomic"
"testing"
"git.arvados.org/arvados.git/lib/cloud"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/config"
"github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/sirupsen/logrus"
check "gopkg.in/check.v1"
}
}
-var TestRateLimitErrorInterface cloud.RateLimitError = rateLimitError{}
+func (*EC2InstanceSetSuite) TestWrapError(c *check.C) {
+ retryError := awserr.New("Throttling", "", nil)
+ wrapped := wrapError(retryError, &atomic.Value{})
+ _, ok := wrapped.(cloud.RateLimitError)
+ c.Check(ok, check.Equals, true)
+
+ quotaError := awserr.New("InsufficientInstanceCapacity", "", nil)
+ wrapped = wrapError(quotaError, nil)
+ _, ok = wrapped.(cloud.QuotaError)
+ c.Check(ok, check.Equals, true)
+}
DispatchCloud:
InternalURLs: {SAMPLE: {}}
ExternalURL: "-"
- SSO:
- InternalURLs: {SAMPLE: {}}
- ExternalURL: ""
Keepproxy:
InternalURLs: {SAMPLE: {}}
ExternalURL: ""
WebDAVLogEvents: true
Login:
- # One of the following mechanisms (SSO, Google, PAM, LDAP, or
+ # One of the following mechanisms (Google, PAM, LDAP, or
# LoginCluster) should be enabled; see
# https://doc.arvados.org/install/setup-login.html
# originally supplied by the user will be used.
UsernameAttribute: uid
- SSO:
- # Authenticate with a separate SSO server. (Deprecated)
- Enable: false
-
- # ProviderAppID and ProviderAppSecret are generated during SSO
- # setup; see
- # https://doc.arvados.org/v2.0/install/install-sso.html#update-config
- ProviderAppID: ""
- ProviderAppSecret: ""
-
Test:
# Authenticate users listed here in the config file. This
# feature is intended to be used in test environments, and
*dst = *n
}
- // Provider* moved to SSO.Provider*
- if dst, n := &cluster.Login.SSO.ProviderAppID, dcluster.Login.ProviderAppID; n != nil && *n != *dst {
- *dst = *n
- if *n != "" {
- // In old config, non-empty ID meant enable
- cluster.Login.SSO.Enable = true
- }
- }
- if dst, n := &cluster.Login.SSO.ProviderAppSecret, dcluster.Login.ProviderAppSecret; n != nil && *n != *dst {
- *dst = *n
- }
-
cfg.Clusters[id] = cluster
}
return nil
"Login.PAM.Enable": true,
"Login.PAM.Service": false,
"Login.RemoteTokenRefresh": true,
- "Login.SSO": true,
- "Login.SSO.Enable": true,
- "Login.SSO.ProviderAppID": false,
- "Login.SSO.ProviderAppSecret": false,
"Login.Test": true,
"Login.Test.Enable": true,
"Login.Test.Users": false,
DispatchCloud:
InternalURLs: {SAMPLE: {}}
ExternalURL: "-"
- SSO:
- InternalURLs: {SAMPLE: {}}
- ExternalURL: ""
Keepproxy:
InternalURLs: {SAMPLE: {}}
ExternalURL: ""
WebDAVLogEvents: true
Login:
- # One of the following mechanisms (SSO, Google, PAM, LDAP, or
+ # One of the following mechanisms (Google, PAM, LDAP, or
# LoginCluster) should be enabled; see
# https://doc.arvados.org/install/setup-login.html
# originally supplied by the user will be used.
UsernameAttribute: uid
- SSO:
- # Authenticate with a separate SSO server. (Deprecated)
- Enable: false
-
- # ProviderAppID and ProviderAppSecret are generated during SSO
- # setup; see
- # https://doc.arvados.org/v2.0/install/install-sso.html#update-config
- ProviderAppID: ""
- ProviderAppSecret: ""
-
Test:
# Authenticate users listed here in the config file. This
# feature is intended to be used in test environments, and
c.Check(jresp["errors"], check.FitsTypeOf, []interface{}{})
}
-func (s *HandlerSuite) TestProxyRedirect(c *check.C) {
- s.cluster.Login.SSO.Enable = true
- s.cluster.Login.SSO.ProviderAppID = "test"
- s.cluster.Login.SSO.ProviderAppSecret = "test"
- req := httptest.NewRequest("GET", "https://0.0.0.0:1/login?return_to=foo", nil)
- resp := httptest.NewRecorder()
- s.handler.ServeHTTP(resp, req)
- if !c.Check(resp.Code, check.Equals, http.StatusFound) {
- c.Log(resp.Body.String())
- }
- // Old "proxy entire request" code path returns an absolute
- // URL. New lib/controller/federation code path returns a
- // relative URL.
- c.Check(resp.Header().Get("Location"), check.Matches, `(https://0.0.0.0:1)?/auth/joshid\?return_to=%2Cfoo&?`)
-}
-
-func (s *HandlerSuite) TestLogoutSSO(c *check.C) {
- s.cluster.Login.SSO.Enable = true
- s.cluster.Login.SSO.ProviderAppID = "test"
- req := httptest.NewRequest("GET", "https://0.0.0.0:1/logout?return_to=https://example.com/foo", nil)
- resp := httptest.NewRecorder()
- s.handler.ServeHTTP(resp, req)
- if !c.Check(resp.Code, check.Equals, http.StatusFound) {
- c.Log(resp.Body.String())
- }
- c.Check(resp.Header().Get("Location"), check.Equals, "http://localhost:3002/users/sign_out?"+url.Values{"redirect_uri": {"https://example.com/foo"}}.Encode())
-}
-
func (s *HandlerSuite) TestLogoutGoogle(c *check.C) {
s.cluster.Login.Google.Enable = true
s.cluster.Login.Google.ClientID = "test"
func chooseLoginController(cluster *arvados.Cluster, parent *Conn) loginController {
wantGoogle := cluster.Login.Google.Enable
wantOpenIDConnect := cluster.Login.OpenIDConnect.Enable
- wantSSO := cluster.Login.SSO.Enable
wantPAM := cluster.Login.PAM.Enable
wantLDAP := cluster.Login.LDAP.Enable
wantTest := cluster.Login.Test.Enable
wantLoginCluster := cluster.Login.LoginCluster != "" && cluster.Login.LoginCluster != cluster.ClusterID
switch {
- case 1 != countTrue(wantGoogle, wantOpenIDConnect, wantSSO, wantPAM, wantLDAP, wantTest, wantLoginCluster):
+ case 1 != countTrue(wantGoogle, wantOpenIDConnect, wantPAM, wantLDAP, wantTest, wantLoginCluster):
return errorLoginController{
- error: errors.New("configuration problem: exactly one of Login.Google, Login.OpenIDConnect, Login.SSO, Login.PAM, Login.LDAP, Login.Test, or Login.LoginCluster must be set"),
+ error: errors.New("configuration problem: exactly one of Login.Google, Login.OpenIDConnect, Login.PAM, Login.LDAP, Login.Test, or Login.LoginCluster must be set"),
}
case wantGoogle:
return &oidcLoginController{
AcceptAccessToken: cluster.Login.OpenIDConnect.AcceptAccessToken,
AcceptAccessTokenScope: cluster.Login.OpenIDConnect.AcceptAccessTokenScope,
}
- case wantSSO:
- return &ssoLoginController{Parent: parent}
case wantPAM:
return &pamLoginController{Cluster: cluster, Parent: parent}
case wantLDAP:
return n
}
-// Login and Logout are passed through to the parent's railsProxy;
-// UserAuthenticate is rejected.
-type ssoLoginController struct{ Parent *Conn }
-
-func (ctrl *ssoLoginController) Login(ctx context.Context, opts arvados.LoginOptions) (arvados.LoginResponse, error) {
- return ctrl.Parent.railsProxy.Login(ctx, opts)
-}
-func (ctrl *ssoLoginController) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) {
- return ctrl.Parent.railsProxy.Logout(ctx, opts)
-}
-func (ctrl *ssoLoginController) UserAuthenticate(ctx context.Context, opts arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) {
- return arvados.APIClientAuthorization{}, httpserver.ErrorWithStatus(errors.New("username/password authentication is not available"), http.StatusBadRequest)
-}
-
type errorLoginController struct{ error }
func (ctrl errorLoginController) Login(context.Context, arvados.LoginOptions) (arvados.LoginResponse, error) {
c.Assert(err, check.IsNil)
s.cluster, err = cfg.GetCluster("")
c.Assert(err, check.IsNil)
- s.cluster.Login.SSO.Enable = false
+ s.cluster.Login.Test.Enable = false
s.cluster.Login.Google.Enable = true
s.cluster.Login.Google.ClientID = "test%client$id"
s.cluster.Login.Google.ClientSecret = "test#client/secret"
opts := arvados.LoginOptions{
ReturnTo: "https://foo.example.com/bar",
}
- resp, err := s.conn.Login(s.ctx, opts)
- c.Check(err, check.IsNil)
- c.Check(resp.RedirectLocation, check.Equals, "/auth/joshid?return_to="+url.QueryEscape(","+opts.ReturnTo))
+ _, err := s.conn.Login(s.ctx, opts)
+ c.Check(err.(*arvados.TransactionError).StatusCode, check.Equals, 404)
}
func (s *RPCSuite) TestLogout(c *check.C) {
}
resp, err := s.conn.Logout(s.ctx, opts)
c.Check(err, check.IsNil)
- c.Check(resp.RedirectLocation, check.Equals, "http://localhost:3002/users/sign_out?redirect_uri="+url.QueryEscape(opts.ReturnTo))
+ c.Check(resp.RedirectLocation, check.Equals, opts.ReturnTo)
}
func (s *RPCSuite) TestCollectionCreate(c *check.C) {
runtimeContext.submit_runner_cluster not in arvrunner.api._rootDesc["remoteHosts"] and
runtimeContext.submit_runner_cluster != arvrunner.api._rootDesc["uuidPrefix"]):
raise WorkflowException("Unknown or invalid cluster id '%s' known remote clusters are %s" % (runtimeContext.submit_runner_cluster,
- ", ".join(list(arvrunner.api._rootDesc["remoteHosts"].keys()))))
+ ", ".join(list(arvrunner.api._rootDesc["remoteHosts"].keys()))))
+ if runtimeContext.project_uuid:
+ cluster_target = runtimeContext.submit_runner_cluster or arvrunner.api._rootDesc["uuidPrefix"]
+ if not runtimeContext.project_uuid.startswith(cluster_target):
+ raise WorkflowException("Project uuid '%s' must be for target cluster '%s'" % (runtimeContext.project_uuid, cluster_target))
+ try:
+ arvrunner.api.groups().get(uuid=runtimeContext.project_uuid).execute()
+ except Exception as e:
+ raise WorkflowException("Invalid project uuid '%s': %s" % (runtimeContext.project_uuid, e))
+
def set_cluster_target(tool, arvrunner, builder, runtimeContext):
cluster_target_req = None
for field in ("hints", "requirements"):
def glob(self, pattern):
collection, rest = self.get_collection(pattern)
- if collection is not None and not rest:
+ if collection is not None and rest in (None, "", "."):
return [pattern]
patternsegments = rest.split("/")
return sorted(self._match(collection, patternsegments, "keep:" + collection.manifest_locator()))
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.2
+class: CommandLineTool
+inputs: []
+outputs:
+ stuff:
+ type: Directory
+ outputBinding:
+ glob: $(runtime.outdir)
+requirements:
+ ShellCommandRequirement: {}
+arguments: [{shellQuote: false, valueFrom: "mkdir -p foo && touch baz.txt && touch foo/bar.txt"}]
}
tool: 10380-trailing-slash-dir.cwl
doc: "Test issue 10380 - bug with trailing slash when capturing an output directory"
+
+- job: null
+ output: {
+ "stuff": {
+ "basename": "78f3957c41d044352303a3fa326dff1e+102",
+ "class": "Directory",
+ "listing": [
+ {
+ "basename": "baz.txt",
+ "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
+ "class": "File",
+ "location": "78f3957c41d044352303a3fa326dff1e+102/baz.txt",
+ "size": 0
+ },
+ {
+ "basename": "foo",
+ "class": "Directory",
+ "listing": [
+ {
+ "basename": "bar.txt",
+ "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
+ "class": "File",
+ "location": "78f3957c41d044352303a3fa326dff1e+102/foo/bar.txt",
+ "size": 0
+ }
+ ],
+ "location": "78f3957c41d044352303a3fa326dff1e+102/foo"
+ }
+ ],
+ "location": "78f3957c41d044352303a3fa326dff1e+102"
+ }
+ }
+ tool: 17801-runtime-outdir.cwl
+ doc: "Test issue 17801 - bug using $(runtime.outdir) to capture the output directory"
stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
self.assertEqual(exited, 1)
+ @stubs
+ def test_submit_validate_project_uuid(self, stubs):
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 1)
+
+ stubs.api.groups().get().execute.side_effect = Exception("Bad project")
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 1)
+
@mock.patch("arvados.collection.CollectionReader")
@stubs
def test_submit_uuid_inputs(self, stubs, collectionReader):
Service string
DefaultEmailDomain string
}
- SSO struct {
- Enable bool
- ProviderAppID string
- ProviderAppSecret string
- }
Test struct {
Enable bool
Users map[string]TestUser
Keepproxy Service
Keepstore Service
RailsAPI Service
- SSO Service
WebDAVDownload Service
WebDAV Service
WebShell Service
output = None
try:
writer.start(save_collection=not(args.stream or args.raw))
- except arvados.errors.ApiError as error:
+ except (arvados.errors.ApiError, arvados.errors.KeepWriteError) as error:
logger.error("\n".join([
"arv-put: %s" % str(error)]))
sys.exit(1)
"http://%s:%s"%(localhost, keep_web_dl_port): {},
},
},
- "SSO": {
- "ExternalURL": "http://localhost:3002",
- },
}
config = {
"RequestTimeout": "30s",
},
"Login": {
- "SSO": {
+ "Test": {
"Enable": True,
- "ProviderAppID": "arvados-server",
- "ProviderAppSecret": "608dbf356a327e2d0d4932b60161e212c2d8d8f5e25690d7b622f850a990cd33",
+ "Users": {
+ "alice": {
+ "Email": "alice@example.com",
+ "Password": "xyzzy"
+ }
+ }
},
},
"SystemLogs": {
gem 'listen'
end
-# Fast app boot times
-gem 'bootsnap', require: false
-
gem 'pg', '~> 1.0'
gem 'multi_json'
# Locking to 5.10.3 to workaround issue in 5.11.1 (https://github.com/seattlerb/minitest/issues/730)
gem 'minitest', '5.10.3'
-# Restricted because omniauth >= 1.5.0 requires Ruby >= 2.1.9:
-gem 'omniauth', '~> 1.4.0'
-gem 'omniauth-oauth2', '~> 1.1'
-
gem 'andand'
gem 'optimist'
addressable (>= 2.3.1)
extlib (>= 0.9.15)
multi_json (>= 1.0.0)
- bootsnap (1.4.7)
- msgpack (~> 1.0)
builder (3.2.4)
byebug (11.0.1)
capistrano (2.15.9)
multi_json (~> 1.11)
os (>= 0.9, < 2.0)
signet (~> 0.7)
- hashie (3.6.0)
highline (2.0.1)
httpclient (2.8.3)
i18n (0.9.5)
minitest (5.10.3)
mocha (1.8.0)
metaclass (~> 0.0.1)
- msgpack (1.3.3)
multi_json (1.15.0)
- multi_xml (0.6.0)
multipart-post (2.1.1)
net-scp (2.0.0)
net-ssh (>= 2.6.5, < 6.0.0)
nokogiri (1.11.7)
mini_portile2 (~> 2.5.0)
racc (~> 1.4)
- oauth2 (1.4.1)
- faraday (>= 0.8, < 0.16.0)
- jwt (>= 1.0, < 3.0)
- multi_json (~> 1.3)
- multi_xml (~> 0.5)
- rack (>= 1.2, < 3)
oj (3.9.2)
- omniauth (1.4.3)
- hashie (>= 1.2, < 4)
- rack (>= 1.6.2, < 3)
- omniauth-oauth2 (1.5.0)
- oauth2 (~> 1.1)
- omniauth (~> 1.2)
optimist (3.0.0)
os (1.1.1)
passenger (6.0.2)
acts_as_api
andand
arvados (~> 2.1.5)
- bootsnap
byebug
factory_bot_rails
httpclient
mocha
multi_json
oj
- omniauth (~> 1.4.0)
- omniauth-oauth2 (~> 1.1)
optimist
passenger
pg (~> 1.0)
uglifier (~> 2.0)
BUNDLED WITH
- 1.17.3
+ 2.2.19
border-bottom: 1px solid #fff;
font-size: 0.8em;
}
-img.curoverse-logo {
+img.arvados-logo {
height: 66px;
}
#intropage {
if not current_user
respond_to do |format|
format.json { send_error("Not logged in", status: 401) }
- format.html { redirect_to '/auth/joshid' }
+ format.html { redirect_to '/login' }
end
false
end
respond_to :html
- # omniauth callback method
+ # create a new session
def create
if !Rails.configuration.Login.LoginCluster.empty? and Rails.configuration.Login.LoginCluster != Rails.configuration.ClusterID
raise "Local login disabled when LoginCluster is set"
authinfo = SafeJSON.load(params[:auth_info])
max_expires_at = authinfo["expires_at"]
else
- # omniauth middleware verified the user and is passing auth_info
- # in request.env.
- authinfo = request.env['omniauth.auth']['info'].with_indifferent_access
+ return send_error "Legacy code path no longer supported", status: 404
end
if !authinfo['user_uuid'].blank?
flash[:notice] = params[:message]
end
- # logout - Clear our rack session BUT essentially redirect to the provider
- # to clean up the Devise session from there too !
+ # logout - this gets intercepted by controller, so this is probably
+ # mostly dead code at this point.
def logout
session[:user_id] = nil
flash[:notice] = 'You have logged off'
return_to = params[:return_to] || root_url
- redirect_to "#{Rails.configuration.Services.SSO.ExternalURL}users/sign_out?redirect_uri=#{CGI.escape return_to}"
+ redirect_to return_to
end
- # login - Just bounce to /auth/joshid. The only purpose of this function is
- # to save the return_to parameter (if it exists; see the application
- # controller). /auth/joshid bypasses the application controller.
+ # login. Redirect to LoginCluster.
def login
if params[:remote] !~ /^[0-9a-z]{5}$/ && !params[:remote].nil?
return send_error 'Invalid remote cluster id', status: 400
p << "return_to=#{CGI.escape(params[:return_to])}" if params[:return_to]
redirect_to "#{login_cluster}/login?#{p.join('&')}"
else
- if params[:return_to]
- # Encode remote param inside callback's return_to, so that we'll get it on
- # create() after login.
- remote_param = params[:remote].nil? ? '' : params[:remote]
- p << "return_to=#{CGI.escape(remote_param + ',' + params[:return_to])}"
- end
- redirect_to "/auth/joshid?#{p.join('&')}"
+ return send_error "Legacy code path no longer supported", status: 404
end
end
<% end %>
•
<a class="logout" href="/logout">Log out</a>
- <% else %>
- <!--<a class="logout" href="/auth/joshid">Log in</a>-->
<% end %>
<% if current_user and session[:real_uid] and session[:switch_back_to] and User.find(session[:real_uid].to_i).verify_userswitch_cookie(session[:switch_back_to]) %>
<% if current_user or session['invite_code'] %>
<div id="footer">
- <div style="float:right">Questions → <a href="mailto:arvados@curoverse.com">arvados@curoverse.com</a></div>
<div style="clear:both"></div>
</div>
<% end %>
});
<% end %>
<div id="intropage">
- <img class="curoverse-logo" src="<%= asset_path('logo.png') %>" style="display:block; margin:2em auto"/>
+ <img class="arvados-logo" src="<%= asset_path('logo.png') %>" style="display:block; margin:2em auto"/>
<div style="width:30em; margin:2em auto 0 auto">
<h1>Welcome</h1>
- <h4>Curoverse ARVADOS</h4>
+ <h4>ARVADOS</h4>
<% if !current_user and session['invite_code'] %>
- <p>Curoverse Arvados lets you manage and process human genomes and exomes. You can start using the private beta
- now with your Google account.</p>
+ <p>Arvados lets you manage and process biomedical data.</p>
<p style="float:right;margin-top:1em">
- <button class="login" href="/auth/joshid">Log in and get started</button>
+ <button class="login" href="/login">Log in and get started</button>
</p>
<% else %>
- <p>Curoverse ARVADOS is transforming how researchers and
- clinical geneticists use whole genome sequences. </p>
- <p>If you’re interested in learning more, we’d love to hear
- from you —
- contact <a href="mailto:arvados@curoverse.com">arvados@curoverse.com</a>.</p>
-
<% if !current_user %>
<p style="float:right;margin-top:1em">
- <a href="/auth/joshid">Log in here.</a>
+ <a href="/login">Log in here.</a>
</p>
<% end %>
<div id="intropage">
- <img class="curoverse-logo" src="<%= asset_path('logo.png') rescue '/logo.png' %>" style="display:block; margin:2em auto"/>
+ <img class="arvados-logo" src="<%= asset_path('logo.png') rescue '/logo.png' %>" style="display:block; margin:2em auto"/>
<div style="width:30em; margin:2em auto 0 auto">
<h1>Error</h1>
<%= notice %>
<br/>
-<a href="/auth/joshid">Retry Login</a>
+<a href="/login">Retry Login</a>
# configured by application.yml (i.e., here!) instead.
end
-if (File.exist?(File.expand_path '../omniauth.rb', __FILE__) and
- not defined? WARNED_OMNIAUTH_CONFIG)
- Rails.logger.warn <<-EOS
-DEPRECATED CONFIGURATION:
- Please move your SSO provider config into config/application.yml
- and delete config/initializers/omniauth.rb.
-EOS
- # Real values will be copied from globals by omniauth_init.rb. For
- # now, assign some strings so the generic *.yml config loader
- # doesn't overwrite them or complain that they're missing.
- Rails.configuration.Login["SSO"]["ProviderAppID"] = 'xxx'
- Rails.configuration.Login["SSO"]["ProviderAppSecret"] = 'xxx'
- Rails.configuration.Services["SSO"]["ExternalURL"] = '//xxx'
- WARNED_OMNIAUTH_CONFIG = true
-end
-
# Load the defaults, used by config:migrate and fallback loading
# legacy application.yml
defaultYAML, stderr, status = Open3.capture3("arvados-server", "config-dump", "-config=-", "-skip-legacy", stdin_data: "Clusters: {xxxxx: {}}")
arvcfg.declare_config "Users.UserNotifierEmailFrom", String, :user_notifier_email_from
arvcfg.declare_config "Users.NewUserNotificationRecipients", Hash, :new_user_notification_recipients, ->(cfg, k, v) { arrayToHash cfg, "Users.NewUserNotificationRecipients", v }
arvcfg.declare_config "Users.NewInactiveUserNotificationRecipients", Hash, :new_inactive_user_notification_recipients, method(:arrayToHash)
-arvcfg.declare_config "Login.SSO.ProviderAppSecret", String, :sso_app_secret
-arvcfg.declare_config "Login.SSO.ProviderAppID", String, :sso_app_id
arvcfg.declare_config "Login.LoginCluster", String
arvcfg.declare_config "Login.TrustedClients", Hash
arvcfg.declare_config "Login.RemoteTokenRefresh", ActiveSupport::Duration
arvcfg.declare_config "Login.TokenLifetime", ActiveSupport::Duration
arvcfg.declare_config "TLS.Insecure", Boolean, :sso_insecure
-arvcfg.declare_config "Services.SSO.ExternalURL", String, :sso_provider_url
arvcfg.declare_config "AuditLogs.MaxAge", ActiveSupport::Duration, :max_audit_log_age
arvcfg.declare_config "AuditLogs.MaxDeleteBatch", Integer, :max_audit_log_delete_batch
arvcfg.declare_config "AuditLogs.UnloggedAttributes", Hash, :unlogged_attributes, ->(cfg, k, v) { arrayToHash cfg, "AuditLogs.UnloggedAttributes", v }
ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
require 'bundler/setup' # Set up gems listed in the Gemfile.
-require 'bootsnap/setup' # Speed up boot time by caching expensive operations.
\ No newline at end of file
# Load the rails application
require_relative 'application'
-require 'josh_id'
# Initialize the rails application
Rails.application.initialize!
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-# This file is called omniauth_init.rb instead of omniauth.rb because
-# older versions had site configuration in omniauth.rb.
-#
-# It must come after omniauth.rb in (lexical) load order.
-
-if defined? CUSTOM_PROVIDER_URL
- Rails.logger.warn "Copying omniauth from globals in legacy config file."
- Rails.configuration.Login["SSO"]["ProviderAppID"] = APP_ID
- Rails.configuration.Login["SSO"]["ProviderAppSecret"] = APP_SECRET
- Rails.configuration.Services["SSO"]["ExternalURL"] = CUSTOM_PROVIDER_URL.sub(/\/$/, "") + "/"
-else
- Rails.application.config.middleware.use OmniAuth::Builder do
- provider(:josh_id,
- Rails.configuration.Login["SSO"]["ProviderAppID"],
- Rails.configuration.Login["SSO"]["ProviderAppSecret"],
- Rails.configuration.Services["SSO"]["ExternalURL"])
- end
- OmniAuth.config.on_failure = StaticController.action(:login_failure)
-end
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-require 'omniauth-oauth2'
-module OmniAuth
- module Strategies
- class JoshId < OmniAuth::Strategies::OAuth2
-
- args [:client_id, :client_secret, :custom_provider_url]
-
- option :custom_provider_url, ''
-
- uid { raw_info['id'] }
-
- option :client_options, {}
-
- info do
- {
- :first_name => raw_info['info']['first_name'],
- :last_name => raw_info['info']['last_name'],
- :email => raw_info['info']['email'],
- :identity_url => raw_info['info']['identity_url'],
- :username => raw_info['info']['username'],
- }
- end
-
- extra do
- {
- 'raw_info' => raw_info
- }
- end
-
- def authorize_params
- options.authorize_params[:auth_provider] = request.params['auth_provider']
- super
- end
-
- def client
- options.client_options[:site] = options[:custom_provider_url]
- options.client_options[:authorize_url] = "#{options[:custom_provider_url]}/auth/josh_id/authorize"
- options.client_options[:access_token_url] = "#{options[:custom_provider_url]}/auth/josh_id/access_token"
- if Rails.configuration.TLS.Insecure
- options.client_options[:ssl] = {verify_mode: OpenSSL::SSL::VERIFY_NONE}
- end
- ::OAuth2::Client.new(options.client_id, options.client_secret, deep_symbolize(options.client_options))
- end
-
- def callback_url
- full_host + script_name + callback_path + "?return_to=" + CGI.escape(request.params['return_to'] || '')
- end
-
- def raw_info
- @raw_info ||= access_token.get("/auth/josh_id/user.json?oauth_token=#{access_token.token}").parsed
- end
- end
- end
-end
test "redirect to joshid" do
api_client_page = 'http://client.example.com/home'
get :login, params: {return_to: api_client_page}
- assert_response :redirect
- assert_equal("http://test.host/auth/joshid?return_to=%2Chttp%3A%2F%2Fclient.example.com%2Fhome", @response.redirect_url)
- assert_nil assigns(:api_client)
+ # Not supported any more
+ assert_response 404
end
test "send token when user is already logged in" do
Rails.configuration.Login.LoginCluster = 'zzzzz'
api_client_page = 'http://client.example.com/home'
get :login, params: {return_to: api_client_page}
- assert_response :redirect
- assert_equal("http://test.host/auth/joshid?return_to=%2Chttp%3A%2F%2Fclient.example.com%2Fhome", @response.redirect_url)
- assert_nil assigns(:api_client)
+ # Doesn't redirect, just fail.
+ assert_response 404
end
test "controller cannot create session without SystemRootToken" do
params: {specimen: {}},
headers: {'HTTP_ACCEPT' => 'text/html'})
assert_response 302
- assert_match(%r{/auth/joshid$}, @response.headers['Location'],
+ assert_match(%r{http://www.example.com/login$}, @response.headers['Location'],
"HTML login prompt did not include expected redirect")
end
end
def mock_auth_with(email: nil, username: nil, identity_url: nil, remote: nil, expected_response: :redirect)
mock = {
- 'provider' => 'josh_id',
- 'uid' => 'https://edward.example.com',
- 'info' => {
'identity_url' => 'https://edward.example.com',
'name' => 'Edward Example',
'first_name' => 'Edward',
'last_name' => 'Example',
- },
}
- mock['info']['email'] = email unless email.nil?
- mock['info']['username'] = username unless username.nil?
- mock['info']['identity_url'] = identity_url unless identity_url.nil?
- post('/auth/josh_id/callback',
- params: {return_to: client_url(remote: remote)},
- headers: {'omniauth.auth' => mock})
+ mock['email'] = email unless email.nil?
+ mock['username'] = username unless username.nil?
+ mock['identity_url'] = identity_url unless identity_url.nil?
+ post('/auth/controller/callback',
+ params: {return_to: client_url(remote: remote), :auth_info => SafeJSON.dump(mock)},
+ headers: {'Authorization' => 'Bearer ' + Rails.configuration.SystemRootToken})
errors = {
:redirect => 'Did not redirect to client with token',
SimpleCov.start do
add_filter '/test/'
add_filter 'initializers/secret_token'
- add_filter 'initializers/omniauth'
end
rescue Exception => e
$stderr.puts "SimpleCov unavailable (#{e}). Proceeding without."
from builtins import str
from builtins import object
import os
-import sys
import llfuse
import errno
import stat
import threading
import arvados
-import pprint
import arvados.events
-import re
-import apiclient
-import json
import logging
import time
-import _strptime
-import calendar
import threading
import itertools
-import ciso8601
import collections
import functools
import arvados.keep
self.add_argument('--read-only', action='store_false', help="Mount will be read only (default)", dest="enable_write", default=False)
self.add_argument('--read-write', action='store_true', help="Mount will be read-write", dest="enable_write", default=False)
+ self.add_argument('--storage-classes', type=str, metavar='CLASSES', help="Specify comma separated list of storage classes to be used when saving data of new collections", default=None)
self.add_argument('--crunchstat-interval', type=float, help="Write stats to stderr every N seconds (default disabled)", default=0)
dir_args = [llfuse.ROOT_INODE, self.operations.inodes, self.api, self.args.retries]
mount_readme = False
+ storage_classes = None
+ if self.args.storage_classes is not None:
+ storage_classes = self.args.storage_classes.replace(' ', '').split(',')
+ self.logger.info("Storage classes requested for new collections: {}".format(', '.join(storage_classes)))
+
if self.args.collection is not None:
# Set up the request handler with the collection at the root
# First check that the collection is readable
mount_readme = True
if dir_class is not None:
- ent = dir_class(*dir_args)
+ if dir_class in [TagsDirectory, CollectionDirectory]:
+ ent = dir_class(*dir_args)
+ else:
+ ent = dir_class(*dir_args, storage_classes=storage_classes)
self.operations.inodes.add_entry(ent)
self.listen_for_events = ent.want_event_subscribe()
return
dir_args[0] = e.inode
for name in self.args.mount_by_id:
- self._add_mount(e, name, MagicDirectory(*dir_args, pdh_only=False))
+ self._add_mount(e, name, MagicDirectory(*dir_args, pdh_only=False, storage_classes=storage_classes))
for name in self.args.mount_by_pdh:
self._add_mount(e, name, MagicDirectory(*dir_args, pdh_only=True))
for name in self.args.mount_by_tag:
self._add_mount(e, name, TagsDirectory(*dir_args))
for name in self.args.mount_home:
- self._add_mount(e, name, ProjectDirectory(*dir_args, project_object=usr, poll=True))
+ self._add_mount(e, name, ProjectDirectory(*dir_args, project_object=usr, poll=True, storage_classes=storage_classes))
for name in self.args.mount_shared:
- self._add_mount(e, name, SharedDirectory(*dir_args, exclude=usr, poll=True))
+ self._add_mount(e, name, SharedDirectory(*dir_args, exclude=usr, poll=True, storage_classes=storage_classes))
for name in self.args.mount_tmp:
- self._add_mount(e, name, TmpCollectionDirectory(*dir_args))
+ self._add_mount(e, name, TmpCollectionDirectory(*dir_args, storage_classes=storage_classes))
if mount_readme:
text = self._readme_text(
new_collection_record["portable_data_hash"] = new_collection_record["uuid"]
if 'manifest_text' not in new_collection_record:
new_collection_record['manifest_text'] = coll_reader.manifest_text()
+ if 'storage_classes_desired' not in new_collection_record:
+ new_collection_record['storage_classes_desired'] = coll_reader.storage_classes_desired()
if self.collection_record is None or self.collection_record["portable_data_hash"] != new_collection_record.get("portable_data_hash"):
self.new_collection(new_collection_record, coll_reader)
def save_new(self):
pass
- def __init__(self, parent_inode, inodes, api_client, num_retries):
+ def __init__(self, parent_inode, inodes, api_client, num_retries, storage_classes=None):
collection = self.UnsaveableCollection(
api_client=api_client,
keep_client=api_client.keep,
- num_retries=num_retries)
+ num_retries=num_retries,
+ storage_classes_desired=storage_classes)
super(TmpCollectionDirectory, self).__init__(
parent_inode, inodes, api_client.config, collection)
self.collection_record_file = None
"uuid": None,
"manifest_text": self.collection.manifest_text(),
"portable_data_hash": self.collection.portable_data_hash(),
+ "storage_classes_desired": self.collection.storage_classes_desired(),
}
def __contains__(self, k):
""".lstrip()
- def __init__(self, parent_inode, inodes, api, num_retries, pdh_only=False):
+ def __init__(self, parent_inode, inodes, api, num_retries, pdh_only=False, storage_classes=None):
super(MagicDirectory, self).__init__(parent_inode, inodes, api.config)
self.api = api
self.num_retries = num_retries
self.pdh_only = pdh_only
+ self.storage_classes = storage_classes
def __setattr__(self, name, value):
super(MagicDirectory, self).__setattr__(name, value)
if project[u'items_available'] == 0:
return False
e = self.inodes.add_entry(ProjectDirectory(
- self.inode, self.inodes, self.api, self.num_retries, project[u'items'][0]))
+ self.inode, self.inodes, self.api, self.num_retries,
+ project[u'items'][0], storage_classes=self.storage_classes))
else:
e = self.inodes.add_entry(CollectionDirectory(
self.inode, self.inodes, self.api, self.num_retries, k))
"""A special directory that contains the contents of a project."""
def __init__(self, parent_inode, inodes, api, num_retries, project_object,
- poll=True, poll_time=3):
+ poll=True, poll_time=3, storage_classes=None):
super(ProjectDirectory, self).__init__(parent_inode, inodes, api.config)
self.api = api
self.num_retries = num_retries
self._updating_lock = threading.Lock()
self._current_user = None
self._full_listing = False
+ self.storage_classes = storage_classes
def want_event_subscribe(self):
return True
if collection_uuid_pattern.match(i['uuid']):
return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i)
elif group_uuid_pattern.match(i['uuid']):
- return ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i, self._poll, self._poll_time)
+ return ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i, self._poll, self._poll_time, self.storage_classes)
elif link_uuid_pattern.match(i['uuid']):
if i['head_kind'] == 'arvados#collection' or portable_data_hash_pattern.match(i['head_uuid']):
return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i['head_uuid'])
def mkdir(self, name):
try:
with llfuse.lock_released:
- self.api.collections().create(body={"owner_uuid": self.project_uuid,
- "name": name,
- "manifest_text": ""}).execute(num_retries=self.num_retries)
+ c = {
+ "owner_uuid": self.project_uuid,
+ "name": name,
+ "manifest_text": "" }
+ if self.storage_classes is not None:
+ c["storage_classes_desired"] = self.storage_classes
+ try:
+ self.api.collections().create(body=c).execute(num_retries=self.num_retries)
+ except Exception as e:
+ raise
self.invalidate()
except apiclient_errors.Error as error:
_logger.error(error)
"""A special directory that represents users or groups who have shared projects with me."""
def __init__(self, parent_inode, inodes, api, num_retries, exclude,
- poll=False, poll_time=60):
+ poll=False, poll_time=60, storage_classes=None):
super(SharedDirectory, self).__init__(parent_inode, inodes, api.config)
self.api = api
self.num_retries = num_retries
self._poll = True
self._poll_time = poll_time
self._updating_lock = threading.Lock()
+ self.storage_classes = storage_classes
@use_counter
def update(self):
obr = objects[r]
if obr.get("name"):
contents[obr["name"]] = obr
- #elif obr.get("username"):
- # contents[obr["username"]] = obr
elif "first_name" in obr:
contents[u"{} {}".format(obr["first_name"], obr["last_name"])] = obr
self.merge(viewitems(contents),
lambda i: i[0],
lambda a, i: a.uuid() == i[1]['uuid'],
- lambda i: ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i[1], poll=self._poll, poll_time=self._poll_time))
+ lambda i: ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i[1], poll=self._poll, poll_time=self._poll_time, storage_classes=self.storage_classes))
except Exception:
_logger.exception("arv-mount shared dir error")
finally:
from .integration_test import IntegrationTest
from .mount_test_base import MountTestBase
+from .test_tmp_collection import storage_classes_desired
logger = logging.getLogger('arvados.arv-mount')
def _test_slash_substitution_conflict(self, tmpdir, fusename):
with open(os.path.join(tmpdir, fusename, 'waz'), 'w') as f:
f.write('foo')
+
+class StorageClassesTest(IntegrationTest):
+ mnt_args = [
+ '--read-write',
+ '--mount-home', 'homedir',
+ ]
+
+ def setUp(self):
+ super(StorageClassesTest, self).setUp()
+ self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
+
+ @IntegrationTest.mount(argv=mnt_args)
+ def test_collection_default_storage_classes(self):
+ coll_path = os.path.join(self.mnt, 'homedir', 'a_collection')
+ self.api.collections().create(body={'name':'a_collection'}).execute()
+ self.pool_test(coll_path)
+ @staticmethod
+ def _test_collection_default_storage_classes(self, coll):
+ self.assertEqual(storage_classes_desired(coll), ['default'])
+
+ @IntegrationTest.mount(argv=mnt_args+['--storage-classes', 'foo'])
+ def test_collection_custom_storage_classes(self):
+ coll_path = os.path.join(self.mnt, 'homedir', 'new_coll')
+ os.mkdir(coll_path)
+ self.pool_test(coll_path)
+ @staticmethod
+ def _test_collection_custom_storage_classes(self, coll):
+ self.assertEqual(storage_classes_desired(coll), ['foo'])
with open(os.path.join(tmpdir, '.arvados#collection')) as tmp:
return json.load(tmp)['manifest_text']
+def storage_classes_desired(tmpdir):
+ with open(os.path.join(tmpdir, '.arvados#collection')) as tmp:
+ return json.load(tmp)['storage_classes_desired']
class TmpCollectionTest(IntegrationTest):
mnt_args = [
'--mount-tmp', 'zzz',
]
+ @IntegrationTest.mount(argv=mnt_args+['--storage-classes', 'foo, bar'])
+ def test_storage_classes(self):
+ self.pool_test(os.path.join(self.mnt, 'zzz'))
+ @staticmethod
+ def _test_storage_classes(self, zzz):
+ self.assertEqual(storage_classes_desired(zzz), ['foo', 'bar'])
+
@IntegrationTest.mount(argv=mnt_args+['--mount-tmp', 'yyy'])
def test_two_tmp(self):
self.pool_test(os.path.join(self.mnt, 'zzz'),
kc.Arvados = &arvclient
// Check if the client specified the number of replicas
- if req.Header.Get("X-Keep-Desired-Replicas") != "" {
+ if desiredReplicas := req.Header.Get(keepclient.XKeepDesiredReplicas); desiredReplicas != "" {
var r int
- _, err := fmt.Sscanf(req.Header.Get(keepclient.XKeepDesiredReplicas), "%d", &r)
+ _, err := fmt.Sscanf(desiredReplicas, "%d", &r)
if err == nil {
kc.Want_replicas = r
}
switch err.(type) {
case nil:
status = http.StatusOK
+ if len(kc.StorageClasses) > 0 {
+ // A successful PUT request with storage classes means that all
+ // storage classes were fulfilled, so the client will get a
+ // confirmation via the X-Storage-Classes-Confirmed header.
+ hdr := ""
+ isFirst := true
+ for _, sc := range kc.StorageClasses {
+ if isFirst {
+ hdr = fmt.Sprintf("%s=%d", sc, wroteReplicas)
+ isFirst = false
+ } else {
+ hdr += fmt.Sprintf(", %s=%d", sc, wroteReplicas)
+ }
+ }
+ resp.Header().Set(keepclient.XKeepStorageClassesConfirmed, hdr)
+ }
_, err = io.WriteString(resp, locatorOut)
-
case keepclient.OversizeBlockError:
// Too much data
status = http.StatusRequestEntityTooLarge
-
case keepclient.InsufficientReplicasError:
- if wroteReplicas > 0 {
- // At least one write is considered success. The
- // client can decide if getting less than the number of
- // replications it asked for is a fatal error.
- status = http.StatusOK
- _, err = io.WriteString(resp, locatorOut)
- } else {
- status = http.StatusServiceUnavailable
- }
-
+ status = http.StatusServiceUnavailable
default:
status = http.StatusBadGateway
}
c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
}
+func (s *ServerRequiredSuite) TestStorageClassesConfirmedHeader(c *C) {
+ runProxy(c, false, false)
+ defer closeListener()
+
+ content := []byte("foo")
+ hash := fmt.Sprintf("%x", md5.Sum(content))
+ client := &http.Client{}
+
+ req, err := http.NewRequest("PUT",
+ fmt.Sprintf("http://%s/%s", listener.Addr().String(), hash),
+ bytes.NewReader(content))
+ c.Assert(err, IsNil)
+ req.Header.Set("X-Keep-Storage-Classes", "default")
+ req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
+ req.Header.Set("Content-Type", "application/octet-stream")
+
+ resp, err := client.Do(req)
+ c.Assert(err, IsNil)
+ c.Assert(resp.StatusCode, Equals, http.StatusOK)
+ c.Assert(resp.Header.Get("X-Keep-Storage-Classes-Confirmed"), Equals, "default=2")
+}
+
func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
kc, _ := runProxy(c, false, false, nil)
defer closeListener()
# Extra states to apply. If you use your own subdir, change this value accordingly
# EXTRA_STATES_DIR="${CONFIG_DIR}/states"
+# These are ARVADOS-related settings.
# Which release of Arvados repo you want to use
RELEASE="production"
# Which version of Arvados you want to install. Defaults to latest stable
# This is an arvados-formula setting.
# If branch is set, the script will switch to it before running salt
# Usually not needed, only used for testing
-# BRANCH="master"
+# BRANCH="main"
##########################################################
# Usually there's no need to modify things below this line
# Formulas versions
-# ARVADOS_TAG="v1.1.4"
+# ARVADOS_TAG="2.2.0"
# POSTGRES_TAG="v0.41.6"
# NGINX_TAG="temp-fix-missing-statements-in-pillar"
# DOCKER_TAG="v1.0.0"
# Extra states to apply. If you use your own subdir, change this value accordingly
# EXTRA_STATES_DIR="${CONFIG_DIR}/states"
+# These are ARVADOS-related settings.
# Which release of Arvados repo you want to use
RELEASE="production"
# Which version of Arvados you want to install. Defaults to latest stable
# This is an arvados-formula setting.
# If branch is set, the script will switch to it before running salt
# Usually not needed, only used for testing
-# BRANCH="master"
+# BRANCH="main"
##########################################################
# Usually there's no need to modify things below this line
# Formulas versions
-# ARVADOS_TAG="v1.1.4"
+# ARVADOS_TAG="2.2.0"
# POSTGRES_TAG="v0.41.6"
# NGINX_TAG="temp-fix-missing-statements-in-pillar"
# DOCKER_TAG="v1.0.0"
# Extra states to apply. If you use your own subdir, change this value accordingly
# EXTRA_STATES_DIR="${CONFIG_DIR}/states"
+# These are ARVADOS-related settings.
# Which release of Arvados repo you want to use
RELEASE="production"
# Which version of Arvados you want to install. Defaults to latest stable
# This is an arvados-formula setting.
# If branch is set, the script will switch to it before running salt
# Usually not needed, only used for testing
-# BRANCH="master"
+# BRANCH="main"
##########################################################
# Usually there's no need to modify things below this line
# Formulas versions
-# ARVADOS_TAG="v1.1.4"
+# ARVADOS_TAG="2.2.0"
# POSTGRES_TAG="v0.41.6"
# NGINX_TAG="temp-fix-missing-statements-in-pillar"
# DOCKER_TAG="v1.0.0"
-#!/bin/bash -x
+#!/usr/bin/env bash
# Copyright (C) The Arvados Authors. All rights reserved.
#
echo >&2
echo >&2 "${0} options:"
echo >&2 " -d, --debug Run salt installation in debug mode"
- echo >&2 " -p <N>, --ssl-port <N> SSL port to use for the web applications"
echo >&2 " -c <local.params>, --config <local.params> Path to the local.params config file"
echo >&2 " -t, --test Test installation running a CWL workflow"
echo >&2 " -r, --roles List of Arvados roles to apply to the host, comma separated"
echo >&2 " workbench2"
echo >&2 " Defaults to applying them all"
echo >&2 " -h, --help Display this help and exit"
+ echo >&2 " --dump-config <dest_dir> Dumps the pillars and states to a directory"
+ echo >&2 " This parameter does not perform any installation at all. It's"
+ echo >&2 " intended to give you a parsed sot of configuration files so"
+ echo >&2 " you can inspect them or use them in you Saltstack infrastructure."
+ echo >&2 " It"
+ echo >&2 " - parses the pillar and states templates,"
+ echo >&2 " - downloads the helper formulas with their desired versions,"
+ echo >&2 " - prepares the 'top.sls' files both for pillars and states"
+ echo >&2 " for the selected role/s"
+ echo >&2 " - writes the resulting files into <dest_dir>"
echo >&2 " -v, --vagrant Run in vagrant and use the /vagrant shared dir"
echo >&2
}
arguments() {
# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
+ if ! which getopt > /dev/null; then
+ echo >&2 "GNU getopt is required to run this script. Please install it and re-reun it"
+ exit 1
+ fi
+
TEMP=$(getopt -o c:dhp:r:tv \
- --long config:,debug,help,ssl-port:,roles:,test,vagrant \
+ --long config:,debug,dump-config:,help,roles:,test,vagrant \
-n "${0}" -- "${@}")
- if [ ${?} != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi
+ if [ ${?} != 0 ];
+ then echo "Please check the parameters you entered and re-run again"
+ exit 1
+ fi
# Note the quotes around `$TEMP': they are essential!
eval set -- "$TEMP"
-d | --debug)
LOG_LEVEL="debug"
shift
+ set -x
;;
- -p | --ssl-port)
- CONTROLLER_EXT_SSL_PORT=${2}
+ --dump-config)
+ if [[ ${2} = /* ]]; then
+ DUMP_SALT_CONFIG_DIR=${2}
+ else
+ DUMP_SALT_CONFIG_DIR=${PWD}/${2}
+ fi
+ ## states
+ S_DIR="${DUMP_SALT_CONFIG_DIR}/salt"
+ ## formulas
+ F_DIR="${DUMP_SALT_CONFIG_DIR}/formulas"
+ ## pillars
+ P_DIR="${DUMP_SALT_CONFIG_DIR}/pillars"
+ ## tests
+ T_DIR="${DUMP_SALT_CONFIG_DIR}/tests"
+ DUMP_CONFIG="yes"
shift 2
;;
-r | --roles)
CONFIG_FILE="${SCRIPT_DIR}/local.params"
CONFIG_DIR="local_config_dir"
+DUMP_CONFIG="no"
LOG_LEVEL="info"
CONTROLLER_EXT_SSL_PORT=443
TESTS_DIR="tests"
WORKBENCH1_EXT_SSL_PORT=443
WORKBENCH2_EXT_SSL_PORT=3001
+## These are ARVADOS-related parameters
# For a stable release, change RELEASE "production" and VERSION to the
# package version (including the iteration, e.g. X.Y.Z-1) of the
# release.
+# The "local.params.example.*" files already set "RELEASE=production"
+# to deploy production-ready packages
RELEASE="development"
VERSION="latest"
-# The arvados-formula version. For a stable release, this should be a
+# These are arvados-formula-related parameters
+# An arvados-formula tag. For a stable release, this should be a
# branch name (e.g. X.Y-dev) or tag for the release.
-ARVADOS_TAG="main"
+# ARVADOS_TAG="2.2.0"
+# BRANCH="main"
# Other formula versions we depend on
POSTGRES_TAG="v0.41.6"
LETSENCRYPT_TAG="v2.1.0"
# Salt's dir
+DUMP_SALT_CONFIG_DIR=""
## states
S_DIR="/srv/salt"
## formulas
F_DIR="/srv/formulas"
-##pillars
+## pillars
P_DIR="/srv/pillars"
+## tests
+T_DIR="/tmp/cluster_tests"
arguments ${@}
if [ -s ${CONFIG_FILE} ]; then
source ${CONFIG_FILE}
else
- echo >&2 "Please create a '${CONFIG_FILE}' file with initial values, as described in"
+ echo >&2 "You don't seem to have a config file with initial values."
+ echo >&2 "Please create a '${CONFIG_FILE}' file as described in"
echo >&2 " * https://doc.arvados.org/install/salt-single-host.html#single_host, or"
echo >&2 " * https://doc.arvados.org/install/salt-multi-host.html#multi_host_multi_hostnames"
exit 1
fi
if [ ! -d ${CONFIG_DIR} ]; then
- echo >&2 "Please create a '${CONFIG_DIR}' with initial values, as described in"
+ echo >&2 "You don't seem to have a config directory with pillars and states."
+ echo >&2 "Please create a '${CONFIG_DIR}' directory (as configured in your '${CONFIG_FILE}'). Please see"
echo >&2 " * https://doc.arvados.org/install/salt-single-host.html#single_host, or"
echo >&2 " * https://doc.arvados.org/install/salt-multi-host.html#multi_host_multi_hostnames"
exit 1
exit 1
fi
-if ! grep -E '^[[:alnum:]]{5}$' <<<${CLUSTER} ; then
+if ! grep -qE '^[[:alnum:]]{5}$' <<<${CLUSTER} ; then
echo >&2 "ERROR: <CLUSTER> must be exactly 5 alphanumeric characters long"
echo >&2 "Fix the cluster name in the 'local.params' file and re-run the provision script"
exit 1
HOSTNAME_EXT="${CLUSTER}.${DOMAIN}"
fi
-apt-get update
-apt-get install -y curl git jq
-
-if which salt-call; then
- echo "Salt already installed"
+if [ "${DUMP_CONFIG}" = "yes" ]; then
+ echo "The provision installer will just dump a config under ${DUMP_SALT_CONFIG_DIR} and exit"
else
- curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
- sh /tmp/bootstrap_salt.sh -XdfP -x python3
- /bin/systemctl stop salt-minion.service
- /bin/systemctl disable salt-minion.service
-fi
+ apt-get update
+ apt-get install -y curl git jq
+
+ if which salt-call; then
+ echo "Salt already installed"
+ else
+ curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
+ sh /tmp/bootstrap_salt.sh -XdfP -x python3
+ /bin/systemctl stop salt-minion.service
+ /bin/systemctl disable salt-minion.service
+ fi
-# Set salt to masterless mode
-cat > /etc/salt/minion << EOFSM
+ # Set salt to masterless mode
+ cat > /etc/salt/minion << EOFSM
file_client: local
file_roots:
base:
base:
- ${P_DIR}
EOFSM
+fi
-mkdir -p ${S_DIR} ${F_DIR} ${P_DIR}
+mkdir -p ${S_DIR} ${F_DIR} ${P_DIR} ${T_DIR}
# Get the formula and dependencies
cd ${F_DIR} || exit 1
-git clone --branch "${ARVADOS_TAG}" https://git.arvados.org/arvados-formula.git
-git clone --branch "${DOCKER_TAG}" https://github.com/saltstack-formulas/docker-formula.git
-git clone --branch "${LOCALE_TAG}" https://github.com/saltstack-formulas/locale-formula.git
-# git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git
-git clone --branch "${NGINX_TAG}" https://github.com/netmanagers/nginx-formula.git
-git clone --branch "${POSTGRES_TAG}" https://github.com/saltstack-formulas/postgres-formula.git
-git clone --branch "${LETSENCRYPT_TAG}" https://github.com/saltstack-formulas/letsencrypt-formula.git
+echo "Cloning formulas"
+rm -rf ${F_DIR}/* || exit 1
+git clone --quiet https://github.com/saltstack-formulas/docker-formula.git ${F_DIR}/docker
+( cd docker && git checkout --quiet tags/"${DOCKER_TAG}" -b "${DOCKER_TAG}" )
+
+git clone --quiet https://github.com/saltstack-formulas/locale-formula.git ${F_DIR}/locale
+( cd locale && git checkout --quiet tags/"${LOCALE_TAG}" -b "${LOCALE_TAG}" )
+
+git clone --quiet https://github.com/netmanagers/nginx-formula.git ${F_DIR}/nginx
+( cd nginx && git checkout --quiet tags/"${NGINX_TAG}" -b "${NGINX_TAG}" )
+
+git clone --quiet https://github.com/saltstack-formulas/postgres-formula.git ${F_DIR}/postgres
+( cd postgres && git checkout --quiet tags/"${POSTGRES_TAG}" -b "${POSTGRES_TAG}" )
+
+git clone --quiet https://github.com/saltstack-formulas/letsencrypt-formula.git ${F_DIR}/letsencrypt
+( cd letsencrypt && git checkout --quiet tags/"${LETSENCRYPT_TAG}" -b "${LETSENCRYPT_TAG}" )
+
+git clone --quiet https://git.arvados.org/arvados-formula.git ${F_DIR}/arvados
# If we want to try a specific branch of the formula
if [ "x${BRANCH}" != "x" ]; then
- cd ${F_DIR}/arvados-formula || exit 1
- git checkout -t origin/"${BRANCH}" -b "${BRANCH}"
- cd -
+ ( cd ${F_DIR}/arvados && git checkout --quiet -t origin/"${BRANCH}" -b "${BRANCH}" )
+elif [ "x${ARVADOS_TAG}" != "x" ]; then
+( cd ${F_DIR}/arvados && git checkout --quiet tags/"${ARVADOS_TAG}" -b "${ARVADOS_TAG}" )
fi
if [ "x${VAGRANT}" = "xyes" ]; then
SOURCE_STATES_DIR="${EXTRA_STATES_DIR}"
+echo "Writing pillars and states"
+
# Replace variables (cluster, domain, etc) in the pillars, states and tests
# to ease deployment for newcomers
if [ ! -d "${SOURCE_PILLARS_DIR}" ]; then
echo "You requested to run tests, but ${SOURCE_TESTS_DIR} does not exist or is not a directory. Exiting."
exit 1
fi
-mkdir -p /tmp/cluster_tests
+mkdir -p ${T_DIR}
# Replace cluster and domain name in the test files
for f in $(ls "${SOURCE_TESTS_DIR}"/*); do
sed "s#__CLUSTER__#${CLUSTER}#g;
s#__INITIAL_USER__#${INITIAL_USER}#g;
s#__DATABASE_PASSWORD__#${DATABASE_PASSWORD}#g;
s#__SYSTEM_ROOT_TOKEN__#${SYSTEM_ROOT_TOKEN}#g" \
- "${f}" > "/tmp/cluster_tests"/$(basename "${f}")
+ "${f}" > ${T_DIR}/$(basename "${f}")
done
-chmod 755 /tmp/cluster_tests/run-test.sh
+chmod 755 ${T_DIR}/run-test.sh
# Replace helper state files that differ from the formula's examples
if [ -d "${SOURCE_STATES_DIR}" ]; then
done
fi
+if [ "${DUMP_CONFIG}" = "yes" ]; then
+ # We won't run the rest of the script because we're just dumping the config
+ exit 0
+fi
+
# FIXME! #16992 Temporary fix for psql call in arvados-api-server
if [ -e /root/.psqlrc ]; then
if ! ( grep 'pset pager off' /root/.psqlrc ); then
# Test that the installation finished correctly
if [ "x${TEST}" = "xyes" ]; then
- cd /tmp/cluster_tests
+ cd ${T_DIR}
./run-test.sh
fi
projects / arvados.git / commitdiff