page.find_field('public_key').set 'first test with an incorrect ssh key value'
click_button 'Submit'
- assert page.has_text?('Public key does not appear to be a valid ssh-rsa or dsa public key'), 'No text - Public key does not appear to be a valid'
+ assert_text 'Public key does not appear to be a valid ssh-rsa or dsa public key'
public_key_str = api_fixture('authorized_keys')['active']['public_key']
page.find_field('public_key').set public_key_str
page.find_field('name').set 'added_in_test'
click_button 'Submit'
- assert page.has_text?('Public key already exists in the database, use a different key.'), 'No text - Public key already exists'
+ assert_text 'Public key already exists in the database, use a different key.'
new_key = SSHKey.generate
page.find_field('public_key').set new_key.ssh_public_key
end
# key must be added. look for it in the refreshed page
- assert page.has_text?('added_in_test'), 'No text - added_in_test'
+ assert_text 'added_in_test'
end
[
--- /dev/null
+On a Debian-based system, install the following packages:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo apt-get install git curl</span>
+</code></pre>
+</notextile>
+
+On a Red Hat-based system, install the following packages:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo yum install git curl</span>
+</code></pre>
+</notextile>
Currently, only Ruby 2.1 is supported.
-h4(#rvm). *Option 1: Install with rvm*
+h4(#rvm). *Option 1: Install with RVM*
<notextile>
<pre><code><span class="userinput">sudo gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
\curl -sSL https://get.rvm.io | sudo bash -s stable --ruby=2.1
-sudo -i gem install bundler
sudo adduser "$USER" rvm
</span></code></pre></notextile>
<pre><code><span class="userinput">source /usr/local/rvm/scripts/rvm
</span></code></pre></notextile>
+Once RVM is activated in your shell, install Bundler:
+
+<notextile>
+<pre><code>~$ <span class="userinput">gem install bundler</span>
+</code></pre></notextile>
+
h4(#fromsource). *Option 2: Install from source*
Install prerequisites for Debian 7 or 8:
<pre><code>~$ <span class="userinput">sudo mkdir -p /etc/arvados/api</span>
~$ <span class="userinput">sudo chmod 700 /etc/arvados/api</span>
~$ <span class="userinput">cd /var/www/arvados-api/current</span>
-/var/www/arvados-api/current$ <span class="userinput">sudo cp config/database.yml.sample /etc/arvados/api/database.yml</span>
+/var/www/arvados-api/current$ <span class="userinput">sudo cp config/database.yml.example /etc/arvados/api/database.yml</span>
/var/www/arvados-api/current$ <span class="userinput">sudo cp config/application.yml.example /etc/arvados/api/application.yml</span>
</code></pre>
</notextile>
Edit @/etc/arvados/api/database.yml@ and replace the @xxxxxxxx@ database password placeholders with the PostgreSQL password you generated above.
-h2. Configure the API server
+h2(#configure_application). Configure the API server
-Edit @/etc/arvados/api/application.yml@ following the instructions below. The deployment script will consistently deploy this to the API server's configuration directory. The API server reads both @application.yml@ and its own @config/application.default.yml@ file. Values in @application.yml@ take precedence over the defaults that are defined in @config/application.default.yml@. The @config/application.yml.example@ file is not read by the API server and is provided for installation convenience only.
+Edit @/etc/arvados/api/application.yml@ to configure the settings described in the following sections. The deployment script will consistently deploy this to the API server's configuration directory. The API server reads both @application.yml@ and its own @config/application.default.yml@ file. The settings in @application.yml@ take precedence over the defaults that are defined in @config/application.default.yml@. The @config/application.yml.example@ file is not read by the API server and is provided as a starting template only.
-Always put your local configuration in @application.yml@ instead of editing @application.default.yml@.
+@config/application.default.yml@ documents additional configuration settings not listed here. You can "view the current source version":https://arvados.org/projects/arvados/repository/revisions/master/entry/services/api/config/application.default.yml for reference.
+
+Only put local configuration in @application.yml@. Do not edit @application.default.yml@.
h3(#uuid_prefix). uuid_prefix
-Define your @uuid_prefix@ in @application.yml@ by setting the @uuid_prefix@ field in the section for your environment. This prefix is used for all database identifiers to identify the record as originating from this site. It must be exactly 5 alphanumeric characters (lowercase ASCII letters and digits).
+Define your @uuid_prefix@ in @application.yml@ by setting the @uuid_prefix@ field in the section for your environment. This prefix is used for all database identifiers to identify the record as originating from this site. It must be exactly 5 lowercase ASCII letters and digits.
-h3(#git_repositories_dir). git_repositories_dir
+Example @application.yml@:
-This field defaults to @/var/lib/arvados/git@. You can override the value by defining it in @application.yml@.
+<notextile>
+<pre><code> uuid_prefix: <span class="userinput">zzzzz</span></code></pre>
+</notextile>
-Make sure a clone of the arvados repository exists in @git_repositories_dir@.
+h3. secret_token
+
+The @secret_token@ is used for for signing cookies. IMPORTANT: This is a site secret. It should be at least 50 characters. Generate a random value and set it in @application.yml@:
<notextile>
-<pre><code>~$ <span class="userinput">sudo mkdir -p /var/lib/arvados/git</span>
-~$ <span class="userinput">sudo git clone --bare git://git.curoverse.com/arvados.git /var/lib/arvados/git/arvados.git</span>
+<pre><code>~$ <span class="userinput">ruby -e 'puts rand(2**400).to_s(36)'</span>
+yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
</code></pre></notextile>
-h3. secret_token
+Example @application.yml@:
+
+<notextile>
+<pre><code> secret_token: <span class="userinput">yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy</span></code></pre>
+</notextile>
+
+h3(#blob_signing_key). blob_signing_key
-Generate a new secret token for signing cookies:
+The @blob_signing_key@ is used to enforce access control to Keep blocks. This same key must be provided to the Keepstore daemons when "installing Keepstore servers.":install-keepstore.html IMPORTANT: This is a site secret. It should be at least 50 characters. Generate a random value and set it in @application.yml@:
<notextile>
<pre><code>~$ <span class="userinput">ruby -e 'puts rand(2**400).to_s(36)'</span>
-zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
</code></pre></notextile>
-Then put that value in the @secret_token@ field.
+Example @application.yml@:
+
+<notextile>
+<pre><code> blob_signing_key: <span class="userinput">xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx</span></code></pre>
+</notextile>
+
+h3(#omniauth). sso_app_secret, sso_app_id, sso_provider_url
+
+The following settings enable the API server to communicate with the "Single Sign On (SSO) server":install-sso.html to authenticate user log in.
+
+Set @sso_provider_url@ to the base URL where your SSO server is installed. This should be a URL consisting of the scheme and host (and optionally, port), without a trailing slash.
-h3. blob_signing_key
+Set @sso_app_secret@ and @sso_app_id@ to the corresponding values for @app_secret@ and @app_id@ used in the "Create arvados-server client for Single Sign On (SSO)":install-sso.html#client step.
-If you want access control on your "Keepstore":install-keepstore.html server(s), you should set @blob_signing_key@ to the same value as the permission key you provide to your Keepstore daemon(s).
+Example @application.yml@:
+
+<notextile>
+<pre><code> sso_app_id: <span class="userinput">arvados-server</span>
+ sso_app_secret: <span class="userinput">wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww</span>
+ sso_provider_url: <span class="userinput">https://sso.example.com</span>
+</code></pre>
+</notextile>
h3. workbench_address
-Fill in the url of your workbench application in @workbench_address@, for example
+Set @workbench_address@ to the URL of your workbench application after following "Install Workbench.":install-workbench-app.html
+
+Example @application.yml@:
+
+<notextile>
+<pre><code> workbench_address: <span class="userinput">https://workbench.zzzzz.example.com</span></code></pre>
+</notextile>
+
+h3. websockets_address
- https://workbench.@uuid_prefix@.your.domain
+Set @websockets_address@ to the @wss://@ URL of the API server websocket endpoint after following "Set up Web servers.":#set_up
-h3(#omniauth). sso_app_id, sso_app_secret, sso_provider_url
+Example @application.yml@:
-For @sso_app_id@ and @sso_app_secret@, provide the same @app_id@ and @app_secret@ used in the "Create arvados-server client for Single Sign On (SSO)":install-sso.html#client step.
+<notextile>
+<pre><code> websockets_address: <span class="userinput">wss://ws.zzzzz.example.com</span></code></pre>
+</notextile>
+
+h3(#git_repositories_dir). git_repositories_dir
+
+The @git_repositories_dir@ setting specifies the directory where user git repositories will be stored. By default this is @/var/lib/arvados/git@.
-For @sso_provider_url@, provide the base URL where your SSO server is installed: just the scheme and host, with no trailing slash.
+Example @application.yml@:
<notextile>
-<pre><code> sso_app_id: arvados-server
- sso_app_secret: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
- sso_provider_url: https://sso.example.com
+<pre><code> git_repositories_dir: <span class="userinput">/var/lib/arvados/git</span>
</code></pre>
</notextile>
-h3. Other options
+Make sure a clone of the arvados repository exists in @git_repositories_dir@.
-Consult @/var/www/arvados-api/current/config/application.default.yml@ for a full list of configuration options. (But don't edit it. Edit @application.yml@ instead.)
+<notextile>
+<pre><code>~$ <span class="userinput">sudo mkdir -p /var/lib/arvados/git</span>
+~$ <span class="userinput">sudo git clone --bare git://git.curoverse.com/arvados.git /var/lib/arvados/git/arvados.git</span>
+</code></pre></notextile>
+
+h3(#git_internal_dir). git_internal_dir
+
+The @git_internal_dir@ setting specifies the location of Arvados' internal git repository. By default this is @/var/lib/arvados/internal.git@. This repository stores git commits that have been used to run Crunch jobs. It should _not_ be a subdirectory of @git_repositories_dir@.
+
+Example @application.yml@:
+
+<notextile>
+<pre><code> git_internal_dir: <span class="userinput">/var/lib/arvados/internal.git</span>
+</code></pre>
+</notextile>
h2. Prepare the API server deployment
This command aborts when it encounters an error. It's safe to rerun multiple times, so if there's a problem with your configuration, you can fix that and try again.
-h2. Set up Web servers
+h2(#set_up). Set up Web servers
For best performance, we recommend you use Nginx as your Web server front-end, with a Passenger backend for the main API server and a Puma backend for API server Websockets. To do that:
{% include 'note_python27_sc' %}
-h2. Set up Docker
+h2. Install Docker
Compute nodes must have Docker installed to run jobs inside containers. This requires a relatively recent version of Linux (at least upstream version 3.10, or a distribution version with the appropriate patches backported). Follow the "Docker Engine installation documentation":https://docs.docker.com/ for your distribution.
For Debian-based systems, the Arvados package repository includes a backported @docker.io@ package with a known-good version you can install.
+h2. Configure Docker
+
+Crunch runs jobs in Docker containers with relatively little configuration. You may need to start the Docker daemon with specific options to make sure these jobs run smoothly in your environment. This section highlights options that are useful to most installations. Refer to the "Docker daemon reference":https://docs.docker.com/reference/commandline/daemon/ for complete information about all available options.
+
+The best way to configure these options varies by distribution.
+
+* If you're using our backported @docker.io@ package, you can list these options in the @DOCKER_OPTS@ setting in @/etc/default/docker.io@.
+* If you're using another Debian-based package, you can list these options in the @DOCKER_OPTS@ setting in @/etc/default/docker@.
+* On Red Hat-based distributions, you can list these options in the @other_args@ setting in @/etc/sysconfig/docker@.
+
+h3. Default ulimits
+
+Docker containers inherit ulimits from the Docker daemon. However, the ulimits for a single Unix daemon may not accommodate a long-running Crunch job. You may want to increase default limits for compute jobs by passing @--default-ulimit@ options to the Docker daemon. For example, to allow jobs to open 10,000 files, set @--default-ulimit nofile=10000:10000@.
+
+h3. DNS
+
+Your containers must be able to resolve the hostname in the ARVADOS_API_HOST environment variable (provided by the Crunch dispatcher) and any hostnames returned in Keep service records. If these names are not in public DNS records, you may need to set a DNS resolver for the containers by specifying the @--dns@ address with the IP address of an appropriate nameserver. You may specify this option more than once to use multiple nameservers.
+
h2. Set up SLURM
Install SLURM following "the same process you used to install the Crunch dispatcher":{{ site.baseurl }}/install/install-crunch-dispatch.html#slurm.
# A GNU/Linux (virtual) machine
# A working Docker installation (see "Installing Docker":https://docs.docker.com/installation/)
# A working Go installation (see "Install the Go tools":https://golang.org/doc/install)
-# A working Ruby installation (see "Install Ruby and bundler":install-manual-prerequisites-ruby.html)
+# A working Ruby installation, with the Bundler gem installed
+
+h3. Install Ruby and Bundler
+
+{% include 'install_ruby_and_bundler' %}
h2. Download the source tree
h2(#dependencies). Install dependencies
+h3(#install_git_curl). Install git and curl
+
+{% include 'install_git_curl' %}
+
h3(#install_ruby_and_bundler). Install Ruby and Bundler
{% include 'install_ruby_and_bundler' %}
<pre><code>~$ <span class="userinput">cd $HOME</span> # (or wherever you want to install)
~$ <span class="userinput">git clone https://github.com/curoverse/sso-devise-omniauth-provider.git</span>
~$ <span class="userinput">cd sso-devise-omniauth-provider</span>
-~/sso-devise-omniauth-provider$ <span class="userinput">bundle</span>
+~/sso-devise-omniauth-provider$ <span class="userinput">bundle install --without=development</span>
</code></pre></notextile>
h2. Configure the SSO server
<pre><code>~/sso-devise-omniauth-provider$ <span class="userinput">cp -i config/application.yml.example config/application.yml</span>
</code></pre></notextile>
-The SSO server reads the @config/application.yml@ file, as well as the @config/application.defaults.yml@ file. Values in @config/application.yml@ take precedence over the defaults that are defined in @config/application.defaults.yml@. The @config/application.yml.example@ file is not read by the SSO server and is provided for installation convenience, only.
+The SSO server reads the @config/application.yml@ file, as well as the @config/application.defaults.yml@ file. Values in @config/application.yml@ take precedence over the defaults that are defined in @config/application.defaults.yml@. The @config/application.yml.example@ file is not read by the SSO server and is provided for installation convenience only.
-Consult @config/application.default.yml@ for a full list of configuration options. Always put your local configuration in @config/application.yml@, never edit @config/application.default.yml@.
+Consult @config/application.default.yml@ for a full list of configuration options. Local configuration goes in @config/application.yml@, do not edit @config/application.default.yml@.
h3(#uuid_prefix). uuid_prefix
-Define your @uuid_prefix@ in @config/application.yml@ by setting the @uuid_prefix@ field in the section for your environment. This prefix is used for all database identifiers to identify the record as originating from this site. It must be exactly 5 alphanumeric characters (lowercase ASCII letters and digits).
+Generate a uuid prefix for the single sign on service. This prefix is used to identify user records as originating from this site. It must be exactly 5 lowercase ASCII letters and/or digits. You may use the following snippet to generate a uuid prefix:
+
+<notextile>
+<pre><code>~/sso-devise-omniauth-provider$ <span class="userinput">ruby -e 'puts "#{rand(2**64).to_s(36)[0,5]}"'</span>
+abcde
+</code></pre></notextile>
+
+Edit @config/application.yml@ and set @uuid_prefix@ in the "common" section.
h3(#secret_token). secret_token
zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
</code></pre></notextile>
-Then put that value in the @secret_token@ field.
-
-h3(#authentication_methods). Authentication methods
+Edit @config/application.yml@ and set @secret_token@ in the "common" section.
-Three authentication methods are supported: Google+, LDAP, and local accounts.
-
-h3(#google). Google+ authentication
-
-In order to use Google+ authentication, you must use the "Google Developers Console":https://console.developers.google.com to create a set of client credentials. In short:
-
-* Enable the Contacts and Google+ APIs.
-* Create an OAuth Client ID for a web application.
-** JavaScript origins: @https://sso.example.com/@
-** Redirect URIs: @https://sso.example.com/auth/google_oauth2/callback@
-
-Copy the "Client ID" and "Client secret" values from the Google Developers Console into the Google section of @config/application.yml@, like this:
-
-<notextile>
-<pre><code>
- # Google API tokens required for OAuth2 login.
- #
- # See https://github.com/zquestz/omniauth-google-oauth2
- #
- # and https://developers.google.com/accounts/docs/OAuth2
- google_oauth2_client_id: "<span class="userinput">---YOUR---CLIENT---ID---HERE---</span>"
- google_oauth2_client_secret: "<span class="userinput">---YOUR---CLIENT---SECRET---HERE---</span>"
-
- # Set this to your OpenId 2.0 realm to enable migration from Google OpenId
- # 2.0 to Google OAuth2 OpenId Connect (Google will provide OpenId 2.0 user
- # identifiers via the openid.realm parameter in the OAuth2 flow until 2017).
- google_openid_realm: <span class="userinput">false</span></code></pre></notextile>
-
-h3(#ldap). LDAP authentication
-
-LDAP authentication can be configured with these options. Make sure to preserve the indentation of the fields beyond @use_ldap@.
-
-<pre>
- # Enable LDAP support.
- #
- # If you want to use LDAP, you need to provide
- # the following set of fields under the use_ldap key.
- #
- # use_ldap: false
- # title: Example LDAP
- # host: ldap.example.com
- # port: 636
- # method: ssl
- # base: "ou=Users, dc=example, dc=com"
- # uid: uid
- # email_domain: example.com
- # #bind_dn: "some_user"
- # #password: "some_password"
- use_ldap: false
-</pre>
-
-h3(#local_accounts). Local account authentication
-
-If neither Google OAuth2 nor LDAP are enabled, the SSO server automatically
-falls back to local accounts. There are two configuration options for local
-accounts:
-
-<pre>
- # If true, allow new creation of new accounts in the SSO server's internal
- # user database.
- allow_account_registration: false
-
- # If true, send an email confirmation before activating new accounts in the
- # SSO server's internal user database.
- require_email_confirmation: false
-</pre>
-
-You can also create local accounts on the SSO server from the rails console:
-
-<notextile>
-<pre><code>~/sso-devise-omniauth-provider$ <span class="userinput">RAILS_ENV=production bundle exec rails console</span>
-:001 > <span class="userinput">user = User.new(:email => "test@example.com")</span>
-:002 > <span class="userinput">user.password = "passw0rd"</span>
-:003 > <span class="userinput">user.save!</span>
-:004 > <span class="userinput">quit</span>
-</code></pre>
-</notextile>
-
-h2. Set up the database
+h2(#database). Set up the database
Generate a new database password. Nobody ever needs to memorize it or type it, so make a strong one:
Enter it again: <span class="userinput">paste-database-password-you-generated</span>
</code></pre></notextile>
-Configure SSO server to connect to your database by creating and updating @config/database.yml@. Replace the @xxxxxxxx@ database password placeholders with the new password you generated above.
+Configure SSO server to connect to your database by creating and updating @config/database.yml@. Replace the @xxxxxxxx@ database password placeholders with the new password you generated above. If you are planning a production system, update the @production@ section, otherwise use @development@.
<notextile>
-<pre><code>~/sso-devise-omniauth-provider$ <span class="userinput">cp -i config/database.yml.sample config/database.yml</span>
+<pre><code>~/sso-devise-omniauth-provider$ <span class="userinput">cp -i config/database.yml.example config/database.yml</span>
~/sso-devise-omniauth-provider$ <span class="userinput">edit config/database.yml</span>
</code></pre></notextile>
Alternatively, if the database user you intend to use for the SSO server is not allowed to create new databases, you can create the database first and then populate it with rake. Be sure to adjust the database name if you are using the @development@ environment. This sequence of commands is functionally equivalent to the rake db:setup command above:
<notextile>
-<pre><code>~/sso-devise-omniauth-provider$ <span class="userinput">su postgres createdb arvados_sso_production -E UTF8 -O arvados_sso</span>
+<pre><code>~/sso-devise-omniauth-provider$ <span class="userinput">sudo -u postgres createdb arvados_sso_production -E UTF8 -O arvados_sso -T template0</span>
~/sso-devise-omniauth-provider$ <span class="userinput">RAILS_ENV=production bundle exec rake db:schema:load</span>
~/sso-devise-omniauth-provider$ <span class="userinput">RAILS_ENV=production bundle exec rake db:seed</span>
</code></pre></notextile>
-h2(#assets). Generate assets
-
-If you are running in the production environment, you'll want to generate the assets:
-
-<notextile>
-<pre><code>~/sso-devise-omniauth-provider$ <span class="userinput">RAILS_ENV=production bundle exec rake assets:precompile</span>
-</code></pre>
-</notextile>
-
h2(#client). Create arvados-server client
-Use @rails console@ to create a @Client@ record that will be used by the Arvados API server. The values of @app_id@ and @app_secret@ correspond to the @APP_ID@ and @APP_SECRET@ that must be set in in "Setting up Omniauth in the API server.":install-api-server.html#omniauth
+Use @rails console@ to create a @Client@ record that will be used by the Arvados API server. The values of @app_id@ and @app_secret@ correspond to the values for @sso_app_id@ and @sso_app_secret@ in the "API server's SSO settings.":install-api-server.html#omniauth
<notextile>
<pre><code>~/sso-devise-omniauth-provider$ <span class="userinput">ruby -e 'puts rand(2**400).to_s(36)'</span>
</code></pre>
</notextile>
-h2. Start the SSO server
+h2(#assets). Precompile assets
+
+If you are running in the production environment, you must precompile the assets:
+
+<notextile>
+<pre><code>~/sso-devise-omniauth-provider$ <span class="userinput">RAILS_ENV=production bundle exec rake assets:precompile</span>
+</code></pre>
+</notextile>
+
+h2(#authentication_methods). Authentication methods
+
+Authentication methods are configured in @application.yml@. Currently three authentication methods are supported: local accounts, LDAP, and Google+. If neither Google+ nor LDAP are enabled, the SSO server defaults to local user accounts. Only one authentication mechanism should be in use at a time.
+
+h3(#local_accounts). Local account authentication
+
+There are two configuration options for local accounts:
+
+<pre>
+ # If true, allow new creation of new accounts in the SSO server's internal
+ # user database.
+ allow_account_registration: false
+
+ # If true, send an email confirmation before activating new accounts in the
+ # SSO server's internal user database (otherwise users are activated immediately.)
+ require_email_confirmation: false
+</pre>
+
+For more information about configuring backend support for sending email (required to send email confirmations) see "Configuring Action Mailer":http://guides.rubyonrails.org/configuring.html#configuring-action-mailer
+
+If @allow_account_registration@ is false, you may manually create local accounts on the SSO server from the rails console:
+
+<notextile>
+<pre><code>~/sso-devise-omniauth-provider$ <span class="userinput">RAILS_ENV=production bundle exec rails console</span>
+:001 > <span class="userinput">user = User.new(:email => "test@example.com")</span>
+:002 > <span class="userinput">user.password = "passw0rd"</span>
+:003 > <span class="userinput">user.save!</span>
+:004 > <span class="userinput">quit</span>
+</code></pre>
+</notextile>
+
+h3(#ldap). LDAP authentication
+
+The following options are available to configure LDAP authentication. Note that you must preserve the indentation of the fields listed under @use_ldap@.
+
+<pre>
+ use_ldap:
+ title: Example LDAP
+ host: ldap.example.com
+ port: 636
+ method: ssl
+ base: "ou=Users, dc=example, dc=com"
+ uid: uid
+ email_domain: example.com
+ #bind_dn: "some_user"
+ #password: "some_password"
+</pre>
+
+table(table).
+|_. Option|_. Description|
+|title |Title displayed to the user on the login page|
+|host |LDAP server hostname|
+|port |LDAP server port|
+|method|One of "plain", "ssl", "tls"|
+|base |Directory lookup base|
+|uid |User id field used for directory lookup|
+|email_domain|Strip off specified email domain from login and perform lookup on bare username|
+|bind_dn|If required by server, username to log with in before performing directory lookup|
+|password|If required by server, password to log with before performing directory lookup|
+
+h3(#google). Google+ authentication
+
+In order to use Google+ authentication, you must use the <a href="https://console.developers.google.com" target="_blank">Google Developers Console</a> to create a set of client credentials.
+
+# Go to the <a href="https://console.developers.google.com" target="_blank">Google Developers Console</a> and select or create a project; this will take you to the project page.
+# On the sidebar, click on *APIs & auth* then select *APIs*.
+## Search for *Contacts API* and click on *Enable API*.
+## Search for *Google+ API* and click on *Enable API*.
+# On the sidebar, click on *Credentials*; under *OAuth* click on *Create new Client ID* to bring up the *Create Client ID* dialog box.
+# Under *Application type* select *Web application*.
+# If the authorization origins are not displayed, clicking on *Create Client ID* will take you to *Consent screen* settings.
+## On consent screen settings, enter the appropriate details and click on *Save*.
+## This will return you to the *Create Client ID* dialog box.
+# You must set the authorization origins. Edit @sso.your-site.com@ to the appropriate hostname that you will use to access the SSO service:
+## JavaScript origin should be @https://sso.your-site.com/@
+## Redirect URI should be @https://sso.your-site.com/users/auth/google_oauth2/callback@
+# Copy the values of *Client ID* and *Client secret* from the Google Developers Console into the Google section of @config/application.yml@, like this:
-h3. Run a simple standalone server
+<notextile>
+<pre><code> # Google API tokens required for OAuth2 login.
+ google_oauth2_client_id: <span class="userinput">"---YOUR---CLIENT---ID---HERE--"-</span>
+ google_oauth2_client_secret: <span class="userinput">"---YOUR---CLIENT---SECRET---HERE--"-</span></code></pre></notextile>
+
+h2(#start). Start the SSO server
-You can use the Webrick server that is bundled with Ruby to quickly verify that your installation is functioning:
+h3. Run a standalone passenger server
<notextile>
-<pre><code>~/sso-devise-omniauth-provider$ <span class="userinput">RAILS_ENV=production bundle exec rails server</span>
+<pre><code>~/sso-devise-omniauth-provider$ <span class="userinput">RAILS_ENV=production passenger start</span>
+=============== Phusion Passenger Standalone web server started ===============
+...
</code></pre>
</notextile>
-h3. Production environment
+You can now test your installation by going to the page reported by passenger as "Accessible via: ..."
-As a Ruby on Rails application, the SSO server should be compatible with any Ruby application server that supports Rack applications. We recommend "Passenger":https://www.phusionpassenger.com/ to run the SSO server in production.
+Note: if you get the following warning "you may safely ignore it:":https://stackoverflow.com/questions/10374871/no-secret-option-provided-to-racksessioncookie-warning
+
+<pre>
+Connecting to database specified by database.yml
+App 4574 stderr: SECURITY WARNING: No secret option provided to Rack::Session::Cookie.
+App 4574 stderr: This poses a security threat. It is strongly recommended that you
+App 4574 stderr: provide a secret to prevent exploits that may be possible from crafted
+App 4574 stderr: cookies. This will not be supported in future versions of Rack, and
+App 4574 stderr: future versions will even invalidate your existing user cookies.
+App 4574 stderr:
+App 4574 stderr: Called from: /var/lib/gems/2.1.0/gems/actionpack-3.2.8/lib/action_dispatch/middleware/session/abstract_store.rb:28:in `initialize'.
+App 4592 stdout:
+</pre>
To use the @arv@ command, you can either install the @arvados-cli@ gem via RubyGems or build and install the package from source.
-h4. Prerequisites: Ruby >= 2.1.0 and curl libraries
+h3. Prerequisites: Ruby, Bundler, and curl libraries
-Make sure you have "Ruby and bundler":{{site.baseurl}}/install/install-manual-prerequisites-ruby.html installed.
+{% include 'install_ruby_and_bundler' %}
Install curl libraries with your system's package manager. For example, on Debian or Ubuntu:
</pre>
</notextile>
-h4. Option 1: install with RubyGems
+h3. Option 1: Install with RubyGems
<notextile>
<pre>
</pre>
</notextile>
-h4. Option 2: build and install from source
+h3. Option 2: Build and install from source
<notextile>
<pre>
my $job_api_token;
my $no_clear_tmp;
my $resume_stash;
-my $docker_bin = "/usr/bin/docker.io";
+my $docker_bin = "docker.io";
GetOptions('force-unlock' => \$force_unlock,
'git-dir=s' => \$git_dir,
'job=s' => \$jobspec,
}
else
{
- $Job = JSON::decode_json($jobspec);
- $local_job = 1;
+ $local_job = JSON::decode_json($jobspec);
}
# at least able to run basic commands: they aren't down or severely
# misconfigured.
my $cmd = ['true'];
-if ($Job->{docker_image_locator}) {
+if (($Job || $local_job)->{docker_image_locator}) {
$cmd = [$docker_bin, 'ps', '-q'];
}
Log(undef, "Sanity check is `@$cmd`");
{
if (!$resume_stash)
{
- map { croak ("No $_ specified") unless $Job->{$_} }
+ map { croak ("No $_ specified") unless $local_job->{$_} }
qw(script script_version script_parameters);
}
- $Job->{'is_locked_by_uuid'} = $User->{'uuid'};
- $Job->{'started_at'} = gmtime;
- $Job->{'state'} = 'Running';
+ $local_job->{'is_locked_by_uuid'} = $User->{'uuid'};
+ $local_job->{'started_at'} = gmtime;
+ $local_job->{'state'} = 'Running';
- $Job = api_call("jobs/create", job => $Job);
+ $Job = api_call("jobs/create", job => $local_job);
}
$job_id = $Job->{'uuid'};
# TODO: When #5036 is done and widely deployed, we can get rid of the
# regular expression and just unmount everything with type fuse.keep.
srun (["srun", "--nodelist=$nodelist", "-D", $ENV{'TMPDIR'}],
- ['bash', '-ec', 'mount -t fuse,fuse.keep | awk \'($3 ~ /\ykeep\y/){print $3}\' | xargs -r -n 1 fusermount -u -z; sleep 1; rm -rf $JOB_WORK $CRUNCH_INSTALL $CRUNCH_TMP/task $CRUNCH_TMP/src* $CRUNCH_TMP/*.cid']);
+ ['bash', '-ec', '-o', 'pipefail', 'mount -t fuse,fuse.keep | awk \'($3 ~ /\ykeep\y/){print $3}\' | xargs -r -n 1 fusermount -u -z; sleep 1; rm -rf $JOB_WORK $CRUNCH_INSTALL $CRUNCH_TMP/task $CRUNCH_TMP/src* $CRUNCH_TMP/*.cid']);
exit (1);
}
while (1)
freeze_if_want_freeze ($cleanpid);
select (undef, undef, undef, 0.1);
}
- Log (undef, "Cleanup command exited ".exit_status_s($?));
+ if ($?) {
+ Log(undef, "Clean work dirs: exit ".exit_status_s($?));
+ exit(EX_RETRY_UNLOCKED);
+ }
}
# If this job requires a Docker image, install that.
unless ($? == 0 && $sha1 =~ /^([0-9a-f]{40})$/) {
croak("`$gitcmd rev-list` exited "
.exit_status_s($?)
- .", '$treeish' not found. Giving up.");
+ .", '$treeish' not found, giving up");
}
$commit = $1;
Log(undef, "Version $treeish is commit $commit");
$command .= "--memory=\${MEMLIMIT}k --memory-swap=\${SWAPLIMIT}k ";
}
- # Dynamically configure the container to use the host system as its
- # DNS server. Get the host's global addresses from the ip command,
- # and turn them into docker --dns options using gawk.
- $command .=
- q{$(ip -o address show scope global |
- gawk 'match($4, /^([0-9\.:]+)\//, x){print "--dns", x[1]}') };
-
# The source tree and $destdir directory (which we have
# installed on the worker host) are available in the container,
# under the same path.
update_progress_stats();
select (undef, undef, undef, 0.1);
}
- elsif (time - $progress_stats_updated >= 30)
+ elsif (time - $progress_stats_updated >= 30 || $progress_is_dirty)
{
update_progress_stats();
}
$progress_stats_updated = time;
return if !$progress_is_dirty;
my ($todo, $done, $running) = (scalar @jobstep_todo,
- scalar @jobstep_done,
- scalar @slot - scalar @freeslot - scalar @holdslot);
+ scalar @jobstep_done,
+ scalar keys(%proc));
$Job->{'tasks_summary'} ||= {};
$Job->{'tasks_summary'}->{'todo'} = $todo;
$Job->{'tasks_summary'}->{'done'} = $done;
--- /dev/null
+#!/bin/sh
+echo >&2 Failing mount stub was called
+exit 1
--- /dev/null
+#!/bin/sh
+true
--- /dev/null
+#!/bin/sh
+exit 8
--- /dev/null
+#!/bin/sh
+exit 7
end
def test_small_collection
- skip "Waiting unitl #4534 is implemented"
-
uuid = Digest::MD5.hexdigest(foo_manifest) + '+' + foo_manifest.size.to_s
out, err = capture_subprocess_io do
assert_arv('--format', 'uuid', 'collection', 'create', '--collection', {
end
def test_file_to_dev_stdout
- skip "Waiting unitl #4534 is implemented"
-
test_file_to_stdout('/dev/stdout')
end
def test_file_to_stdout(specify_stdout_as='-')
- skip "Waiting unitl #4534 is implemented"
-
out, err = capture_subprocess_io do
assert_arv_get @@foo_manifest_locator + '/foo', specify_stdout_as
end
end
def test_file_to_file
- skip "Waiting unitl #4534 is implemented"
-
remove_tmp_foo
out, err = capture_subprocess_io do
assert_arv_get @@foo_manifest_locator + '/foo', 'tmp/foo'
end
def test_file_to_file_no_overwrite_file
- skip "Waiting unitl #4534 is implemented"
File.open './tmp/foo', 'wb' do |f|
f.write 'baz'
end
out, err = capture_subprocess_io do
assert_arv_get false, @@foo_manifest_locator + '/foo', 'tmp/foo'
end
- assert_match /Error:/, err
+ assert_match /Local file tmp\/foo already exists/, err
assert_equal '', out
assert_equal 'baz', IO.read('tmp/foo')
end
def test_file_to_file_no_overwrite_file_in_dir
- skip "Waiting unitl #4534 is implemented"
File.open './tmp/foo', 'wb' do |f|
f.write 'baz'
end
out, err = capture_subprocess_io do
assert_arv_get false, @@foo_manifest_locator + '/', 'tmp/'
end
- assert_match /Error:/, err
+ assert_match /Local file tmp\/foo already exists/, err
assert_equal '', out
assert_equal 'baz', IO.read('tmp/foo')
end
def test_file_to_file_force_overwrite
- skip "Waiting unitl #4534 is implemented"
-
File.open './tmp/foo', 'wb' do |f|
f.write 'baz'
end
end
def test_file_to_file_skip_existing
- skip "Waiting unitl #4534 is implemented"
-
File.open './tmp/foo', 'wb' do |f|
f.write 'baz'
end
end
def test_file_to_dir
- skip "Waiting unitl #4534 is implemented"
-
remove_tmp_foo
out, err = capture_subprocess_io do
assert_arv_get @@foo_manifest_locator + '/foo', 'tmp/'
end
def test_nonexistent_block
- skip "Waiting unitl #4534 is implemented"
-
out, err = capture_subprocess_io do
- assert_arv_get false, 'f1554a91e925d6213ce7c3103c5110c6'
+ assert_arv_get false, 'e796ab2294f3e48ec709ffa8d6daf58c'
end
assert_equal '', out
assert_match /Error:/, err
end
def test_nonexistent_manifest
- skip "Waiting unitl #4534 is implemented"
-
out, err = capture_subprocess_io do
- assert_arv_get false, 'f1554a91e925d6213ce7c3103c5110c6/', 'tmp/'
+ assert_arv_get false, 'acbd18db4cc2f85cedef654fccc4a4d8/', 'tmp/'
end
assert_equal '', out
assert_match /Error:/, err
end
def test_manifest_root_to_dir
- skip "Waiting unitl #4534 is implemented"
-
remove_tmp_foo
out, err = capture_subprocess_io do
assert_arv_get '-r', @@foo_manifest_locator + '/', 'tmp/'
end
def test_manifest_root_to_dir_noslash
- skip "Waiting unitl #4534 is implemented"
-
remove_tmp_foo
out, err = capture_subprocess_io do
assert_arv_get '-r', @@foo_manifest_locator + '/', 'tmp'
end
def test_display_md5sum
- skip "Waiting unitl #4534 is implemented"
-
remove_tmp_foo
out, err = capture_subprocess_io do
assert_arv_get '-r', '--md5sum', @@foo_manifest_locator + '/', 'tmp/'
end
def test_md5sum_nowrite
- skip "Waiting unitl #4534 is implemented"
-
remove_tmp_foo
out, err = capture_subprocess_io do
assert_arv_get '-n', '--md5sum', @@foo_manifest_locator + '/', 'tmp/'
end
def test_sha1_nowrite
- skip "Waiting unitl #4534 is implemented"
-
remove_tmp_foo
out, err = capture_subprocess_io do
assert_arv_get '-n', '-r', '--hash', 'sha1', @@foo_manifest_locator+'/', 'tmp/'
end
def test_block_to_file
- skip "Waiting unitl #4534 is implemented"
-
remove_tmp_foo
out, err = capture_subprocess_io do
assert_arv_get @@foo_manifest_locator, 'tmp/foo'
end
def test_create_directory_tree
- skip "Waiting unitl #4534 is implemented"
-
`rm -rf ./tmp/arv-get-test/`
Dir.mkdir './tmp/arv-get-test'
out, err = capture_subprocess_io do
end
def test_create_partial_directory_tree
- skip "Waiting unitl #4534 is implemented"
-
`rm -rf ./tmp/arv-get-test/`
Dir.mkdir './tmp/arv-get-test'
out, err = capture_subprocess_io do
end
def test_raw_stdin
- skip "Waiting unitl #4534 is implemented"
-
out, err = capture_subprocess_io do
r,w = IO.pipe
wpid = fork do
end
def test_raw_file
- skip "Waiting unitl #4534 is implemented"
-
out, err = capture_subprocess_io do
assert arv_put('--raw', './tmp/foo')
end
end
def test_raw_empty_file
- skip "Waiting unitl #4534 is implemented"
-
out, err = capture_subprocess_io do
assert arv_put('--raw', './tmp/empty_file')
end
end
def test_filename_arg_with_empty_file
- skip "Waiting unitl #4534 is implemented"
-
out, err = capture_subprocess_io do
assert arv_put('--filename', 'foo', './tmp/empty_file')
end
end
def test_as_stream
- skip "Waiting unitl #4534 is implemented"
-
out, err = capture_subprocess_io do
assert arv_put('--as-stream', './tmp/foo')
end
end
def test_progress
- skip "Waiting unitl #4534 is implemented"
-
out, err = capture_subprocess_io do
assert arv_put('--manifest', '--progress', './tmp/foo')
end
end
def test_batch_progress
- skip "Waiting unitl #4534 is implemented"
-
out, err = capture_subprocess_io do
assert arv_put('--manifest', '--batch-progress', './tmp/foo')
end
end
def test_read_from_implicit_stdin
- skip "Waiting unitl #4534 is implemented"
-
test_read_from_stdin(specify_stdin_as='--manifest')
end
def test_read_from_dev_stdin
- skip "Waiting unitl #4534 is implemented"
-
test_read_from_stdin(specify_stdin_as='/dev/stdin')
end
def test_read_from_stdin(specify_stdin_as='-')
- skip "Waiting unitl #4534 is implemented"
-
out, err = capture_subprocess_io do
r,w = IO.pipe
wpid = fork do
end
def test_read_from_implicit_stdin_implicit_manifest
- skip "Waiting unitl #4534 is implemented"
-
test_read_from_stdin_implicit_manifest(specify_stdin_as=nil,
expect_filename='stdin')
end
def test_read_from_dev_stdin_implicit_manifest
- skip "Waiting unitl #4534 is implemented"
-
test_read_from_stdin_implicit_manifest(specify_stdin_as='/dev/stdin')
end
def test_read_from_stdin_implicit_manifest(specify_stdin_as='-',
expect_filename=nil)
- skip "Waiting unitl #4534 is implemented"
-
expect_filename = expect_filename || specify_stdin_as.split('/').last
out, err = capture_subprocess_io do
r,w = IO.pipe
end
def test_run_pipeline_instance_get_help
- skip "Waiting unitl #4534 is implemented"
-
out, err = capture_subprocess_io do
system ('arv-run-pipeline-instance -h')
end
class TestArvTag < Minitest::Test
def test_no_args
- skip "Waiting unitl #4534 is implemented"
+ skip "Waiting until #4534 is implemented"
# arv-tag exits with failure if run with no args
out, err = capture_subprocess_io do
--- /dev/null
+require 'minitest/autorun'
+
+class TestCrunchJob < Minitest::Test
+ SPECIAL_EXIT = {
+ EX_RETRY_UNLOCKED: 93,
+ EX_TEMPFAIL: 75,
+ }
+
+ JOBSPEC = {
+ grep_local: {
+ script: 'grep',
+ script_version: 'master',
+ repository: File.absolute_path('../../../..', __FILE__),
+ script_parameters: {foo: 'bar'},
+ },
+ }
+
+ def setup
+ end
+
+ def crunchjob
+ File.absolute_path '../../bin/crunch-job', __FILE__
+ end
+
+ # Return environment suitable for running crunch-job.
+ def crunchenv opts={}
+ env = ENV.to_h
+ env['CRUNCH_REFRESH_TRIGGER'] =
+ File.absolute_path('../../../../tmp/crunch-refresh-trigger', __FILE__)
+ env
+ end
+
+ def jobspec label
+ JOBSPEC[label].dup
+ end
+
+ # Encode job record to json and run it with crunch-job.
+ #
+ # opts[:binstubs] is an array of X where ./binstub_X is added to
+ # PATH in order to mock system programs.
+ def tryjobrecord jobrecord, opts={}
+ env = crunchenv
+ (opts[:binstubs] || []).each do |binstub|
+ env['PATH'] = File.absolute_path('../binstub_'+binstub, __FILE__) + ':' + env['PATH']
+ end
+ system env, crunchjob, '--job', jobrecord.to_json
+ end
+
+ def test_bogus_json
+ out, err = capture_subprocess_io do
+ system crunchenv, crunchjob, '--job', '"}{"'
+ end
+ assert_equal false, $?.success?
+ # Must not conflict with our special exit statuses
+ assert_jobfail $?
+ assert_match /JSON/, err
+ end
+
+ def test_fail_sanity_check
+ out, err = capture_subprocess_io do
+ j = {}
+ tryjobrecord j, binstubs: ['sanity_check']
+ end
+ assert_equal 75, $?.exitstatus
+ assert_match /Sanity check failed: 7/, err
+ end
+
+ def test_fail_docker_sanity_check
+ out, err = capture_subprocess_io do
+ j = {}
+ j[:docker_image_locator] = '4d449b9d34f2e2222747ef79c53fa3ff+1234'
+ tryjobrecord j, binstubs: ['sanity_check']
+ end
+ assert_equal 75, $?.exitstatus
+ assert_match /Sanity check failed: 8/, err
+ end
+
+ def test_no_script_specified
+ out, err = capture_subprocess_io do
+ j = jobspec :grep_local
+ j.delete :script
+ tryjobrecord j
+ end
+ assert_match /No script specified/, err
+ assert_jobfail $?
+ end
+
+ def test_fail_clean_tmp
+ out, err = capture_subprocess_io do
+ j = jobspec :grep_local
+ tryjobrecord j, binstubs: ['clean_fail']
+ end
+ assert_match /Failing mount stub was called/, err
+ assert_match /Clean work dirs: exit 1\n$/, err
+ assert_equal SPECIAL_EXIT[:EX_RETRY_UNLOCKED], $?.exitstatus
+ end
+
+ def test_docker_image_missing
+ skip 'API bug: it refuses to create this job in Running state'
+ out, err = capture_subprocess_io do
+ j = jobspec :grep_local
+ j[:docker_image_locator] = '4d449b9d34f2e2222747ef79c53fa3ff+1234'
+ tryjobrecord j, binstubs: ['docker_noop']
+ end
+ assert_match /No Docker image hash found from locator/, err
+ assert_jobfail $?
+ end
+
+ def test_script_version_not_found_in_repository
+ bogus_version = 'f8b72707c1f5f740dbf1ed56eb429a36e0dee770'
+ out, err = capture_subprocess_io do
+ j = jobspec :grep_local
+ j[:script_version] = bogus_version
+ tryjobrecord j
+ end
+ assert_match /'#{bogus_version}' not found, giving up/, err
+ assert_jobfail $?
+ end
+
+ # Ensure procstatus is not interpreted as a temporary infrastructure
+ # problem. Would be assert_http_4xx if this were http.
+ def assert_jobfail procstatus
+ refute_includes SPECIAL_EXIT.values, procstatus.exitstatus
+ assert_equal false, procstatus.success?
+ end
+end
WriteMakefile(
NAME => 'Arvados',
- VERSION_FROM => 'lib/Arvados.pm'
+ VERSION_FROM => 'lib/Arvados.pm',
+ PREREQ_PM => {
+ 'JSON' => 0,
+ 'LWP' => 0,
+ 'Net::SSL' => 0,
+ },
);
def check_git_availability():
try:
arvados.util.run_command(['git', '--help'])
- except:
+ except Exception:
abort('git command is not available. Please ensure git is installed.')
# copy_pipeline_instance(pi_uuid, src, dst, args)
obj = arvados.util.portable_data_hash_pattern.sub(copy_collection_fn, obj)
obj = arvados.util.collection_uuid_pattern.sub(copy_collection_fn, obj)
return obj
- elif type(obj) == dict:
+ elif isinstance(obj, dict):
return {v: copy_collections(obj[v], src, dst, args) for v in obj}
- elif type(obj) == list:
+ elif isinstance(obj, list):
return [copy_collections(v, src, dst, args) for v in obj]
return obj
skip_before_filter :find_object_by_uuid, :only => :get_all_permissions
skip_before_filter :render_404_if_no_object, :only => :get_all_permissions
before_filter :admin_required, :only => :get_all_permissions
+
def get_all_permissions
- @users = {}
- User.includes(:authorized_keys).find_each do |u|
- @users[u.uuid] = u
+ # users is a map of {user_uuid => User object}
+ users = {}
+ # user_aks is a map of {user_uuid => array of public keys}
+ user_aks = {}
+ # admins is an array of user_uuids
+ admins = []
+ User.eager_load(:authorized_keys).find_each do |u|
+ next unless u.is_active or u.uuid == anonymous_user_uuid
+ users[u.uuid] = u
+ user_aks[u.uuid] = u.authorized_keys.collect do |ak|
+ {
+ public_key: ak.public_key,
+ authorized_key_uuid: ak.uuid
+ }
+ end
+ admins << u.uuid if u.is_admin
end
- admins = @users.select { |k,v| v.is_admin }
- @user_aks = {}
@repo_info = {}
- Repository.includes(:permissions).find_each do |repo|
+ Repository.eager_load(:permissions).find_each do |repo|
@repo_info[repo.uuid] = {
uuid: repo.uuid,
name: repo.name,
fetch_url: repo.fetch_url,
user_permissions: {},
}
- gitolite_permissions = ''
- perms = []
+ # evidence is an array of {name: 'can_xxx', user_uuid: 'x-y-z'},
+ # one entry for each piece of evidence we find in the permission
+ # database that establishes that a user can access this
+ # repository. Multiple entries can be added for a given user,
+ # possibly with different access levels; these will be compacted
+ # below.
+ evidence = []
repo.permissions.each do |perm|
if ArvadosModel::resource_class_for_uuid(perm.tail_uuid) == Group
- @users.each do |user_uuid, user|
- user.group_permissions.each do |group_uuid, perm_mask|
- if perm_mask[:manage]
- perms << {name: 'can_manage', user_uuid: user_uuid}
- elsif perm_mask[:write]
- perms << {name: 'can_write', user_uuid: user_uuid}
- elsif perm_mask[:read]
- perms << {name: 'can_read', user_uuid: user_uuid}
- end
+ # A group has permission. Each user who has access to this
+ # group also has access to the repository. Access level is
+ # min(group-to-repo permission, user-to-group permission).
+ users.each do |user_uuid, user|
+ perm_mask = user.group_permissions[perm.tail_uuid]
+ if not perm_mask
+ next
+ elsif perm_mask[:manage] and perm.name == 'can_manage'
+ evidence << {name: 'can_manage', user_uuid: user_uuid}
+ elsif perm_mask[:write] and ['can_manage', 'can_write'].index perm.name
+ evidence << {name: 'can_write', user_uuid: user_uuid}
+ elsif perm_mask[:read]
+ evidence << {name: 'can_read', user_uuid: user_uuid}
end
end
- else
- perms << {name: perm.name, user_uuid: perm.tail_uuid}
+ elsif users[perm.tail_uuid]
+ # A user has permission; the user exists; and either the
+ # user is active, or it's the special case of the anonymous
+ # user which is never "active" but is allowed to read
+ # content from public repositories.
+ evidence << {name: perm.name, user_uuid: perm.tail_uuid}
end
end
- # Owner of the repository, and all admins, can RW
- ([repo.owner_uuid] + admins.keys).each do |user_uuid|
- perms << {name: 'can_write', user_uuid: user_uuid}
+ # Owner of the repository, and all admins, can do everything.
+ ([repo.owner_uuid] | admins).each do |user_uuid|
+ # Except: no permissions for inactive users, even if they own
+ # repositories.
+ next unless users[user_uuid]
+ evidence << {name: 'can_manage', user_uuid: user_uuid}
end
- perms.each do |perm|
+ # Distill all the evidence about permissions on this repository
+ # into one hash per user, of the form {'can_xxx' => true, ...}.
+ # The hash is nil for a user who has no permissions at all on
+ # this particular repository.
+ evidence.each do |perm|
user_uuid = perm[:user_uuid]
- @user_aks[user_uuid] = @users[user_uuid].andand.authorized_keys.andand.
- collect do |ak|
- {
- public_key: ak.public_key,
- authorized_key_uuid: ak.uuid
- }
- end || []
- if @user_aks[user_uuid].any?
- ri = (@repo_info[repo.uuid][:user_permissions][user_uuid] ||= {})
- ri[perm[:name]] = true
- end
+ user_perms = (@repo_info[repo.uuid][:user_permissions][user_uuid] ||= {})
+ user_perms[perm[:name]] = true
end
end
- @repo_info.values.each do |repo_users|
- repo_users[:user_permissions].each do |user_uuid,perms|
- if perms['can_manage']
- perms[:gitolite_permissions] = 'RW'
- perms['can_write'] = true
- perms['can_read'] = true
- elsif perms['can_write']
- perms[:gitolite_permissions] = 'RW'
- perms['can_read'] = true
- elsif perms['can_read']
- perms[:gitolite_permissions] = 'R'
+ # Revisit each {'can_xxx' => true, ...} hash for some final
+ # cleanup to make life easier for the requestor.
+ #
+ # Add a 'gitolite_permissions' key alongside the 'can_xxx' keys,
+ # for the convenience of the gitolite config file generator.
+ #
+ # Add all lesser permissions when a greater permission is
+ # present. If the requestor only wants to know who can write, it
+ # only has to test for 'can_write' in the response.
+ @repo_info.values.each do |repo|
+ repo[:user_permissions].each do |user_uuid, user_perms|
+ if user_perms['can_manage']
+ user_perms['gitolite_permissions'] = 'RW'
+ user_perms['can_write'] = true
+ user_perms['can_read'] = true
+ elsif user_perms['can_write']
+ user_perms['gitolite_permissions'] = 'RW'
+ user_perms['can_read'] = true
+ elsif user_perms['can_read']
+ user_perms['gitolite_permissions'] = 'R'
end
end
end
+ # The response looks like
+ # {"kind":"...",
+ # "repositories":[r1,r2,r3,...],
+ # "user_keys":usermap}
+ # where each of r1,r2,r3 looks like
+ # {"uuid":"repo-uuid-1",
+ # "name":"username/reponame",
+ # "push_url":"...",
+ # "user_permissions":{"user-uuid-a":{"can_read":true,"gitolite_permissions":"R"}}}
+ # and usermap looks like
+ # {"user-uuid-a":[{"public_key":"ssh-rsa g...","authorized_key_uuid":"ak-uuid-g"},...],
+ # "user-uuid-b":[{"public_key":"ssh-rsa h...","authorized_key_uuid":"ak-uuid-h"},...],...}
send_json(kind: 'arvados#RepositoryPermissionSnapshot',
repositories: @repo_info.values,
- user_keys: @user_aks)
+ user_keys: user_aks)
end
end
after_destroy :log_destroy
after_find :convert_serialized_symbols_to_strings
before_validation :normalize_collection_uuids
+ before_validation :set_default_owner
validate :ensure_serialized_attribute_type
validate :ensure_valid_uuids
true
end
- def ensure_owner_uuid_is_permitted
- raise PermissionDeniedError if !current_user
-
- if new_record? and respond_to? :owner_uuid=
+ def set_default_owner
+ if new_record? and current_user and respond_to? :owner_uuid=
self.owner_uuid ||= current_user.uuid
end
+ end
+
+ def ensure_owner_uuid_is_permitted
+ raise PermissionDeniedError if !current_user
if self.owner_uuid.nil?
errors.add :owner_uuid, "cannot be nil"
def public_key_must_be_unique
if self.public_key
- #key = /^ssh-(rsa|dss) [A-Za-z0-9+\/=\+]+\b/.match(self.public_key)
valid_key = SSHKey.valid_ssh_public_key? self.public_key
if not valid_key
errors.add(:public_key, "does not appear to be a valid ssh-rsa or dsa public key")
else
# Valid if no other rows have this public key
- if self.class.where('public_key like ?', "%#{self.public_key}%").any?
+ if self.class.where('uuid != ? and public_key like ?',
+ uuid || '', "%#{self.public_key}%").any?
errors.add(:public_key, "already exists in the database, use a different key.")
return false
end
# Get the commit hash for the upper bound
max_hash = nil
- IO.foreach("|git rev-list --max-count=1 #{maximum.shellescape} --") do |line|
+ git_max_hash_cmd = "git rev-list --max-count=1 #{maximum.shellescape} --"
+ IO.foreach("|#{git_max_hash_cmd}") do |line|
max_hash = line.strip
end
- # If not found or string is invalid, nothing else to do
- return [] if !max_hash or !git_check_ref_format(max_hash)
+ # If not found, nothing else to do
+ if !max_hash
+ logger.warn "no refs found looking for max_hash: `GIT_DIR=#{gitdir} #{git_max_hash_cmd}` returned no output"
+ return []
+ end
+
+ # If string is invalid, nothing else to do
+ if !git_check_ref_format(max_hash)
+ logger.warn "ref returned by `GIT_DIR=#{gitdir} #{git_max_hash_cmd}` was invalid for max_hash: #{max_hash}"
+ return []
+ end
resolved_exclude = nil
if exclude
if minimum
# Get the commit hash for the lower bound
min_hash = nil
- IO.foreach("|git rev-list --max-count=1 #{minimum.shellescape} --") do |line|
+ git_min_hash_cmd = "git rev-list --max-count=1 #{minimum.shellescape} --"
+ IO.foreach("|#{git_min_hash_cmd}") do |line|
min_hash = line.strip
end
- # If not found or string is invalid, nothing else to do
- return [] if !min_hash or !git_check_ref_format(min_hash)
+ # If not found, nothing else to do
+ if !min_hash
+ logger.warn "no refs found looking for min_hash: `GIT_DIR=#{gitdir} #{git_min_hash_cmd}` returned no output"
+ return []
+ end
+
+ # If string is invalid, nothing else to do
+ if !git_check_ref_format(min_hash)
+ logger.warn "ref returned by `GIT_DIR=#{gitdir} #{git_min_hash_cmd}` was invalid for min_hash: #{min_hash}"
+ return []
+ end
# Now find all commits between them
IO.foreach("|git rev-list #{min_hash.shellescape}..#{max_hash.shellescape} --") do |line|
# Do not use this file for site configuration. Create application.yml
# instead (see application.yml.example).
-
-development:
- force_ssl: false
- cache_classes: false
- whiny_nils: true
- consider_all_requests_local: true
- action_controller.perform_caching: false
- action_mailer.raise_delivery_errors: false
- action_mailer.perform_deliveries: false
- active_support.deprecation: :log
- action_dispatch.best_standards_support: :builtin
- active_record.mass_assignment_sanitizer: :strict
- active_record.auto_explain_threshold_in_seconds: 0.5
- assets.compress: false
- assets.debug: true
- local_modified: "<%= '-modified' if `git status -s` != '' %>"
-
-production:
- force_ssl: true
- cache_classes: true
- consider_all_requests_local: false
- action_controller.perform_caching: true
- serve_static_assets: false
- assets.compress: true
- assets.compile: false
- assets.digest: true
-
-test:
- force_ssl: false
- cache_classes: true
- serve_static_assets: true
- static_cache_control: public, max-age=3600
- whiny_nils: true
- consider_all_requests_local: true
- action_controller.perform_caching: false
- action_dispatch.show_exceptions: false
- action_controller.allow_forgery_protection: false
- action_mailer.delivery_method: :test
- active_support.deprecation: :stderr
- active_record.mass_assignment_sanitizer: :strict
- uuid_prefix: zzzzz
- sso_app_id: arvados-server
- sso_app_secret: <%= rand(2**512).to_s(36) %>
- sso_provider_url: http://localhost:3002
- secret_token: <%= rand(2**512).to_s(36) %>
- blob_signing_key: zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc
- user_profile_notification_address: arvados@example.com
- workbench_address: https://localhost:3001/
- git_repositories_dir: <%= Rails.root.join 'tmp', 'git', 'test' %>
- git_internal_dir: <%= Rails.root.join 'tmp', 'internal.git' %>
+#
+# The order of precedence is:
+# 1. config/environments/{RAILS_ENV}.rb (deprecated)
+# 2. Section in application.yml corresponding to RAILS_ENV (e.g., development)
+# 3. Section in application.yml called "common"
+# 4. Section in application.default.yml corresponding to RAILS_ENV
+# 5. Section in application.default.yml called "common"
common:
+ ###
+ ### Essential site configuration
+ ###
+
# The prefix used for all database identifiers to identify the record as
# originating from this site. Must be exactly 5 alphanumeric characters
# (lowercase ASCII letters and digits).
- uuid_prefix: <%= Digest::MD5.hexdigest(`hostname`).to_i(16).to_s(36)[0..4] %>
+ uuid_prefix: ~
- # If not false, this is the hostname that will be used for root_url and
- # advertised in the discovery document. By default, use the default Rails
- # logic for deciding on a hostname.
- host: false
+ # secret_token is a string of alphanumeric characters used by Rails
+ # to sign session tokens. IMPORTANT: This is a site secret. It
+ # should be at least 50 characters.
+ secret_token: ~
- # Base part of SSH git clone url given with repository resources. If
- # true, the default "git@git.(uuid_prefix).arvadosapi.com:" is
- # used. If false, SSH clone URLs are not advertised. Include a
- # trailing ":" or "/" if needed: it will not be added automatically.
- git_repo_ssh_base: true
+ # blob_signing_key is a string of alphanumeric characters used to
+ # generate permission signatures for Keep locators. It must be
+ # identical to the permission key given to Keep. IMPORTANT: This is
+ # a site secret. It should be at least 50 characters.
+ blob_signing_key: ~
- # Base part of HTTPS git clone urls given with repository
- # resources. This is expected to be an arv-git-httpd service which
- # accepts API tokens as HTTP-auth passwords. If true, the default
- # "https://git.(uuid_prefix).arvadosapi.com/" is used. If false,
- # HTTPS clone URLs are not advertised. Include a trailing ":" or "/"
- # if needed: it will not be added automatically.
- git_repo_https_base: true
+ # These settings are provided by your OAuth2 provider (e.g.,
+ # sso-provider).
+ sso_app_secret: ~
+ sso_app_id: ~
+ sso_provider_url: ~
# If this is not false, HTML requests at the API server's root URL
# are redirected to this location, and it is provided in the text of
# to log in.
workbench_address: false
+ # The ARVADOS_WEBSOCKETS environment variable determines whether to
+ # serve http, websockets, or both.
+ #
+ # If ARVADOS_WEBSOCKETS="true", http and websockets are both served
+ # from the same process.
+ #
+ # If ARVADOS_WEBSOCKETS="ws-only", only websockets is served.
+ #
+ # If ARVADOS_WEBSOCKETS="false" or not set at all, only http is
+ # served. In this case, you should have a separate process serving
+ # websockets, and the address of that service should be given here
+ # as websocket_address.
+ #
+ # If websocket_address is false (which is the default), the
+ # discovery document will tell clients to use the current server as
+ # the websocket service, or (if the current server does not have
+ # websockets enabled) not to use websockets at all.
+ #
+ # Example: Clients will connect to the specified endpoint.
+ #websocket_address: wss://127.0.0.1:3333/websocket
+ # Default: Clients will connect to this server if it's running
+ # websockets, otherwise none at all.
+ websocket_address: false
+
# Git repositories must be readable by api server, or you won't be
# able to submit crunch jobs. To pass the test suites, put a clone
# of the arvados tree in {git_repositories_dir}/arvados.git or
# subdirectory of {git_repositiories_dir}.
git_internal_dir: /var/lib/arvados/internal.git
- # :none or :slurm_immediate
- crunch_job_wrapper: :none
-
- # username, or false = do not set uid when running jobs.
- crunch_job_user: crunch
-
- # The web service must be able to create/write this file, and
- # crunch-job must be able to stat() it.
- crunch_refresh_trigger: /tmp/crunch_refresh_trigger
-
- # These two settings control how frequently log events are flushed to the
- # database. Log lines are buffered until either crunch_log_bytes_per_event
- # has been reached or crunch_log_seconds_between_events has elapsed since
- # the last flush.
- crunch_log_bytes_per_event: 4096
- crunch_log_seconds_between_events: 1
-
- # The sample period for throttling logs, in seconds.
- crunch_log_throttle_period: 60
-
- # Maximum number of bytes that job can log over crunch_log_throttle_period
- # before being silenced until the end of the period.
- crunch_log_throttle_bytes: 65536
-
- # Maximum number of lines that job can log over crunch_log_throttle_period
- # before being silenced until the end of the period.
- crunch_log_throttle_lines: 1024
-
- # Maximum bytes that may be logged by a single job. Log bytes that are
- # silenced by throttling are not counted against this total.
- crunch_limit_log_bytes_per_job: 67108864
+ # Default replication level for collections. This is used when a
+ # collection's replication_desired attribute is nil.
+ default_collection_replication: 2
- # Path to dns server configuration directory
- # (e.g. /etc/unbound.d/conf.d). If false, do not write any config
- # files or touch restart.txt (see below).
- dns_server_conf_dir: false
- # Template file for the dns server host snippets. See
- # unbound.template in this directory for an example. If false, do
- # not write any config files.
- dns_server_conf_template: false
+ ###
+ ### Overriding default advertised hostnames/URLs
+ ###
- # String to write to {dns_server_conf_dir}/restart.txt (with a
- # trailing newline) after updating local data. If false, do not
- # open or write the restart.txt file.
- dns_server_reload_command: false
+ # If not false, this is the hostname that will be used for root_url and
+ # advertised in the discovery document. By default, use the default Rails
+ # logic for deciding on a hostname.
+ host: false
- # Command to run after each DNS update. Template variables will be
- # substituted; see the "unbound" example below. If false, do not run
- # a command.
- dns_server_update_command: false
+ # Base part of SSH git clone url given with repository resources. If
+ # true, the default "git@git.(uuid_prefix).arvadosapi.com:" is
+ # used. If false, SSH clone URLs are not advertised. Include a
+ # trailing ":" or "/" if needed: it will not be added automatically.
+ git_repo_ssh_base: true
- ## Example for unbound:
- #dns_server_conf_dir: /etc/unbound/conf.d
- #dns_server_conf_template: /path/to/your/api/server/config/unbound.template
- ## ...plus one of the following two methods of reloading:
- #dns_server_reload_command: unbound-control reload
- #dns_server_update_command: echo %{hostname} %{hostname}.%{uuid_prefix} %{hostname}.%{uuid_prefix}.arvadosapi.com %{ptr_domain} | xargs -n 1 unbound-control local_data_remove && unbound-control local_data %{hostname} IN A %{ip_address} && unbound-control local_data %{hostname}.%{uuid_prefix} IN A %{ip_address} && unbound-control local_data %{hostname}.%{uuid_prefix}.arvadosapi.com IN A %{ip_address} && unbound-control local_data %{ptr_domain}. IN PTR %{hostname}.%{uuid_prefix}.arvadosapi.com
+ # Base part of HTTPS git clone urls given with repository
+ # resources. This is expected to be an arv-git-httpd service which
+ # accepts API tokens as HTTP-auth passwords. If true, the default
+ # "https://git.(uuid_prefix).arvadosapi.com/" is used. If false,
+ # HTTPS clone URLs are not advertised. Include a trailing ":" or "/"
+ # if needed: it will not be added automatically.
+ git_repo_https_base: true
- compute_node_domain: false
- compute_node_nameservers:
- - 192.168.1.1
- # The version below is suitable for AWS.
- # To use it, copy it to your application.yml, uncomment, and change <%# to <%=
- # compute_node_nameservers: <%#
- # require 'net/http'
- # ['local', 'public'].collect do |iface|
- # Net::HTTP.get(URI("http://169.254.169.254/latest/meta-data/#{iface}-ipv4")).match(/^[\d\.]+$/)[0]
- # end << '172.16.0.23'
- # %>
+ ###
+ ### New user and & email settings
+ ###
- accept_api_token: {}
+ # Config parameters to automatically setup new users.
+ # The params auto_setup_new_users_with_* are meaningful only when auto_setup_new_users is turned on.
+ # auto_setup_name_blacklist is a list of usernames to be blacklisted for auto setup.
+ auto_setup_new_users: false
+ auto_setup_new_users_with_vm_uuid: false
+ auto_setup_new_users_with_repository: false
+ auto_setup_name_blacklist: [arvados, git, gitolite, gitolite-admin, root, syslog]
# When new_users_are_active is set to true, the user agreement check is skipped.
new_users_are_active: false
- admin_notifier_email_from: arvados@example.com
- email_subject_prefix: "[ARVADOS] "
- user_notifier_email_from: arvados@example.com
- new_user_notification_recipients: [ ]
- new_inactive_user_notification_recipients: [ ]
-
# The e-mail address of the user you would like to become marked as an admin
# user on their first login.
# In the default configuration, authentication happens through the Arvados SSO
- # server, which uses openid against Google's servers, so in that case this
+ # server, which uses OAuth2 against Google's servers, so in that case this
# should be an address associated with a Google account.
auto_admin_user: false
# other admin users exist will automatically become an admin user.
auto_admin_first_user: false
- ## Set Time.zone default to the specified zone and make Active
- ## Record auto-convert to this zone. Run "rake -D time" for a list
- ## of tasks for finding time zone names. Default is UTC.
- #time_zone: Central Time (US & Canada)
-
- ## Default encoding used in templates for Ruby 1.9.
- encoding: utf-8
-
- # Enable the asset pipeline
- assets.enabled: true
-
- # Version of your assets, change this if you want to expire all your assets
- assets.version: "1.0"
+ # Email address to notify whenever a user creates a profile for the
+ # first time
+ user_profile_notification_address: false
- arvados_theme: default
+ admin_notifier_email_from: arvados@example.com
+ email_subject_prefix: "[ARVADOS] "
+ user_notifier_email_from: arvados@example.com
+ new_user_notification_recipients: [ ]
+ new_inactive_user_notification_recipients: [ ]
- # The ARVADOS_WEBSOCKETS environment variable determines whether to
- # serve http, websockets, or both.
- #
- # If ARVADOS_WEBSOCKETS="true", http and websockets are both served
- # from the same process.
- #
- # If ARVADOS_WEBSOCKETS="ws-only", only websockets is served.
- #
- # If ARVADOS_WEBSOCKETS="false" or not set at all, only http is
- # served. In this case, you should have a separate process serving
- # websockets, and the address of that service should be given here
- # as websocket_address.
- #
- # If websocket_address is false (which is the default), the
- # discovery document will tell clients to use the current server as
- # the websocket service, or (if the current server does not have
- # websockets enabled) not to use websockets at all.
- #
- # Example: Clients will connect to the specified endpoint.
- #websocket_address: wss://127.0.0.1:3333/websocket
- # Default: Clients will connect to this server if it's running
- # websockets, otherwise none at all.
- websocket_address: false
- # blob_signing_key is a string of alphanumeric characters used to
- # generate permission signatures for Keep locators. It must be
- # identical to the permission key given to Keep. IMPORTANT: This is
- # a site secret. It should be at least 50 characters.
- blob_signing_key: ~
+ ###
+ ### Limits, timeouts and durations
+ ###
# Lifetime (in seconds) of blob permission signatures generated by
# the API server. This determines how long a client can take (after
# The default is 2 weeks.
blob_signature_ttl: 1209600
- # Allow clients to create collections by providing a manifest with
- # unsigned data blob locators. IMPORTANT: This effectively disables
- # access controls for data stored in Keep: a client who knows a hash
- # can write a manifest that references the hash, pass it to
- # collections.create (which will create a permission link), use
- # collections.get to obtain a signature for that data locator, and
- # use that signed locator to retrieve the data from Keep. Therefore,
- # do not turn this on if your users expect to keep data private from
- # one another!
- permit_create_collection_with_unsigned_manifest: false
-
- # secret_token is a string of alphanumeric characters used by Rails
- # to sign session tokens. IMPORTANT: This is a site secret. It
- # should be at least 50 characters.
- secret_token: ~
-
- # Email address to notify whenever a user creates a profile for the
- # first time
- user_profile_notification_address: false
-
- default_openid_prefix: https://www.google.com/accounts/o8/id
-
- # Config parameters to automatically setup new users.
- # The params auto_setup_new_users_with_* are meaningful only when auto_setup_new_users is turned on.
- # auto_setup_name_blacklist is a list of usernames to be blacklisted for auto setup.
- auto_setup_new_users: false
- auto_setup_new_users_with_vm_uuid: false
- auto_setup_new_users_with_repository: false
- auto_setup_name_blacklist: [arvados, git, gitolite, gitolite-admin, root, syslog]
-
- # source_version
- source_version: "<%= `git log -n 1 --format=%h`.strip %>"
- local_modified: false
-
# Default lifetime for ephemeral collections: 2 weeks.
default_trash_lifetime: 1209600
- # Permit insecure (OpenSSL::SSL::VERIFY_NONE) connections to the Single Sign
- # On (sso) server. Should only be enabled during development when the SSO
- # server is using a self-signed cert.
- sso_insecure: false
-
- # These settings are provided by your OAuth2 provider (e.g.,
- # sso-provider).
- sso_app_id: ~
- sso_app_secret: ~
- sso_provider_url: ~
-
- # Default replication level for collections. This is used when a
- # collection's replication_desired attribute is nil.
- default_collection_replication: 2
-
# Maximum size (in bytes) allowed for a single API request. This
# limit is published in the discovery document for use by clients.
# Note: You must separately configure the upstream web server or
# go down.
max_compute_nodes: 64
+ # These two settings control how frequently log events are flushed to the
+ # database. Log lines are buffered until either crunch_log_bytes_per_event
+ # has been reached or crunch_log_seconds_between_events has elapsed since
+ # the last flush.
+ crunch_log_bytes_per_event: 4096
+ crunch_log_seconds_between_events: 1
+
+ # The sample period for throttling logs, in seconds.
+ crunch_log_throttle_period: 60
+
+ # Maximum number of bytes that job can log over crunch_log_throttle_period
+ # before being silenced until the end of the period.
+ crunch_log_throttle_bytes: 65536
+
+ # Maximum number of lines that job can log over crunch_log_throttle_period
+ # before being silenced until the end of the period.
+ crunch_log_throttle_lines: 1024
+
+ # Maximum bytes that may be logged by a single job. Log bytes that are
+ # silenced by throttling are not counted against this total.
+ crunch_limit_log_bytes_per_job: 67108864
+
+
+ ###
+ ### Crunch, DNS & compute node management
+ ###
+
# Docker image to be used when none found in runtime_constraints of a job
default_docker_image_for_jobs: false
+ # :none or :slurm_immediate
+ crunch_job_wrapper: :none
+
+ # username, or false = do not set uid when running jobs.
+ crunch_job_user: crunch
+
+ # The web service must be able to create/write this file, and
+ # crunch-job must be able to stat() it.
+ crunch_refresh_trigger: /tmp/crunch_refresh_trigger
+
+ # Path to dns server configuration directory
+ # (e.g. /etc/unbound.d/conf.d). If false, do not write any config
+ # files or touch restart.txt (see below).
+ dns_server_conf_dir: false
+
+ # Template file for the dns server host snippets. See
+ # unbound.template in this directory for an example. If false, do
+ # not write any config files.
+ dns_server_conf_template: false
+
+ # String to write to {dns_server_conf_dir}/restart.txt (with a
+ # trailing newline) after updating local data. If false, do not
+ # open or write the restart.txt file.
+ dns_server_reload_command: false
+
+ # Command to run after each DNS update. Template variables will be
+ # substituted; see the "unbound" example below. If false, do not run
+ # a command.
+ dns_server_update_command: false
+
+ ## Example for unbound:
+ #dns_server_conf_dir: /etc/unbound/conf.d
+ #dns_server_conf_template: /path/to/your/api/server/config/unbound.template
+ ## ...plus one of the following two methods of reloading:
+ #dns_server_reload_command: unbound-control reload
+ #dns_server_update_command: echo %{hostname} %{hostname}.%{uuid_prefix} %{hostname}.%{uuid_prefix}.arvadosapi.com %{ptr_domain} | xargs -n 1 unbound-control local_data_remove && unbound-control local_data %{hostname} IN A %{ip_address} && unbound-control local_data %{hostname}.%{uuid_prefix} IN A %{ip_address} && unbound-control local_data %{hostname}.%{uuid_prefix}.arvadosapi.com IN A %{ip_address} && unbound-control local_data %{ptr_domain}. IN PTR %{hostname}.%{uuid_prefix}.arvadosapi.com
+
+ compute_node_domain: false
+ compute_node_nameservers:
+ - 192.168.1.1
+
# Hostname to assign to a compute node when it sends a "ping" and the
# hostname in its Node record is nil.
# During bootstrapping, the "ping" script is expected to notice the
# assign_node_hostname: compute%<slot_number>04d
# (See http://ruby-doc.org/core-2.2.2/Kernel.html#method-i-format for more.)
assign_node_hostname: compute%<slot_number>d
+
+
+ ###
+ ### Remaining assorted configuration options.
+ ###
+
+ arvados_theme: default
+
+ # Permit insecure (OpenSSL::SSL::VERIFY_NONE) connections to the Single Sign
+ # On (sso) server. Should only be enabled during development when the SSO
+ # server is using a self-signed cert.
+ sso_insecure: false
+
+ ## Set Time.zone default to the specified zone and make Active
+ ## Record auto-convert to this zone. Run "rake -D time" for a list
+ ## of tasks for finding time zone names. Default is UTC.
+ #time_zone: Central Time (US & Canada)
+
+ ## Default encoding used in templates for Ruby 1.9.
+ encoding: utf-8
+
+ # Enable the asset pipeline
+ assets.enabled: true
+
+ # Version of your assets, change this if you want to expire all your assets
+ assets.version: "1.0"
+
+ # Allow clients to create collections by providing a manifest with
+ # unsigned data blob locators. IMPORTANT: This effectively disables
+ # access controls for data stored in Keep: a client who knows a hash
+ # can write a manifest that references the hash, pass it to
+ # collections.create (which will create a permission link), use
+ # collections.get to obtain a signature for that data locator, and
+ # use that signed locator to retrieve the data from Keep. Therefore,
+ # do not turn this on if your users expect to keep data private from
+ # one another!
+ permit_create_collection_with_unsigned_manifest: false
+
+ default_openid_prefix: https://www.google.com/accounts/o8/id
+
+ # source_version
+ source_version: "<%= `git log -n 1 --format=%h`.strip %>"
+ local_modified: false
+
+
+development:
+ force_ssl: false
+ cache_classes: false
+ whiny_nils: true
+ consider_all_requests_local: true
+ action_controller.perform_caching: false
+ action_mailer.raise_delivery_errors: false
+ action_mailer.perform_deliveries: false
+ active_support.deprecation: :log
+ action_dispatch.best_standards_support: :builtin
+ active_record.mass_assignment_sanitizer: :strict
+ active_record.auto_explain_threshold_in_seconds: 0.5
+ assets.compress: false
+ assets.debug: true
+ local_modified: "<%= '-modified' if `git status -s` != '' %>"
+
+production:
+ force_ssl: true
+ cache_classes: true
+ consider_all_requests_local: false
+ action_controller.perform_caching: true
+ serve_static_assets: false
+ assets.compress: true
+ assets.compile: false
+ assets.digest: true
+
+test:
+ force_ssl: false
+ cache_classes: true
+ serve_static_assets: true
+ static_cache_control: public, max-age=3600
+ whiny_nils: true
+ consider_all_requests_local: true
+ action_controller.perform_caching: false
+ action_dispatch.show_exceptions: false
+ action_controller.allow_forgery_protection: false
+ action_mailer.delivery_method: :test
+ active_support.deprecation: :stderr
+ active_record.mass_assignment_sanitizer: :strict
+ uuid_prefix: zzzzz
+ sso_app_id: arvados-server
+ sso_app_secret: <%= rand(2**512).to_s(36) %>
+ sso_provider_url: http://localhost:3002
+ secret_token: <%= rand(2**512).to_s(36) %>
+ blob_signing_key: zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc
+ user_profile_notification_address: arvados@example.com
+ workbench_address: https://localhost:3001/
+ git_repositories_dir: <%= Rails.root.join 'tmp', 'git', 'test' %>
+ git_internal_dir: <%= Rails.root.join 'tmp', 'internal.git' %>
# 4. Section in application.default.yml corresponding to RAILS_ENV
# 5. Section in application.default.yml called "common"
-development:
- # Mandatory site secrets. See application.default.yml for more info.
+production:
+ # Mandatory site configuration. See application.default.yml and
+ # http://http://doc.arvados.org/install/install-api-server.html#configure_application
+ # for more information.
+ uuid_prefix: ~
secret_token: ~
blob_signing_key: ~
- uuid_prefix: bogus
- workbench_address: https://localhost:3031
+ sso_app_secret: ~
+ sso_app_id: ~
+ sso_provider_url: ~
+ workbench_address: ~
+ websockets_address: ~
+ #git_repositories_dir: ~
+ #git_internal_dir: ~
-production:
- # Mandatory site secrets. See application.default.yml for more info.
+development:
+ # Separate settings for development configuration.
+ uuid_prefix: ~
secret_token: ~
blob_signing_key: ~
- uuid_prefix: bogus
- workbench_address: https://workbench.bogus.arvadosapi.com
- sso_app_id: arvados-server
+ sso_app_id: ~
sso_app_secret: ~
- sso_provider_url: https://login.bogus.arvadosapi.com
+ sso_provider_url: ~
+ workbench_address: ~
+ websockets_address: ~
+ #git_repositories_dir: ~
+ #git_internal_dir: ~
test:
# Tests should be able to run without further configuration, but if you do
+++ /dev/null
-Server::Application.configure do
- config.accept_api_token = { 'foobar' => true }
-end
head_uuid: zzzzz-j7d0g-v955i6s2oi1cbso
properties: {}
+project_viewer_member_of_all_users_group:
+ uuid: zzzzz-o0j2j-cdnq6627g0h0r2x
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2015-07-28T21:34:41.361747000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2015-07-28T21:34:41.361747000Z
+ updated_at: 2015-07-28T21:34:41.361747000Z
+ tail_uuid: zzzzz-tpzed-projectviewer1a
+ link_class: permission
+ name: can_read
+ head_uuid: zzzzz-j7d0g-fffffffffffffff
+ properties: {}
+
project_viewer_can_read_project:
uuid: zzzzz-o0j2j-projviewerreadp
owner_uuid: zzzzz-tpzed-000000000000000
end
end
+ test "get_all_permissions takes into account is_active flag" do
+ r = nil
+ act_as_user users(:active) do
+ r = Repository.create! name: 'active/testrepo'
+ end
+ act_as_system_user do
+ u = users(:active)
+ u.is_active = false
+ u.save!
+ end
+ authorize_with :admin
+ get :get_all_permissions
+ assert_response :success
+ json_response['repositories'].each do |r|
+ r['user_permissions'].each do |user_uuid, perms|
+ refute_equal user_uuid, users(:active).uuid
+ end
+ end
+ end
+
test "get_all_permissions does not give any access to user without permission" do
viewer_uuid = users(:project_viewer).uuid
assert_equal(authorized_keys(:project_viewer).authorized_user_uuid,
end
end
- test "get_all_permissions lists repos with no authorized keys" do
+ test "get_all_permissions lists all repos regardless of permissions" do
+ act_as_system_user do
+ # Create repos that could potentially be left out of the
+ # permission list by accident.
+
+ # No authorized_key, no username (this can't even be done
+ # without skipping validations)
+ r = Repository.create name: 'root/testrepo'
+ assert r.save validate: false
+
+ r = Repository.create name: 'invalid username / repo name', owner_uuid: users(:inactive).uuid
+ assert r.save validate: false
+ end
+ authorize_with :admin
+ get :get_all_permissions
+ assert_response :success
+ assert_equal(Repository.count, json_response["repositories"].size)
+ end
+
+ test "get_all_permissions lists user permissions for users with no authorized keys" do
authorize_with :admin
AuthorizedKey.destroy_all
get :get_all_permissions
assert_response :success
assert_equal(Repository.count, json_response["repositories"].size)
- assert(json_response["repositories"].any? do |repo|
- repo["user_permissions"].empty?
- end, "test is invalid - all repositories have authorized keys")
+ repos_with_perms = []
+ json_response['repositories'].each do |repo|
+ if repo['user_permissions'].any?
+ repos_with_perms << repo['uuid']
+ end
+ end
+ assert_not_empty repos_with_perms, 'permissions are missing'
+ end
+
+ # Ensure get_all_permissions correctly describes what the normal
+ # permission system would do.
+ test "get_all_permissions obeys group permissions" do
+ act_as_user system_user do
+ r = Repository.create!(name: 'admin/groupcanwrite', owner_uuid: users(:admin).uuid)
+ g = Group.create!(group_class: 'group', name: 'repo-writers')
+ u1 = users(:active)
+ u2 = users(:spectator)
+ Link.create!(tail_uuid: g.uuid, head_uuid: r.uuid, link_class: 'permission', name: 'can_manage')
+ Link.create!(tail_uuid: u1.uuid, head_uuid: g.uuid, link_class: 'permission', name: 'can_write')
+ Link.create!(tail_uuid: u2.uuid, head_uuid: g.uuid, link_class: 'permission', name: 'can_read')
+
+ r = Repository.create!(name: 'admin/groupreadonly', owner_uuid: users(:admin).uuid)
+ g = Group.create!(group_class: 'group', name: 'repo-readers')
+ u1 = users(:active)
+ u2 = users(:spectator)
+ Link.create!(tail_uuid: g.uuid, head_uuid: r.uuid, link_class: 'permission', name: 'can_read')
+ Link.create!(tail_uuid: u1.uuid, head_uuid: g.uuid, link_class: 'permission', name: 'can_write')
+ Link.create!(tail_uuid: u2.uuid, head_uuid: g.uuid, link_class: 'permission', name: 'can_read')
+ end
+ authorize_with :admin
+ get :get_all_permissions
+ assert_response :success
+ json_response['repositories'].each do |repo|
+ repo['user_permissions'].each do |user_uuid, perms|
+ u = User.find_by_uuid(user_uuid)
+ if perms['can_read']
+ assert u.can? read: repo['uuid']
+ assert_match /R/, perms['gitolite_permissions']
+ else
+ refute_match /R/, perms['gitolite_permissions']
+ end
+ if perms['can_write']
+ assert u.can? write: repo['uuid']
+ assert_match /RW/, perms['gitolite_permissions']
+ else
+ refute_match /W/, perms['gitolite_permissions']
+ end
+ if perms['can_manage']
+ assert u.can? manage: repo['uuid']
+ assert_match /RW/, perms['gitolite_permissions']
+ end
+ end
+ end
end
test "default index includes fetch_url" do
require 'test_helper'
class AuthorizedKeyTest < ActiveSupport::TestCase
- # test "the truth" do
- # assert true
- # end
+ TEST_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCf5aTI55uyWr44TckP/ELUAyPsdnf5fTZDcSDN4qiMZYAL7TYV2ixwnbPObLObM0GmHSSFLV1KqsuFICUPgkyKoHbAH6XPgmtfOLU60VkGf1v5uxQ/kXCECRCJmPb3K9dIXGEw+1DXPdOV/xG7rJNvo4a9WK9iqqZr8p+VGKM6C017b8BDLk0tuEEjZ5jXcT/ka/hTScxWkKgF6auPOVQ79OA5+0VaYm4uQLzVUdgwVUPWQQecRrtnc08XYM1htpcLDIAbWfUNK7uE6XR3/OhtrJGf05FGbtGguPgi33F9W3Q3yw6saOK5Y3TfLbskgFaEdLgzqK/QSBRk2zBF49Tj test@localhost"
+
+ test 'create and update key' do
+ u1 = users(:active)
+ act_as_user u1 do
+ ak = AuthorizedKey.new(name: "foo", public_key: TEST_KEY, authorized_user_uuid: u1.uuid)
+ assert ak.save, ak.errors.full_messages.to_s
+ ak.name = "bar"
+ assert ak.valid?, ak.errors.full_messages.to_s
+ assert ak.save, ak.errors.full_messages.to_s
+ end
+ end
+
+ test 'duplicate key not permitted' do
+ u1 = users(:active)
+ act_as_user u1 do
+ ak = AuthorizedKey.new(name: "foo", public_key: TEST_KEY, authorized_user_uuid: u1.uuid)
+ assert ak.save
+ end
+ u2 = users(:spectator)
+ act_as_user u2 do
+ ak2 = AuthorizedKey.new(name: "bar", public_key: TEST_KEY, authorized_user_uuid: u2.uuid)
+ refute ak2.valid?
+ refute ak2.save
+ assert_match /already exists/, ak2.errors.full_messages.to_s
+ end
+ end
+
+ test 'attach key to wrong user account' do
+ act_as_user users(:active) do
+ ak = AuthorizedKey.new(name: "foo", public_key: TEST_KEY)
+ ak.authorized_user_uuid = users(:spectator).uuid
+ refute ak.save
+ ak.uuid = nil
+ ak.authorized_user_uuid = users(:admin).uuid
+ refute ak.save
+ ak.uuid = nil
+ ak.authorized_user_uuid = users(:active).uuid
+ assert ak.save, ak.errors.full_messages.to_s
+ ak.authorized_user_uuid = users(:admin).uuid
+ refute ak.save
+ end
+ end
end
projects / arvados.git / commitdiff