remote: https://rubygems.org/
specs:
RedCloth (4.3.2)
- actioncable (5.2.8)
- actionpack (= 5.2.8)
+ actioncable (5.2.8.1)
+ actionpack (= 5.2.8.1)
nio4r (~> 2.0)
websocket-driver (>= 0.6.1)
- actionmailer (5.2.8)
- actionpack (= 5.2.8)
- actionview (= 5.2.8)
- activejob (= 5.2.8)
+ actionmailer (5.2.8.1)
+ actionpack (= 5.2.8.1)
+ actionview (= 5.2.8.1)
+ activejob (= 5.2.8.1)
mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 2.0)
- actionpack (5.2.8)
- actionview (= 5.2.8)
- activesupport (= 5.2.8)
+ actionpack (5.2.8.1)
+ actionview (= 5.2.8.1)
+ activesupport (= 5.2.8.1)
rack (~> 2.0, >= 2.0.8)
rack-test (>= 0.6.3)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.2)
- actionview (5.2.8)
- activesupport (= 5.2.8)
+ actionview (5.2.8.1)
+ activesupport (= 5.2.8.1)
builder (~> 3.1)
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.3)
- activejob (5.2.8)
- activesupport (= 5.2.8)
+ activejob (5.2.8.1)
+ activesupport (= 5.2.8.1)
globalid (>= 0.3.6)
- activemodel (5.2.8)
- activesupport (= 5.2.8)
- activerecord (5.2.8)
- activemodel (= 5.2.8)
- activesupport (= 5.2.8)
+ activemodel (5.2.8.1)
+ activesupport (= 5.2.8.1)
+ activerecord (5.2.8.1)
+ activemodel (= 5.2.8.1)
+ activesupport (= 5.2.8.1)
arel (>= 9.0)
- activestorage (5.2.8)
- actionpack (= 5.2.8)
- activerecord (= 5.2.8)
+ activestorage (5.2.8.1)
+ actionpack (= 5.2.8.1)
+ activerecord (= 5.2.8.1)
marcel (~> 1.0.0)
- activesupport (5.2.8)
+ activesupport (5.2.8.1)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 0.7, < 2)
minitest (~> 5.1)
net-ssh-gateway (2.0.0)
net-ssh (>= 4.0.0)
nio4r (2.5.8)
- nokogiri (1.13.6)
+ nokogiri (1.13.7)
mini_portile2 (~> 2.8.0)
racc (~> 1.4)
npm-rails (0.2.1)
websocket-driver (>= 0.2.0)
public_suffix (4.0.6)
racc (1.6.0)
- rack (2.2.3.1)
+ rack (2.2.4)
rack-mini-profiler (1.0.2)
rack (>= 1.2.0)
- rack-test (1.1.0)
- rack (>= 1.0, < 3)
- rails (5.2.8)
- actioncable (= 5.2.8)
- actionmailer (= 5.2.8)
- actionpack (= 5.2.8)
- actionview (= 5.2.8)
- activejob (= 5.2.8)
- activemodel (= 5.2.8)
- activerecord (= 5.2.8)
- activestorage (= 5.2.8)
- activesupport (= 5.2.8)
+ rack-test (2.0.2)
+ rack (>= 1.3)
+ rails (5.2.8.1)
+ actioncable (= 5.2.8.1)
+ actionmailer (= 5.2.8.1)
+ actionpack (= 5.2.8.1)
+ actionview (= 5.2.8.1)
+ activejob (= 5.2.8.1)
+ activemodel (= 5.2.8.1)
+ activerecord (= 5.2.8.1)
+ activestorage (= 5.2.8.1)
+ activesupport (= 5.2.8.1)
bundler (>= 1.3.0)
- railties (= 5.2.8)
+ railties (= 5.2.8.1)
sprockets-rails (>= 2.0.0)
rails-controller-testing (1.0.4)
actionpack (>= 5.0.1.x)
rails-html-sanitizer (1.4.3)
loofah (~> 2.3)
rails-perftest (0.0.7)
- railties (5.2.8)
- actionpack (= 5.2.8)
- activesupport (= 5.2.8)
+ railties (5.2.8.1)
+ actionpack (= 5.2.8.1)
+ activesupport (= 5.2.8.1)
method_source
rake (>= 0.8.7)
thor (>= 0.19.0, < 2.0)
thor (1.2.1)
thread_safe (0.3.6)
tilt (2.0.9)
- tzinfo (1.2.9)
+ tzinfo (1.2.10)
thread_safe (~> 0.1)
uglifier (2.7.2)
execjs (>= 0.3.0)
<div class="col-md-6">
<p>
- This page enables you to <a href="https://doc.arvados.org/master/admin/user-management.html">manage users</a>.
+ This page enables you to <a href="https://doc.arvados.org/main/admin/user-management.html">manage users</a>.
</p>
<p>
As an admin, you can deactivate and reset this user. This will
remove all repository/VM permissions for the user. If you
"setup" the user again, the user will have to sign the user
- agreement again. You may also want to <a href="https://doc.arvados.org/master/admin/reassign-ownership.html">reassign data ownership</a>.
+ agreement again. You may also want to <a href="https://doc.arvados.org/main/admin/reassign-ownership.html">reassign data ownership</a>.
</p>
<%= button_to "Deactivate #{@object.full_name}", unsetup_user_url(id: @object.uuid), class: 'btn btn-primary', data: {confirm: "Are you sure you want to deactivate #{@object.full_name}?"} %>
arvados-server
arvados-src
arvados-sync-groups
+ arvados-sync-users
arvados-workbench
arvados-workbench2
arvados-ws
"Arvados Websocket server"
package_go_binary tools/sync-groups arvados-sync-groups "$FORMAT" "$ARCH" \
"Synchronize remote groups into Arvados from an external source"
+package_go_binary tools/sync-users arvados-sync-users "$FORMAT" "$ARCH" \
+ "Synchronize remote users into Arvados from an external source"
package_go_binary tools/keep-block-check keep-block-check "$FORMAT" "$ARCH" \
"Verify that all data from one set of Keep servers to another was copied"
package_go_binary tools/keep-rsync keep-rsync "$FORMAT" "$ARCH" \
"shell": shellCommand{},
"connect-ssh": connectSSHCommand{},
"diagnostics": diagnostics.Command{},
+ "sudo": sudoCommand{},
})
)
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "flag"
+ "fmt"
+ "io"
+ "os"
+
+ "git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+)
+
+// sudoCommand runs another command using API connection info and
+// SystemRootToken from the system config file instead of the caller's
+// environment vars.
+type sudoCommand struct{}
+
+func (sudoCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ ldr := config.NewLoader(stdin, ctxlog.New(stderr, "text", "info"))
+ flags := flag.NewFlagSet(prog, flag.ContinueOnError)
+ ldr.SetupFlags(flags)
+ if ok, code := cmd.ParseFlags(flags, prog, args, "subcommand ...", stderr); !ok {
+ return code
+ }
+ cfg, err := ldr.Load()
+ if err != nil {
+ fmt.Fprintln(stderr, err)
+ return 1
+ }
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ fmt.Fprintln(stderr, err)
+ return 1
+ }
+ os.Setenv("ARVADOS_API_HOST", cluster.Services.Controller.ExternalURL.Host)
+ os.Setenv("ARVADOS_API_TOKEN", cluster.SystemRootToken)
+ if cluster.TLS.Insecure {
+ os.Setenv("ARVADOS_API_HOST_INSECURE", "1")
+ } else {
+ os.Unsetenv("ARVADOS_API_HOST_INSECURE")
+ }
+ return handler.RunCommand(prog, flags.Args(), stdin, stdout, stderr)
+}
return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
}
- cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-document", "fpm")
- cmd.Stdout = stdout
- cmd.Stderr = stderr
- err := cmd.Run()
- if err != nil {
- return fmt.Errorf("gem install fpm: %w", err)
- }
-
- cmd = exec.Command("/var/lib/arvados/bin/gem", "env", "gempath")
+ cmd := exec.Command("/var/lib/arvados/bin/gem", "env", "gempath")
cmd.Stderr = stderr
buf, err := cmd.Output() // /root/.gem/ruby/2.7.0:...
if err != nil || len(buf) == 0 {
}
gempath := string(bytes.TrimRight(bytes.Split(buf, []byte{':'})[0], "\n"))
+ cmd = exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-document", "fpm")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ // Avoid "WARNING: You don't have [...] in your PATH, gem
+ // executables will not run"
+ cmd.Env = append(os.Environ(), "PATH="+os.Getenv("PATH")+":"+gempath)
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("gem install fpm: %w", err)
+ }
+
if _, err := os.Stat(gempath + "/gems/fpm-1.11.0/lib/fpm/package/deb.rb"); err == nil {
// Workaround for fpm bug https://github.com/jordansissel/fpm/issues/1739
cmd = exec.Command("sed", "-i", `/require "digest"/a require "zlib"`, gempath+"/gems/fpm-1.11.0/lib/fpm/package/deb.rb")
"--verbose",
"--deb-use-file-permissions",
"--rpm-use-file-permissions",
- "/etc/systemd/system/multi-user.target.wants/arvados.service",
- "/lib/systemd/system/arvados.service",
+ "--deb-systemd", "/lib/systemd/system/arvados.service",
+ "--deb-systemd-enable",
+ "--no-deb-systemd-auto-start",
+ "--no-deb-systemd-restart-after-upgrade",
+ "--deb-suggests", "postgresql",
+ "--deb-suggests", "docker.io",
"/usr/bin/arvados-client",
"/usr/bin/arvados-server",
+ "/usr/bin/arv",
+ "/usr/bin/arv-ruby",
+ "/usr/bin/arv-tag",
"/var/lib/arvados",
+ "/usr/bin/arv-copy",
+ "/usr/bin/arv-federation-migrate",
+ "/usr/bin/arv-get",
+ "/usr/bin/arv-keepdocker",
+ "/usr/bin/arv-ls",
+ "/usr/bin/arv-migrate-docker19",
+ "/usr/bin/arv-normalize",
+ "/usr/bin/arv-put",
+ "/usr/bin/arv-ws",
+ "/usr/bin/arv-mount",
"/var/www/.gem",
"/var/www/.passenger",
"/var/www/.bundle",
if opts.Live != "" {
cmd.Args = append(cmd.Args,
"--env=domain="+opts.Live,
+ "--env=initargs=-tls=/var/lib/acme/live/"+opts.Live,
"--env=bootargs=",
"--publish=:443:443",
"--publish=:4440-4460:4440-4460",
} else {
cmd.Args = append(cmd.Args,
"--env=domain=localhost",
+ "--env=initargs=-tls=insecure",
"--env=bootargs=-shutdown")
}
cmd.Args = append(cmd.Args,
SUDO_FORCE_REMOVE=yes apt-get autoremove -y
/etc/init.d/postgresql start
-arvados-server init -cluster-id x1234 -domain=$domain -login=test -insecure
-exec arvados-server boot -listen-host=0.0.0.0 -no-workbench2=false $bootargs
+arvados-server init -cluster-id x1234 -domain=$domain -login=test -start=false $initargs
+exec arvados-server boot -listen-host=0.0.0.0 $bootargs
`)
cmd.Stdout = stdout
cmd.Stderr = stderr
"io"
"net/http"
"os"
+ "path"
+ "path/filepath"
+ "strings"
"git.arvados.org/arvados.git/lib/boot"
"git.arvados.org/arvados.git/lib/cloud/cloudtest"
fmt.Fprintf(stderr, "json.Marshal: %s\n", err)
return 1
}
+ servefs := http.FileServer(http.Dir(args[2]))
mux := http.NewServeMux()
- mux.Handle("/", http.FileServer(http.Dir(args[2])))
+ mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ for _, ent := range strings.Split(req.URL.Path, "/") {
+ if ent == ".." {
+ http.Error(w, "invalid URL path", http.StatusBadRequest)
+ return
+ }
+ }
+ fnm := filepath.Join(args[2], filepath.FromSlash(path.Clean("/"+req.URL.Path)))
+ if _, err := os.Stat(fnm); os.IsNotExist(err) {
+ req.URL.Path = "/"
+ }
+ servefs.ServeHTTP(w, req)
+ }))
mux.HandleFunc("/config.json", func(w http.ResponseWriter, _ *http.Request) {
w.Write(configJSON)
})
- admin/federation.html.textile.liquid
- admin/merge-remote-account.html.textile.liquid
- admin/migrating-providers.html.textile.liquid
- - user/topics/arvados-sync-groups.html.textile.liquid
+ - user/topics/arvados-sync-external-sources.html.textile.liquid
- admin/scoped-tokens.html.textile.liquid
- admin/token-expiration-policy.html.textile.liquid
- admin/user-activity.html.textile.liquid
The @ExternalURL@ is the address where the service should be reachable by clients, both from inside and from outside the Arvados cluster. Some services do not expose an Arvados API, only Prometheus metrics. In that case, @ExternalURL@ is not used.
-The keys under @InternalURLs@ are addresses that are used by the reverse proxy (e.g. Nginx) that fronts Arvados services. The exception is the @Keepstore@ service, where clients connect directly to the addresses listed under @InternalURLs@. If a service is not fronted by a reverse proxy, e.g. when its endpoint only exposes Prometheus metrics, the intention is that metrics are collected directly from the endpoints defined in @InternalURLs@.
+The keys under @InternalURLs@ are the URLs through which Arvados system components can connect to one another, including the reverse proxy (e.g. Nginx) that fronts Arvados services. The exception is the @Keepstore@ service, where clients on the local network connect directly to @Keepstore.InternalURLs@ (while clients from outside networks connect to @Keepproxy.ExternalURL@). If a service is not fronted by a reverse proxy, e.g. when its endpoint only exposes Prometheus metrics, the intention is that metrics are collected directly from the endpoints defined in @InternalURLs@.
-@InternalURLs@ are also used by the service itself to figure out which address/port to listen on.
+Each entry in the @InternalURLs@ section may also indicate a @ListenURL@ to determine the protocol, address/interface, and port where the service process will listen, in case the desired listening address differs from the @InternalURLs@ key itself -- for example, when passing internal traffic through a reverse proxy.
If the Arvados service lives behind a reverse proxy (e.g. Nginx), configuring the reverse proxy and the @InternalURLs@ and @ExternalURL@ values must be done in concert.
{% codeblock as yaml %}
Controller:
InternalURLs:
- "http://localhost:8003": {}
+ "https://ctrl-0.internal":
+ ListenURL: "http://localhost:8003"
ExternalURL: "https://ClusterID.example.com"
{% endcodeblock %}
-The @ExternalURL@ advertised is @https://ClusterID.example.com@. The @Controller@ service will start up on @localhost@ port 8003. Nginx is configured to sit in front of the @Controller@ service and terminates SSL:
+The @ExternalURL@ advertised to clients is @https://ClusterID.example.com@. The @arvados-controller@ process will listen on @localhost@ port 8003. Other Arvados service processes in the cluster can connect to this specific controller instance, using the URL @https://ctrl-0.internal@. Nginx is configured to sit in front of the @Controller@ service and terminate TLS:
<notextile><pre><code>
# This is the port where nginx expects to contact arvados-controller.
# the request is reverse proxied to the upstream 'controller'
listen 443 ssl;
- server_name ClusterID.example.com;
+ server_name ClusterID.example.com ctrl-0.internal;
ssl_certificate /YOUR/PATH/TO/cert.pem;
ssl_certificate_key /YOUR/PATH/TO/cert.key;
}
</code></pre></notextile>
+If the host part of @ListenURL@ is ambiguous, in the sense that more than one system host is able to listen on that address (e.g., @localhost@), configure each host's startup scripts to set the environment variable @ARVADOS_SERVICE_INTERNAL_URL@ to the @InternalURLs@ key that will reach that host. In the example above, this would be @ARVADOS_SERVICE_INTERNAL_URL=https://ctrl-0.internal@.
+
+If the cluster has just a single node running all of the Arvados server processes, configuration can be simplified:
+{% codeblock as yaml %}
+ Controller:
+ InternalURLs:
+ "http://localhost:8003": {}
+ ExternalURL: "https://ClusterID.example.com"
+{% endcodeblock %}
Bucket versioning is presently not supported, so this will always respond that bucket versioning is not enabled.
+h3. Accessing collection/project properties as metadata
+
+GetObject, HeadObject, and HeadBucket return Arvados object properties as S3 metadata headers, e.g., @X-Amz-Meta-Foo: bar@.
+
+If the requested path indicates a file or directory placeholder inside a collection, or the top level of a collection, GetObject and HeadObject return the collection properties.
+
+If the requested path indicates a directory placeholder corresponding to a project, GetObject and HeadObject return the properties of the project.
+
+HeadBucket returns the properties of the collection or project corresponding to the bucket name.
+
+Non-string property values are returned in a JSON representation, e.g., @["foo","bar"]@.
+
+As in Amazon S3, property values containing non-ASCII characters are returned in BASE64-encoded form as described in RFC 2047, e.g., @=?UTF-8?b?4pu1?=@.
+
+GetBucketTagging and GetObjectTagging APIs are _not_ supported.
+
+It is not possible to modify collection or project properties using the S3 API.
+
h3. Authorization mechanisms
Keep-web accepts AWS Signature Version 4 (AWS4-HMAC-SHA256) as well as the older V2 AWS signature.
h2. Prerequisites
You will need:
-* a server host running Debian 10 (buster).
+* a server host running Debian 10 (buster) or Debian 11 (bullseye).
* a unique 5-character ID like @x9999@ for your cluster (first character should be @[a-w]@ for a long-lived / production cluster; all characters are @[a-z0-9]@).
-* a DNS name like @x9999.example.com@ that resolves to your server host (or a load balancer / proxy that passes HTTP and HTTPS requests through to your server host).
-* a Google account (use it in place of <code>example@gmail.com.example</code> in the instructions below).
+* a DNS name like @x9999.example.com@ that resolves to your server host (or a load balancer / proxy that passes HTTP requests on port 80[1] and HTTPS requests on ports 443 and 4440-4460 through to the same port on your server host).
+* a firewall setup that allows incoming connections to ports 80[1], 443, and 4440-4460.
+
+fn1. Port 80 is only used to obtain TLS certificates automatically from Let's Encrypt. It is not needed if you have another way to provision certificates.
+
+h2. Options
+
+Arvados needs a PostgreSQL database. To get started quickly, install the postgresql-server package on your server host.
+
+<pre>
+# apt install postgresql
+</pre>
+
+Arvados normally uses cloud VMs or a Slurm/LSF cluster to run containers. To get started quickly, install Docker on your system host. The @arvados-server init@ command, as shown below, will configure Arvados to run containers on the system host.
+
+<pre>
+# apt install docker.io
+</pre>
+
+Arvados needs a login backend. To get started quickly, add a user account on your server host and assign a password. The @arvados-server init ... -login pam@ option, as shown below, will configure Arvados so you can log in with this username and password.
+
+<pre>
+# adduser exampleUserName
+</pre>
h2. Initialize the cluster
<pre>
-# echo > /etc/apt/sources.list.d/arvados.list "deb http://apt.arvados.org/buster buster main"
-# apt-get update
-# apt-get install arvados-server-easy
-# arvados-server init -type production -cluster-id x9999 -controller-address x9999.example.com -admin-email example@gmail.com.example
+# echo > /etc/apt/sources.list.d/arvados.list "deb http://apt.arvados.org/$(lsb_release -sc) $(lsb_release -sc) main"
+# apt update
+# apt install arvados-server-easy
+# arvados-server init -cluster-id x9999 -domain x9999.example.com -tls acme -login pam
</pre>
-When the "init" command is finished, navigate to the link shown in the terminal (e.g., @https://x9999.example.com/?api_token=zzzzzzzzzzzzzzzzzzzzzz@). This will log you in to your admin account.
+When the "init" command is finished, navigate to the link shown in the terminal (e.g., @https://x9999.example.com/@) and log in with the account you created above.
-h2. Enable login
+Activate your new Arvados user account. Copy the UUID (looks like @x9999-tpzed-xxxxxxxxxxxxxxx@) from your browser's location bar and run:
-Follow the instructions to "set up Google login":{{site.baseurl}}/install/setup-login.html or another authentication option.
+<pre>
+# arv sudo user setup --uuid x9999-tpzed-xxxxxxxxxxxxxxx
+</pre>
+
+Run the diagnostics tool to ensure everything is working.
+
+<pre>
+# arv sudo diagnostics
+</pre>
+
+h2. Customize the cluster
+
+Things you should plan to update before using your cluster in production:
+* "Set up Google login":{{site.baseurl}}/install/setup-login.html or another authentication option.
+* "Set up a wildcard TLS certificate and DNS name,":{{site.baseurl}}/install/install-manual-prerequisites.html#dnstls or enable @TrustAllContent@ mode.
+* Update storage configuration to use a cloud storage bucket ("S3":{{site.baseurl}}/install/configure-s3-object-storage.html or "Azure":{{site.baseurl}}/install/configure-azure-blob-storage.html) instead of the local filesystem.
+* Update "CloudVMs configuration":{{site.baseurl}}/install/crunch2-cloud/install-dispatch-cloud.html to use a cloud provider to bring up VMs on demand instead of running containers on the server host.
+
+h2. Updating configuration
+
+After updating your configuration file (@/etc/arvados/config.yml@), notify the server:
+
+<pre>
+# systemctl reload arvados-server
+</pre>
-After updating your configuration file (@/etc/arvados/config.yml@), restart the server to make your changes take effect:
+Optionally, add "AutoReloadConfig: true" at the top of @/etc/arvados/config.yml@. Arvados will automatically reload the config file when it changes.
<pre>
-# systemctl restart arvados-server
+AutoReloadConfig: true
+Clusters:
+ [...]
</pre>
---
layout: default
navsection: admin
-title: "Synchronizing external groups"
+title: "Synchronizing from external sources"
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-The @arvados-sync-groups@ tool allows to synchronize groups in Arvados from an external source.
+The @arvados-sync-users@ and @arvados-sync-groups@ tools allow to manage Arvados users & groups from external sources.
+
+These tools are designed to be run periodically reading a file created by a remote auth system (ie: LDAP) dump script, applying what's included on the file as the source of truth.
+
+bq. NOTE: Both tools need to perform several administrative tasks on Arvados, so must be run using a superuser token via @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ environment variables or @~/.config/arvados/settings.conf@ file.
+
+h1. Using arvados-sync-users
+
+This tool reads a CSV (comma-separated values) file having information about user accounts and their expected state on Arvados.
+
+Every line on the file should have 5 fields:
+
+# A user identifier: it could be an email address (default) or a username.
+# The user's first name.
+# The user's last name.
+# The intended user's active state.
+# The intended user's admin state: will always be read as @false@ when @active=false@.
+
+The last 2 fields should be represented as @true@/@false@, @yes@/@no@, or @1@/@0@ values.
+
+h2. Options
+
+The following command line options are supported:
+
+table(table table-bordered table-condensed).
+|_. Option |_. Description |
+|==--help==|This list of options|
+|==--case-insensitive==|Uses case-insensitive username matching|
+|==--deactivate-unlisted==|Deactivate users that aren't listed on the input file. (Current & system users won't be affected)|
+|==--user-id==|Identifier to use in looking up user. One of 'email' or 'username' (Default: 'email')|
+|==--verbose==|Log informational messages|
+|==--version==|Print version and exit|
+
+The tool will create users when needed, and update those existing records to match the desired state described by the fields on the CSV file.
+System users like the root and anonymous are unaffected by this tool.
+In the case of a @LoginCluster@ federation, this tool should be run on the cluster that manages the user accounts, and will fail otherwise.
+
+h2. Example
+
+To sync users using the username to identify every account, reading from some @external_users.csv@ file and deactivating existing users that aren't included in it, the command should be called as follows:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arvados-sync-users --deactivate-unlisted --user-id username /path/to/external_users.csv </span>
+</code></pre>
+</notextile>
h1. Using arvados-sync-groups
Permission level can be one of the following: @can_read@, @can_write@ or @can_manage@, giving the group member read, read/write or managing privileges on the group. For backwards compatibility purposes, if any record omits the third (permission) field, it will default to @can_write@ permission. You can read more about permissions on the "group management admin guide":{{ site.baseurl }}/admin/group-management.html.
-This tool is designed to be run periodically reading a file created by a remote auth system (ie: LDAP) dump script, applying what's included on the file as the source of truth.
-
-
-bq. NOTE: @arvados-sync-groups@ needs to perform several administrative tasks on Arvados, so must be run using a superuser token
-
h2. Options
The following command line options are supported:
import (
"context"
+ "crypto/rsa"
+ "crypto/tls"
+ "crypto/x509"
+ "encoding/pem"
+ "errors"
"fmt"
"io/ioutil"
"net"
+ "net/http"
+ "net/url"
"os"
"path/filepath"
+ "strings"
+ "time"
+
+ "golang.org/x/crypto/acme"
+ "golang.org/x/crypto/acme/autocert"
)
-// Create a root CA key and use it to make a new server
-// certificate+key pair.
-//
-// In future we'll make one root CA key per host instead of one per
-// cluster, so it only needs to be imported to a browser once for
-// ongoing dev/test usage.
+const stagingDirectoryURL = "https://acme-staging-v02.api.letsencrypt.org/directory"
+
+var errInvalidHost = errors.New("unrecognized target host in incoming TLS request")
+
type createCertificates struct{}
func (createCertificates) String() string {
}
func (createCertificates) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ if super.cluster.TLS.ACME.Server != "" {
+ return bootAutoCert(ctx, fail, super)
+ } else if super.cluster.TLS.Key == "" && super.cluster.TLS.Certificate == "" {
+ return createSelfSignedCert(ctx, fail, super)
+ } else {
+ return nil
+ }
+}
+
+// bootAutoCert uses Let's Encrypt to get certificates for all the
+// domains appearing in ExternalURLs, writes them to files where Nginx
+// can load them, and updates super.cluster.TLS fields (Key and
+// Certificiate) to point to those files.
+//
+// It also runs a background task to keep the files up to date.
+//
+// After bootAutoCert returns, other service components will get the
+// certificates they need by reading these files or by using a
+// read-only autocert cache.
+//
+// Currently this only works when port 80 of every ExternalURL domain
+// is routed to this host, i.e., on a single-node cluster. Wildcard
+// domains [for WebDAV] are not supported.
+func bootAutoCert(ctx context.Context, fail func(error), super *Supervisor) error {
+ hosts := map[string]bool{}
+ for _, svc := range super.cluster.Services.Map() {
+ u := url.URL(svc.ExternalURL)
+ if u.Scheme == "https" || u.Scheme == "wss" {
+ hosts[strings.ToLower(u.Hostname())] = true
+ }
+ }
+ mgr := &autocert.Manager{
+ Cache: autocert.DirCache(super.tempdir + "/autocert"),
+ Prompt: autocert.AcceptTOS,
+ HostPolicy: func(ctx context.Context, host string) error {
+ if hosts[strings.ToLower(host)] {
+ return nil
+ } else {
+ return errInvalidHost
+ }
+ },
+ }
+ if srv := super.cluster.TLS.ACME.Server; srv == "LE" {
+ // Leaving mgr.Client == nil means use Let's Encrypt
+ // production environment
+ } else if srv == "LE-staging" {
+ mgr.Client = &acme.Client{DirectoryURL: stagingDirectoryURL}
+ } else if strings.HasPrefix(srv, "https://") {
+ mgr.Client = &acme.Client{DirectoryURL: srv}
+ } else {
+ return fmt.Errorf("autocert setup: invalid directory URL in TLS.ACME.Server: %q", srv)
+ }
+ go func() {
+ err := http.ListenAndServe(":80", mgr.HTTPHandler(nil))
+ fail(fmt.Errorf("autocert http-01 challenge handler stopped: %w", err))
+ }()
+ u := url.URL(super.cluster.Services.Controller.ExternalURL)
+ extHost := u.Hostname()
+ update := func() error {
+ for h := range hosts {
+ cert, err := mgr.GetCertificate(&tls.ClientHelloInfo{ServerName: h})
+ if err != nil {
+ return err
+ }
+ if h == extHost {
+ err = writeCert(super.tempdir, "server.key", "server.crt", cert)
+ if err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+ }
+ err := update()
+ if err != nil {
+ return err
+ }
+ go func() {
+ for range time.NewTicker(time.Hour).C {
+ err := update()
+ if err != nil {
+ super.logger.WithError(err).Error("error getting certificate from autocert")
+ }
+ }
+ }()
+ super.cluster.TLS.Key = "file://" + super.tempdir + "/server.key"
+ super.cluster.TLS.Certificate = "file://" + super.tempdir + "/server.crt"
+ return nil
+}
+
+// Save cert chain and key in a format Nginx can read.
+func writeCert(outdir, keyfile, certfile string, cert *tls.Certificate) error {
+ keytmp, err := os.CreateTemp(outdir, keyfile+".tmp.*")
+ if err != nil {
+ return err
+ }
+ defer keytmp.Close()
+ defer os.Remove(keytmp.Name())
+
+ certtmp, err := os.CreateTemp(outdir, certfile+".tmp.*")
+ if err != nil {
+ return err
+ }
+ defer certtmp.Close()
+ defer os.Remove(certtmp.Name())
+
+ switch privkey := cert.PrivateKey.(type) {
+ case *rsa.PrivateKey:
+ err = pem.Encode(keytmp, &pem.Block{
+ Type: "RSA PRIVATE KEY",
+ Bytes: x509.MarshalPKCS1PrivateKey(privkey),
+ })
+ if err != nil {
+ return err
+ }
+ default:
+ buf, err := x509.MarshalPKCS8PrivateKey(privkey)
+ if err != nil {
+ return err
+ }
+ err = pem.Encode(keytmp, &pem.Block{
+ Type: "PRIVATE KEY",
+ Bytes: buf,
+ })
+ if err != nil {
+ return err
+ }
+ }
+ err = keytmp.Close()
+ if err != nil {
+ return err
+ }
+
+ for _, cert := range cert.Certificate {
+ err = pem.Encode(certtmp, &pem.Block{
+ Type: "CERTIFICATE",
+ Bytes: cert,
+ })
+ if err != nil {
+ return err
+ }
+ }
+ err = certtmp.Close()
+ if err != nil {
+ return err
+ }
+
+ err = os.Rename(keytmp.Name(), filepath.Join(outdir, keyfile))
+ if err != nil {
+ return err
+ }
+ err = os.Rename(certtmp.Name(), filepath.Join(outdir, certfile))
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+// Create a root CA key and use it to make a new server
+// certificate+key pair.
+//
+// In future we'll make one root CA key per host instead of one per
+// cluster, so it only needs to be imported to a browser once for
+// ongoing dev/test usage.
+func createSelfSignedCert(ctx context.Context, fail func(error), super *Supervisor) error {
+ san := "DNS:localhost,DNS:localhost.localdomain"
+ if net.ParseIP(super.ListenHost) != nil {
+ san += fmt.Sprintf(",IP:%s", super.ListenHost)
+ } else {
+ san += fmt.Sprintf(",DNS:%s", super.ListenHost)
+ }
+ hostname, err := os.Hostname()
+ if err != nil {
+ return fmt.Errorf("hostname: %w", err)
+ }
+ if hostname != super.ListenHost {
+ san += ",DNS:" + hostname
+ }
+
// Generate root key
- err := super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "genrsa", "-out", "rootCA.key", "4096")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "genrsa", "-out", "rootCA.key", "4096")
if err != nil {
return err
}
if err != nil {
return err
}
- hostname, err := os.Hostname()
- if err != nil {
- return fmt.Errorf("hostname: %w", err)
- }
- san := "DNS:localhost,DNS:localhost.localdomain,DNS:" + hostname
- if super.ListenHost == hostname || super.ListenHost == "localhost" {
- // already have it
- } else if net.ParseIP(super.ListenHost) != nil {
- san += fmt.Sprintf(",IP:%s", super.ListenHost)
- } else {
- san += fmt.Sprintf(",DNS:%s", super.ListenHost)
- }
conf := append(defaultconf, []byte(fmt.Sprintf("\n[SAN]\nsubjectAltName=%s\n", san))...)
err = ioutil.WriteFile(filepath.Join(super.tempdir, "server.cfg"), conf, 0644)
if err != nil {
if err != nil {
return err
}
+ super.cluster.TLS.Key = "file://" + super.tempdir + "/server.key"
+ super.cluster.TLS.Certificate = "file://" + super.tempdir + "/server.crt"
return nil
}
"git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/coreos/go-systemd/daemon"
)
var Command cmd.Handler = bootCommand{}
flags.StringVar(&super.ConfigPath, "config", "/etc/arvados/config.yml", "arvados config file `path`")
flags.StringVar(&super.SourcePath, "source", ".", "arvados source tree `directory`")
flags.StringVar(&super.ClusterType, "type", "production", "cluster `type`: development, test, or production")
- flags.StringVar(&super.ListenHost, "listen-host", "localhost", "host name or interface address for external services, and internal services whose InternalURLs are not configured")
+ flags.StringVar(&super.ListenHost, "listen-host", "localhost", "host name or interface address for internal services whose InternalURLs are not configured")
flags.StringVar(&super.ControllerAddr, "controller-address", ":0", "desired controller address, `host:port` or `:port`")
flags.StringVar(&super.Workbench2Source, "workbench2-source", "../arvados-workbench2", "path to arvados-workbench2 source tree")
flags.BoolVar(&super.NoWorkbench1, "no-workbench1", false, "do not run workbench1")
- flags.BoolVar(&super.NoWorkbench2, "no-workbench2", true, "do not run workbench2")
+ flags.BoolVar(&super.NoWorkbench2, "no-workbench2", false, "do not run workbench2")
flags.BoolVar(&super.OwnTemporaryDatabase, "own-temporary-database", false, "bring up a postgres server and create a temporary database")
timeout := flags.Duration("timeout", 0, "maximum time to wait for cluster to be ready")
shutdown := flags.Bool("shutdown", false, "shut down when the cluster becomes ready")
return nil
}
}
+ if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
+ super.logger.WithError(err).Errorf("error notifying init daemon")
+ }
// Wait for signal/crash + orderly shutdown
return super.Wait()
}
"os/exec"
"path/filepath"
"regexp"
+ "strings"
"git.arvados.org/arvados.git/sdk/go/arvados"
)
if err != nil {
return err
}
+ extListenHost := "0.0.0.0"
+ if super.ClusterType == "test" {
+ // Our dynamic port number assignment strategy (choose
+ // an available port, write it in a config file, and
+ // have another process/goroutine bind to it) is prone
+ // to races when used by concurrent supervisors. In
+ // test mode we don't accept remote connections, so we
+ // can avoid collisions by using the per-cluster
+ // loopback address instead of 0.0.0.0.
+ extListenHost = super.ListenHost
+ }
vars := map[string]string{
- "LISTENHOST": super.ListenHost,
+ "LISTENHOST": extListenHost,
+ "UPSTREAMHOST": super.ListenHost,
"SSLCERT": filepath.Join(super.tempdir, "server.crt"),
"SSLKEY": filepath.Join(super.tempdir, "server.key"),
"ACCESSLOG": filepath.Join(super.tempdir, "nginx_access.log"),
}
u := url.URL(super.cluster.Services.Controller.ExternalURL)
ctrlHost := u.Hostname()
- if f, err := os.Open("/var/lib/acme/live/" + ctrlHost + "/privkey"); err == nil {
+ if strings.HasPrefix(super.cluster.TLS.Certificate, "file:/") && strings.HasPrefix(super.cluster.TLS.Key, "file:/") {
+ vars["SSLCERT"] = filepath.Clean(super.cluster.TLS.Certificate[5:])
+ vars["SSLKEY"] = filepath.Clean(super.cluster.TLS.Key[5:])
+ } else if f, err := os.Open("/var/lib/acme/live/" + ctrlHost + "/privkey"); err == nil {
f.Close()
vars["SSLCERT"] = "/var/lib/acme/live/" + ctrlHost + "/cert"
vars["SSLKEY"] = "/var/lib/acme/live/" + ctrlHost + "/privkey"
"fmt"
"os"
"path/filepath"
+ "runtime"
"strings"
"sync"
break
}
}
- err = super.RunProgram(ctx, appdir, runOptions{}, "bundle", "install", "--jobs", "4", "--path", filepath.Join(os.Getenv("HOME"), ".gem"))
+ err = super.RunProgram(ctx, appdir, runOptions{}, "bundle", "config", "--set", "local", "path", filepath.Join(os.Getenv("HOME"), ".gem"))
+ if err != nil {
+ return err
+ }
+ err = super.RunProgram(ctx, appdir, runOptions{}, "bundle", "install", "--jobs", fmt.Sprintf("%d", runtime.NumCPU()))
if err != nil {
return err
}
if err != nil {
return err
}
+ super.wait(ctx, createCertificates{})
super.wait(ctx, runner.depends...)
for u := range runner.svc.InternalURLs {
u := u
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- fail(super.RunProgram(ctx, super.tempdir, runOptions{env: []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}}, binfile, runner.name, "-config", super.configfile))
+ fail(super.RunProgram(ctx, super.tempdir, runOptions{
+ env: []string{
+ "ARVADOS_SERVICE_INTERNAL_URL=" + u.String(),
+ // Child process should not
+ // try to tell systemd that we
+ // are ready.
+ "NOTIFY_SOCKET=",
+ },
+ }, binfile, runner.name, "-config", super.configfile))
}()
}
return nil
return err
}
+ super.wait(ctx, createCertificates{})
super.wait(ctx, runner.depends...)
for u := range runner.svc.InternalURLs {
u := u
super.done = make(chan struct{})
sigch := make(chan os.Signal)
- signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM)
- defer signal.Stop(sigch)
+ signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
go func() {
- for sig := range sigch {
- super.logger.WithField("signal", sig).Info("caught signal")
- if super.err == nil {
- super.err = fmt.Errorf("caught signal %s", sig)
- }
- super.cancel()
- }
- }()
-
- hupch := make(chan os.Signal)
- signal.Notify(hupch, syscall.SIGHUP)
- defer signal.Stop(hupch)
- go func() {
- for sig := range hupch {
- super.logger.WithField("signal", sig).Info("caught signal")
- if super.err == nil {
- super.err = errNeedConfigReload
+ defer signal.Stop(sigch)
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case sig := <-sigch:
+ super.logger.WithField("signal", sig).Info("caught signal")
+ if super.err == nil {
+ if sig == syscall.SIGHUP {
+ super.err = errNeedConfigReload
+ } else {
+ super.err = fmt.Errorf("caught signal %s", sig)
+ }
+ }
+ super.cancel()
}
- super.cancel()
}
}()
}
if super.ListenHost == "" {
- if urlhost := super.cluster.Services.Controller.ExternalURL.Host; urlhost != "" {
- if h, _, _ := net.SplitHostPort(urlhost); h != "" {
- super.ListenHost = h
- } else {
- super.ListenHost = urlhost
- }
- } else {
+ u := url.URL(super.cluster.Services.Controller.ExternalURL)
+ super.ListenHost = u.Hostname()
+ if super.ListenHost == "" {
super.ListenHost = "0.0.0.0"
}
}
super.logger.Infof("waiting for %s to be ready", id)
if !super2.WaitReady() {
super.logger.Infof("%s startup failed", id)
+ super.Stop()
return false
}
super.logger.Infof("%s is ready", id)
select {
case <-ticker.C:
case <-super.ctx.Done():
+ super.Stop()
return false
}
if super.healthChecker == nil {
return 2
}
model := split[len(split)-1]
- return externalCmd{"arv"}.RunCommand("arv", legacyFlagsToFront(model, args), stdin, stdout, stderr)
+ return rubyArvCmd{model}.RunCommand(prog, args, stdin, stdout, stderr)
}
type rubyArvCmd struct {
}
func (rc rubyArvCmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
- return externalCmd{"arv"}.RunCommand("arv", legacyFlagsToFront(rc.subcommand, args), stdin, stdout, stderr)
+ wrapprog := "arv-ruby"
+ if _, err := exec.LookPath(wrapprog); err != nil && !strings.Contains(prog, "arv ") {
+ // arv-ruby isn't in PATH (i.e., installation method
+ // wasn't a recent "arvados-server install", which
+ // symlinks /usr/bin/arv-ruby ->
+ // /var/lib/arvados/bin/arv), so fall back to looking
+ // for the arvados-cli program as "arv". (But don't do
+ // this if we are being run as "arv" -- that would
+ // probably cause a recursive fork bomb.)
+ wrapprog = "arv"
+ }
+ return externalCmd{wrapprog}.RunCommand(wrapprog, legacyFlagsToFront(rc.subcommand, args), stdin, stdout, stderr)
}
type externalCmd struct {
return 1
case *exec.Error:
fmt.Fprintln(stderr, err)
- if ec.prog == "arv" {
+ if ec.prog == "arv" || ec.prog == "arv-ruby" {
fmt.Fprint(stderr, rubyInstallHints)
} else if strings.HasPrefix(ec.prog, "arv-") {
fmt.Fprint(stderr, pythonInstallHints)
code := DumpCommand.RunCommand("arvados config-dump", []string{"-config", "-"}, bytes.NewBufferString(in), &stdout, &stderr)
c.Check(code, check.Equals, 0)
c.Check(stdout.String(), check.Matches, `(?ms).*TimeoutBooting: 10m\n.*`)
- c.Check(stdout.String(), check.Matches, `(?ms).*http://localhost:12345/: {}\n.*`)
+ c.Check(stdout.String(), check.Matches, `(?ms).*http://localhost:12345/:\n +ListenURL: ""\n.*`)
}
func (s *CommandSuite) TestDump_UnknownKey(c *check.C) {
Services:
- # In each of the service sections below, the keys under
- # InternalURLs are the endpoints where the service should be
- # listening, and reachable from other hosts in the
- # cluster. Example:
+ # Each of the service sections below specifies InternalURLs
+ # (each with optional ListenURL) and ExternalURL.
+ #
+ # InternalURLs specify how other Arvados service processes will
+ # connect to the service. Typically these use internal hostnames
+ # and high port numbers. Example:
+ #
+ # InternalURLs:
+ # "http://host1.internal.example:12345": {}
+ # "http://host2.internal.example:12345": {}
+ #
+ # ListenURL specifies the address and port the service process's
+ # HTTP server should listen on, if different from the
+ # InternalURL itself. Example, using an intermediate TLS proxy:
#
# InternalURLs:
- # "http://host1.example:12345": {}
- # "http://host2.example:12345": {}
+ # "https://host1.internal.example":
+ # ListenURL: "http://10.0.0.7:12345"
+ #
+ # When there are multiple InternalURLs configured, the service
+ # process will try listening on each InternalURLs (using
+ # ListenURL if provided) until one works. If you use a ListenURL
+ # like "0.0.0.0" which can be bound on any machine, use an
+ # environment variable
+ # ARVADOS_SERVICE_INTERNAL_URL=http://host1.internal.example to
+ # control which entry to use.
+ #
+ # ExternalURL specifies how applications/clients will connect to
+ # the service, regardless of whether they are inside or outside
+ # the cluster. Example:
+ #
+ # ExternalURL: "https://keep.zzzzz.example.com/"
+ #
+ # To avoid routing internal traffic through external networks,
+ # use split-horizon DNS for ExternalURL host names: inside the
+ # cluster's private network "host.zzzzz.example.com" resolves to
+ # the host's private IP address, while outside the cluster
+ # "host.zzzzz.example.com" resolves to the host's public IP
+ # address (or its external gateway or load balancer).
RailsAPI:
- InternalURLs: {SAMPLE: {}}
+ InternalURLs: {SAMPLE: {ListenURL: ""}}
ExternalURL: ""
Controller:
- InternalURLs: {SAMPLE: {}}
+ InternalURLs: {SAMPLE: {ListenURL: ""}}
ExternalURL: ""
Websocket:
- InternalURLs: {SAMPLE: {}}
+ InternalURLs: {SAMPLE: {ListenURL: ""}}
ExternalURL: ""
Keepbalance:
- InternalURLs: {SAMPLE: {}}
+ InternalURLs: {SAMPLE: {ListenURL: ""}}
ExternalURL: ""
GitHTTP:
- InternalURLs: {SAMPLE: {}}
+ InternalURLs: {SAMPLE: {ListenURL: ""}}
ExternalURL: ""
GitSSH:
- InternalURLs: {SAMPLE: {}}
+ InternalURLs: {SAMPLE: {ListenURL: ""}}
ExternalURL: ""
DispatchCloud:
- InternalURLs: {SAMPLE: {}}
+ InternalURLs: {SAMPLE: {ListenURL: ""}}
ExternalURL: ""
DispatchLSF:
- InternalURLs: {SAMPLE: {}}
+ InternalURLs: {SAMPLE: {ListenURL: ""}}
ExternalURL: ""
DispatchSLURM:
- InternalURLs: {SAMPLE: {}}
+ InternalURLs: {SAMPLE: {ListenURL: ""}}
ExternalURL: ""
Keepproxy:
- InternalURLs: {SAMPLE: {}}
+ InternalURLs: {SAMPLE: {ListenURL: ""}}
ExternalURL: ""
WebDAV:
- InternalURLs: {SAMPLE: {}}
+ InternalURLs: {SAMPLE: {ListenURL: ""}}
# Base URL for Workbench inline preview. If blank, use
# WebDAVDownload instead, and disable inline preview.
# If both are empty, downloading collections from workbench
ExternalURL: ""
WebDAVDownload:
- InternalURLs: {SAMPLE: {}}
+ InternalURLs: {SAMPLE: {ListenURL: ""}}
# Base URL for download links. If blank, serve links to WebDAV
# with disposition=attachment query param. Unlike preview links,
# browsers do not render attachments, so there is no risk of XSS.
Keepstore:
InternalURLs:
SAMPLE:
+ ListenURL: ""
# Rendezvous is normally empty/omitted. When changing the
# URL of a Keepstore service, Rendezvous should be set to
# the old URL (with trailing slash omitted) to preserve
Rendezvous: ""
ExternalURL: ""
Composer:
- InternalURLs: {SAMPLE: {}}
+ InternalURLs: {SAMPLE: {ListenURL: ""}}
ExternalURL: ""
WebShell:
- InternalURLs: {SAMPLE: {}}
+ InternalURLs: {SAMPLE: {ListenURL: ""}}
# ShellInABox service endpoint URL for a given VM. If empty, do not
# offer web shell logins.
#
# https://*.webshell.uuid_prefix.arvadosapi.com
ExternalURL: ""
Workbench1:
- InternalURLs: {SAMPLE: {}}
+ InternalURLs: {SAMPLE: {ListenURL: ""}}
ExternalURL: ""
Workbench2:
- InternalURLs: {SAMPLE: {}}
+ InternalURLs: {SAMPLE: {ListenURL: ""}}
ExternalURL: ""
Health:
- InternalURLs: {SAMPLE: {}}
+ InternalURLs: {SAMPLE: {ListenURL: ""}}
ExternalURL: ""
PostgreSQL:
Repositories: /var/lib/arvados/git/repositories
TLS:
+ # Use "file:///var/lib/acme/live/example.com/cert" and
+ # ".../privkey" to load externally managed certificates.
Certificate: ""
Key: ""
+
+ # Accept invalid certificates when connecting to servers. Never
+ # use this in production.
Insecure: false
+ ACME:
+ # Obtain certificates automatically for ExternalURL domains
+ # using an ACME server and http-01 validation.
+ #
+ # To use Let's Encrypt, specify "LE". To use the Let's
+ # Encrypt staging environment, specify "LE-staging". To use a
+ # different ACME server, specify the full directory URL
+ # ("https://...").
+ #
+ # Note: this feature is not yet implemented in released
+ # versions, only in the alpha/prerelease arvados-server-easy
+ # package.
+ #
+ # Implies agreement with the server's terms of service.
+ Server: ""
+
Containers:
# List of supported Docker Registry image formats that compute nodes
# are able to use. `arv keep docker` will error out if a user tries
"net/url"
"strings"
"sync"
- "time"
"git.arvados.org/arvados.git/lib/controller/api"
"git.arvados.org/arvados.git/lib/controller/federation"
req.URL.Path = strings.Replace(req.URL.Path, "//", "/", -1)
}
}
- if h.Cluster.API.RequestTimeout > 0 {
- ctx, cancel := context.WithDeadline(req.Context(), time.Now().Add(time.Duration(h.Cluster.API.RequestTimeout)))
- req = req.WithContext(ctx)
- defer cancel()
- }
-
h.handlerStack.ServeHTTP(w, req)
}
c.Check(len(dd.Schemas), check.Not(check.Equals), 0)
}
-func (s *HandlerSuite) TestRequestTimeout(c *check.C) {
- s.cluster.API.RequestTimeout = arvados.Duration(time.Nanosecond)
- req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
+// Handler should give up and exit early if request context is
+// cancelled due to client hangup, httpserver.HandlerWithDeadline,
+// etc.
+func (s *HandlerSuite) TestRequestCancel(c *check.C) {
+ ctx, cancel := context.WithCancel(context.Background())
+ req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil).WithContext(ctx)
resp := httptest.NewRecorder()
+ cancel()
s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusBadGateway)
var jresp httpserver.ErrorResponse
err := json.Unmarshal(resp.Body.Bytes(), &jresp)
c.Check(err, check.IsNil)
c.Assert(len(jresp.Errors), check.Equals, 1)
- c.Check(jresp.Errors[0], check.Matches, `.*context deadline exceeded.*`)
+ c.Check(jresp.Errors[0], check.Matches, `.*context canceled`)
}
func (s *HandlerSuite) TestProxyWithoutToken(c *check.C) {
}
func (s *IntegrationSuite) TestRunTrivialContainer(c *check.C) {
- outcoll := s.runContainer(c, "z1111", map[string]interface{}{
+ outcoll, _ := s.runContainer(c, "z1111", "", map[string]interface{}{
"command": []string{"sh", "-c", "touch \"/out/hello world\" /out/ohai"},
"container_image": "busybox:uclibc",
"cwd": "/tmp",
c.Check(outcoll.PortableDataHash, check.Equals, "8fa5dee9231a724d7cf377c5a2f4907c+65")
}
-func (s *IntegrationSuite) runContainer(c *check.C, clusterID string, ctrSpec map[string]interface{}, expectExitCode int) arvados.Collection {
+func (s *IntegrationSuite) TestContainerInputOnDifferentCluster(c *check.C) {
+ conn := s.super.Conn("z1111")
+ rootctx, _, _ := s.super.RootClients("z1111")
+ userctx, ac, _, _ := s.super.UserClients("z1111", rootctx, c, conn, s.oidcprovider.AuthEmail, true)
+ z1coll, err := conn.CollectionCreate(userctx, arvados.CreateOptions{Attrs: map[string]interface{}{
+ "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:ocelot\n",
+ }})
+ c.Assert(err, check.IsNil)
+
+ outcoll, logcfs := s.runContainer(c, "z2222", ac.AuthToken, map[string]interface{}{
+ "command": []string{"ls", "/in"},
+ "container_image": "busybox:uclibc",
+ "cwd": "/tmp",
+ "environment": map[string]string{},
+ "mounts": map[string]arvados.Mount{
+ "/in": {Kind: "collection", PortableDataHash: z1coll.PortableDataHash},
+ "/out": {Kind: "tmp", Capacity: 10000},
+ },
+ "output_path": "/out",
+ "runtime_constraints": arvados.RuntimeConstraints{RAM: 100000000, VCPUs: 1},
+ "priority": 1,
+ "state": arvados.ContainerRequestStateCommitted,
+ "container_count_max": 1,
+ }, -1)
+ if outcoll.UUID == "" {
+ arvmountlog, err := fs.ReadFile(arvados.FS(logcfs), "/arv-mount.txt")
+ c.Check(err, check.IsNil)
+ c.Check(string(arvmountlog), check.Matches, `(?ms).*cannot use a locally issued token to forward a request to our login cluster \(z1111\).*`)
+ c.Skip("this use case is not supported yet")
+ }
+ stdout, err := fs.ReadFile(arvados.FS(logcfs), "/stdout.txt")
+ c.Check(err, check.IsNil)
+ c.Check(string(stdout), check.Equals, "ocelot\n")
+}
+
+func (s *IntegrationSuite) runContainer(c *check.C, clusterID string, token string, ctrSpec map[string]interface{}, expectExitCode int) (outcoll arvados.Collection, logcfs arvados.CollectionFileSystem) {
conn := s.super.Conn(clusterID)
rootctx, _, _ := s.super.RootClients(clusterID)
- _, ac, kc, _ := s.super.UserClients(clusterID, rootctx, c, conn, s.oidcprovider.AuthEmail, true)
+ if token == "" {
+ _, ac, _, _ := s.super.UserClients(clusterID, rootctx, c, conn, s.oidcprovider.AuthEmail, true)
+ token = ac.AuthToken
+ }
+ _, ac, kc := s.super.ClientsWithToken(clusterID, token)
c.Log("[docker load]")
out, err := exec.Command("docker", "load", "--input", arvadostest.BusyboxDockerImage(c)).CombinedOutput()
})
c.Assert(err, check.IsNil)
- showlogs := func(collectionID string) {
+ showlogs := func(collectionID string) arvados.CollectionFileSystem {
var logcoll arvados.Collection
err = ac.RequestAndDecode(&logcoll, "GET", "/arvados/v1/collections/"+collectionID, nil, nil)
c.Assert(err, check.IsNil)
c.Logf("=== %s\n%s\n", path, buf)
return nil
})
+ return cfs
}
var ctr arvados.Container
var lastState arvados.ContainerState
deadline := time.Now().Add(time.Minute)
-wait:
- for ; ; lastState = ctr.State {
- if time.Now().After(deadline) {
- c.Errorf("timed out, container request state is %q", cr.State)
- showlogs(ctr.Log)
- c.FailNow()
- }
+ for cr.State != arvados.ContainerRequestStateFinal {
+ err = ac.RequestAndDecode(&cr, "GET", "/arvados/v1/container_requests/"+cr.UUID, nil, nil)
+ c.Assert(err, check.IsNil)
err = ac.RequestAndDecode(&ctr, "GET", "/arvados/v1/containers/"+cr.ContainerUUID, nil, nil)
if err != nil {
- // container req is being auto-retried with a new container uuid
- ac.RequestAndDecode(&cr, "GET", "/arvados/v1/container_requests/"+cr.UUID, nil, nil)
- c.Assert(err, check.IsNil)
- time.Sleep(time.Second / 2)
- continue
- }
- switch ctr.State {
- case lastState:
- time.Sleep(time.Second / 2)
- case arvados.ContainerStateComplete:
- break wait
- case arvados.ContainerStateQueued, arvados.ContainerStateLocked, arvados.ContainerStateRunning:
+ c.Logf("error getting container state: %s", err)
+ } else if ctr.State != lastState {
c.Logf("container state changed to %q", ctr.State)
- default:
- c.Errorf("unexpected container state %q", ctr.State)
- showlogs(ctr.Log)
- c.FailNow()
+ lastState = ctr.State
+ } else {
+ if time.Now().After(deadline) {
+ c.Errorf("timed out, container state is %q", cr.State)
+ showlogs(ctr.Log)
+ c.FailNow()
+ }
+ time.Sleep(time.Second / 2)
}
}
- c.Check(ctr.ExitCode, check.Equals, 0)
-
- err = ac.RequestAndDecode(&cr, "GET", "/arvados/v1/container_requests/"+cr.UUID, nil, nil)
- c.Assert(err, check.IsNil)
-
- showlogs(cr.LogUUID)
-
- var outcoll arvados.Collection
- err = ac.RequestAndDecode(&outcoll, "GET", "/arvados/v1/collections/"+cr.OutputUUID, nil, nil)
- c.Assert(err, check.IsNil)
- return outcoll
+ if expectExitCode >= 0 {
+ c.Check(ctr.State, check.Equals, arvados.ContainerStateComplete)
+ c.Check(ctr.ExitCode, check.Equals, expectExitCode)
+ err = ac.RequestAndDecode(&outcoll, "GET", "/arvados/v1/collections/"+cr.OutputUUID, nil, nil)
+ c.Assert(err, check.IsNil)
+ }
+ logcfs = showlogs(cr.LogUUID)
+ return outcoll, logcfs
}
env["ARVADOS_API_TOKEN"] = tok
env["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST")
env["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE")
+ env["ARVADOS_KEEP_SERVICES"] = os.Getenv("ARVADOS_KEEP_SERVICES")
}
workdir := runner.Container.Cwd
if workdir == "." {
// modify the cluster configuration that we feed it on stdin.
configData.Cluster.API.MaxKeepBlobBuffers = configData.KeepBuffers
- ln, err := net.Listen("tcp", "localhost:0")
+ localaddr := localKeepstoreAddr()
+ ln, err := net.Listen("tcp", net.JoinHostPort(localaddr, "0"))
if err != nil {
return nil, err
}
return nil, err
}
ln.Close()
- url := "http://localhost:" + port
+ url := "http://" + net.JoinHostPort(localaddr, port)
fmt.Fprintf(logbuf, "starting keepstore on %s\n", url)
}
return s
}
+
+// Return a suitable local interface address for a local keepstore
+// service. Currently this is the numerically lowest non-loopback ipv4
+// address assigned to a local interface that is not in any of the
+// link-local/vpn/loopback ranges 169.254/16, 100.64/10, or 127/8.
+func localKeepstoreAddr() string {
+ var ips []net.IP
+ // Ignore error (proceed with zero IPs)
+ addrs, _ := processIPs(os.Getpid())
+ for addr := range addrs {
+ ip := net.ParseIP(addr)
+ if ip == nil {
+ // invalid
+ continue
+ }
+ if ip.Mask(net.CIDRMask(8, 32)).Equal(net.IPv4(127, 0, 0, 0)) ||
+ ip.Mask(net.CIDRMask(10, 32)).Equal(net.IPv4(100, 64, 0, 0)) ||
+ ip.Mask(net.CIDRMask(16, 32)).Equal(net.IPv4(169, 254, 0, 0)) {
+ // unsuitable
+ continue
+ }
+ ips = append(ips, ip)
+ }
+ if len(ips) == 0 {
+ return "0.0.0.0"
+ }
+ sort.Slice(ips, func(ii, jj int) bool {
+ i, j := ips[ii], ips[jj]
+ if len(i) != len(j) {
+ return len(i) < len(j)
+ }
+ for x := range i {
+ if i[x] != j[x] {
+ return i[x] < j[x]
+ }
+ }
+ return false
+ })
+ return ips[0].String()
+}
c.Check(log, trial.matchGetReq, `(?ms).*"reqMethod":"GET".*`)
c.Check(log, trial.matchPutReq, `(?ms).*"reqMethod":"PUT".*,"reqPath":"0e3bcff26d51c895a60ea0d4585e134d".*`)
}
+
+ c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*using local keepstore process .* at http://[\d\.]{7,}:\d+.*`)
+ c.Check(s.logFiles["crunch-run.txt"], Not(Matches), `(?ms).* at http://127\..*`)
+ c.Check(s.logFiles["crunch-run.txt"], Not(Matches), `(?ms).* at http://169\.254\..*`)
+ c.Check(s.logFiles["stderr.txt"], Matches, `(?ms).*ARVADOS_KEEP_SERVICES=http://[\d\.]{7,}:\d+\n.*`)
}
}
if err := exec.Command("which", s.engine).Run(); err != nil {
c.Skip(fmt.Sprintf("%s: %s", s.engine, err))
}
- s.cr.Command = []string{"sh", "-c", "cat /mnt/in/inputfile >/mnt/out/inputfile && cat /mnt/json >/mnt/out/json && ! touch /mnt/in/shouldbereadonly && mkdir /mnt/out/emptydir"}
+ s.cr.Command = []string{"sh", "-c", "env >&2 && cat /mnt/in/inputfile >/mnt/out/inputfile && cat /mnt/json >/mnt/out/json && ! touch /mnt/in/shouldbereadonly && mkdir /mnt/out/emptydir"}
s.setup(c)
args := []string{
f := flag.NewFlagSet(prog, flag.ContinueOnError)
f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects")
f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
+ f.StringVar(&diag.dockerImage, "docker-image", "alpine:latest", "image to use when running a test container")
f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client")
f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client")
f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000, or 0 to skip)")
logLevel string
priority int
projectName string
+ dockerImage string
checkInternal bool
checkExternal bool
timeout time.Duration
var cluster arvados.Cluster
cfgpath := "arvados/v1/config"
+ cfgOK := false
diag.dotest(20, fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath), func() error {
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
defer cancel()
return err
}
diag.debugf("Collections.BlobSigning = %v", cluster.Collections.BlobSigning)
+ cfgOK = true
return nil
})
return nil
})
+ if !cfgOK {
+ diag.errorf("cannot proceed without cluster config -- aborting without running any further tests")
+ return
+ }
+
// uncomment to create some spurious errors
// cluster.Services.WebDAVDownload.ExternalURL.Host = "0.0.0.0:9"
})
davurl := cluster.Services.WebDAV.ExternalURL
+ davWildcard := strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.")
diag.dotest(110, fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl), func() error {
if davurl.Host == "" {
return fmt.Errorf("host missing - content previews will not work")
}
- if !strings.HasPrefix(davurl.Host, "*--") && !strings.HasPrefix(davurl.Host, "*.") && !cluster.Collections.TrustAllContent {
+ if !davWildcard && !cluster.Collections.TrustAllContent {
diag.warnf("WebDAV ExternalURL has no leading wildcard and TrustAllContent==false - content previews will not work")
}
return nil
})
for i, trial := range []struct {
- needcoll bool
- status int
- fileurl string
+ needcoll bool
+ needWildcard bool
+ status int
+ fileurl string
}{
- {false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
- {false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"},
- {false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
- {false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"},
- {true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"},
- {true, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
+ {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
+ {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"},
+ {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
+ {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"},
+ {true, true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"},
+ {true, false, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
} {
diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error {
+ if trial.needWildcard && !davWildcard {
+ diag.warnf("skipping collection-id-in-vhost test because WebDAV ExternalURL has no leading wildcard")
+ return nil
+ }
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
defer cancel()
if trial.needcoll && collection.UUID == "" {
return err
}
if len(vmlist.Items) < 1 {
- return fmt.Errorf("no VMs found")
+ diag.warnf("no VMs found")
+ } else {
+ vm = vmlist.Items[0]
}
- vm = vmlist.Items[0]
return nil
})
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
defer cancel()
if vm.UUID == "" {
- return fmt.Errorf("skipping, no vm available")
+ diag.warnf("skipping, no vm available")
+ return nil
}
webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername"
diag.debugf("url %s", webshelltermurl)
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
defer cancel()
if vm.UUID == "" {
- return fmt.Errorf("skipping, no vm available")
+ diag.warnf("skipping, no vm available")
+ return nil
}
u := cluster.Services.WebShell.ExternalURL
webshellurl := u.String() + vm.Hostname + "?"
err := client.RequestAndDecodeContext(ctx, &cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{
"owner_uuid": project.UUID,
"name": fmt.Sprintf("diagnostics container request %s", timestamp),
- "container_image": "arvados/jobs",
+ "container_image": diag.dockerImage,
"command": []string{"echo", timestamp},
"use_existing": false,
"output_path": "/mnt/output",
Type=notify
EnvironmentFile=-/etc/arvados/environment
ExecStart=/usr/bin/arvados-server boot
-# Set a reasonable default for the open file limit
-LimitNOFILE=65536
+ExecReload=/usr/bin/arvados-server config-check
+ExecReload=kill -HUP $MAINPID
Restart=always
RestartSec=1
+LimitNOFILE=65536
# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
StartLimitInterval=0
"os/exec"
"os/user"
"path/filepath"
+ "runtime"
"strconv"
"strings"
"syscall"
gradleversion = "5.3.1"
nodejsversion = "v12.22.11"
devtestDatabasePassword = "insecure_arvados_test"
- workbench2version = "5e020488f67b5bc919796e0dc8b0b9f3b3ff23b0"
+ workbench2version = "2454ac35292a79594c32a80430740317ed5005cf"
)
//go:embed arvados.service
"uuid-dev",
"wget",
"xvfb",
+ "zlib1g-dev", // services/api
)
if test {
if osv.Debian && osv.Major <= 10 {
}
switch {
case osv.Debian && osv.Major >= 11:
- pkgs = append(pkgs, "libcurl4", "perl-modules-5.32")
+ pkgs = append(pkgs, "g++", "libcurl4", "libcurl4-openssl-dev", "perl-modules-5.32")
case osv.Debian && osv.Major >= 10:
- pkgs = append(pkgs, "libcurl4", "perl-modules")
- default:
- pkgs = append(pkgs, "libcurl3", "perl-modules")
+ pkgs = append(pkgs, "g++", "libcurl4", "libcurl4-openssl-dev", "perl-modules")
+ case osv.Debian || osv.Ubuntu:
+ pkgs = append(pkgs, "g++", "libcurl3", "libcurl3-openssl-dev", "perl-modules")
+ case osv.Centos:
+ pkgs = append(pkgs, "gcc", "gcc-c++", "libcurl-devel", "postgresql-devel")
}
cmd := exec.CommandContext(ctx, "apt-get")
if inst.EatMyData {
for _, srcdir := range []string{
"cmd/arvados-client",
"cmd/arvados-server",
- "services/crunch-dispatch-local",
- "services/crunch-dispatch-slurm",
} {
fmt.Fprintf(stderr, "building %s...\n", srcdir)
cmd := exec.Command("go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+inst.PackageVersion+" -X main.version="+inst.PackageVersion+" -s -w")
}
}
- // Symlink user-facing Go programs /usr/bin/x ->
- // /var/lib/arvados/bin/x
- for _, prog := range []string{"arvados-client", "arvados-server"} {
- err = os.Remove("/usr/bin/" + prog)
- if err != nil && !errors.Is(err, os.ErrNotExist) {
- return 1
- }
- err = os.Symlink("/var/lib/arvados/bin/"+prog, "/usr/bin/"+prog)
- if err != nil {
- return 1
- }
- }
-
// Copy assets from source tree to /var/lib/arvados/share
cmd := exec.Command("install", "-v", "-t", "/var/lib/arvados/share", filepath.Join(inst.SourcePath, "sdk/python/tests/nginx.conf"))
cmd.Stdout = stdout
return 1
}
+ // Install python SDK and arv-mount in
+ // /var/lib/arvados/lib/python.
+ //
+ // setup.py writes a file in the source directory in
+ // order to include the version number in the package
+ // itself. We don't want to write to the source tree
+ // (in "arvados-package" context it's mounted
+ // readonly) so we run setup.py in a temporary copy of
+ // the source dir.
+ if err = inst.runBash(`
+v=/var/lib/arvados/lib/python
+tmp=/var/lib/arvados/tmp/python
+python3 -m venv "$v"
+. "$v/bin/activate"
+pip3 install --no-cache-dir 'setuptools>=18.5' 'pip>=7'
+export ARVADOS_BUILDING_VERSION="`+inst.PackageVersion+`"
+for src in "`+inst.SourcePath+`/sdk/python" "`+inst.SourcePath+`/services/fuse"; do
+ rsync -a --delete-after "$src/" "$tmp/"
+ cd "$tmp"
+ python3 setup.py install
+ cd ..
+ rm -rf "$tmp"
+done
+`, stdout, stderr); err != nil {
+ return 1
+ }
+
// Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/
for dstdir, srcdir := range map[string]string{
"railsapi": "services/api",
{"touch", "log/production.log"},
{"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.bundle", "/var/www/.gem", "/var/www/.npm", "/var/www/.passenger", "log", "tmp", "vendor", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:" + bundlerversion},
- {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem", "--without", "development test diagnostics performance"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "deployment", "true"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "path", "/var/www/.gem"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "without", "development test diagnostics performance"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--jobs", fmt.Sprintf("%d", runtime.NumCPU())},
{"chown", "www-data:www-data", ".", "public/assets"},
// {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "system", "true"},
- {"sudo", "-u", "www-data", "ARVADOS_CONFIG=none", "RAILS_GROUPS=assets", "RAILS_ENV=production", "/var/lib/arvados/bin/bundle", "exec", "rake", "npm:install"},
- {"sudo", "-u", "www-data", "ARVADOS_CONFIG=none", "RAILS_GROUPS=assets", "RAILS_ENV=production", "/var/lib/arvados/bin/bundle", "exec", "rake", "assets:precompile"},
+ {"sudo", "-u", "www-data", "ARVADOS_CONFIG=none", "RAILS_GROUPS=assets", "RAILS_ENV=production", "PATH=/var/lib/arvados/bin:" + os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "npm:install"},
+ {"sudo", "-u", "www-data", "ARVADOS_CONFIG=none", "RAILS_GROUPS=assets", "RAILS_ENV=production", "PATH=/var/lib/arvados/bin:" + os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "assets:precompile"},
{"chown", "root:root", "."},
{"chown", "-R", "root:root", "public/assets", "vendor"},
return 1
}
+ // Install arvados-cli gem (binaries go in
+ // /var/lib/arvados/bin)
+ if err = inst.runBash(`
+/var/lib/arvados/bin/gem install --conservative --no-document arvados-cli
+`, stdout, stderr); err != nil {
+ return 1
+ }
+
err = os.WriteFile("/lib/systemd/system/arvados.service", arvadosServiceFile, 0777)
if err != nil {
return 1
}
- // This is equivalent to "systemd enable", but does
- // not depend on the systemctl program being
- // available.
- symlink := "/etc/systemd/system/multi-user.target.wants/arvados.service"
- err = os.Remove(symlink)
- if err != nil && !errors.Is(err, os.ErrNotExist) {
- return 1
+ if prod {
+ // (fpm will do this for us in the pkg case)
+ // This is equivalent to "systemd enable", but
+ // does not depend on the systemctl program
+ // being available:
+ symlink := "/etc/systemd/system/multi-user.target.wants/arvados.service"
+ err = os.Remove(symlink)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ return 1
+ }
+ err = os.Symlink("/lib/systemd/system/arvados.service", symlink)
+ if err != nil {
+ return 1
+ }
}
- err = os.Symlink("/lib/systemd/system/arvados.service", symlink)
- if err != nil {
- return 1
+
+ // Add symlinks in /usr/bin for user-facing programs
+ for _, srcdst := range [][]string{
+ // go
+ {"bin/arvados-client"},
+ {"bin/arvados-client", "arv"},
+ {"bin/arvados-server"},
+ // sdk/cli
+ {"bin/arv", "arv-ruby"},
+ {"bin/arv-tag"},
+ // sdk/python
+ {"lib/python/bin/arv-copy"},
+ {"lib/python/bin/arv-federation-migrate"},
+ {"lib/python/bin/arv-get"},
+ {"lib/python/bin/arv-keepdocker"},
+ {"lib/python/bin/arv-ls"},
+ {"lib/python/bin/arv-migrate-docker19"},
+ {"lib/python/bin/arv-normalize"},
+ {"lib/python/bin/arv-put"},
+ {"lib/python/bin/arv-ws"},
+ // services/fuse
+ {"lib/python/bin/arv-mount"},
+ } {
+ src := "/var/lib/arvados/" + srcdst[0]
+ if _, err = os.Stat(src); err != nil {
+ return 1
+ }
+ dst := srcdst[len(srcdst)-1]
+ _, dst = filepath.Split(dst)
+ dst = "/usr/bin/" + dst
+ err = os.Remove(dst)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ return 1
+ }
+ err = os.Symlink(src, dst)
+ if err != nil {
+ return 1
+ }
}
}
"libcurl3-gnutls",
"libxslt1.1",
"nginx",
- "python",
+ "python3",
"sudo",
}
if osv.Debian || osv.Ubuntu {
pkgs = append(pkgs, "python3-distutils") // sdk/cwl
}
return append(pkgs,
- "g++",
- "libcurl4-openssl-dev", // services/api
- "libpq-dev",
- "libpython2.7", // services/fuse
"mime-support", // keep-web
- "zlib1g-dev", // services/api
)
} else if osv.Centos {
return append(pkgs,
"fuse-libs", // services/fuse
- "gcc",
- "gcc-c++",
- "libcurl-devel", // services/api
- "mailcap", // keep-web
- "postgresql-devel", // services/api
+ "mailcap", // keep-web
)
} else {
panic("os version not supported")
"flag"
"fmt"
"io"
+ "net"
+ "net/http"
+ "net/url"
"os"
"os/exec"
"os/user"
"regexp"
"strconv"
+ "strings"
+ "sync/atomic"
"text/template"
+ "time"
"git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/lib/controller/rpc"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"github.com/lib/pq"
)
var InitCommand cmd.Handler = &initCommand{}
type initCommand struct {
- ClusterID string
- Domain string
- PostgreSQLPassword string
- Login string
- Insecure bool
+ ClusterID string
+ Domain string
+ CreateDB bool
+ Login string
+ TLS string
+ AdminEmail string
+ Start bool
+
+ PostgreSQL struct {
+ Host string
+ User string
+ Password string
+ DB string
+ }
+ LoginPAM bool
+ LoginTest bool
+ LoginGoogle bool
+ LoginGoogleClientID string
+ LoginGoogleClientSecret string
+ TLSDir string
}
func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
flags.SetOutput(stderr)
versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
flags.StringVar(&initcmd.ClusterID, "cluster-id", "", "cluster `id`, like x1234 for a dev cluster")
+ flags.BoolVar(&initcmd.CreateDB, "create-db", true, "create an 'arvados' postgresql role and database using 'sudo -u postgres psql ...' (if false, use existing database specified by POSTGRES_HOST, POSTGRES_USER, POSTGRES_PASSWORD, and POSTGRES_DB env vars, and assume 'CREATE EXTENSION IF NOT EXISTS pg_trgm' has already been done)")
flags.StringVar(&initcmd.Domain, "domain", hostname, "cluster public DNS `name`, like x1234.arvadosapi.com")
- flags.StringVar(&initcmd.Login, "login", "", "login `backend`: test, pam, or ''")
- flags.BoolVar(&initcmd.Insecure, "insecure", false, "accept invalid TLS certificates and configure TrustAllContent (do not use in production!)")
+ flags.StringVar(&initcmd.Login, "login", "", "login `backend`: test, pam, 'google {client-id} {client-secret}', or ''")
+ flags.StringVar(&initcmd.AdminEmail, "admin-email", "", "give admin privileges to user with given `email`")
+ flags.StringVar(&initcmd.TLS, "tls", "none", "tls certificate `source`: acme, insecure, none, or /path/to/dir containing privkey and cert files")
+ flags.BoolVar(&initcmd.Start, "start", true, "start systemd service after creating config")
if ok, code := cmd.ParseFlags(flags, prog, args, "", stderr); !ok {
return code
} else if *versionFlag {
return 1
}
+ if fields := strings.Fields(initcmd.Login); len(fields) == 3 && fields[0] == "google" {
+ initcmd.LoginGoogle = true
+ initcmd.LoginGoogleClientID = fields[1]
+ initcmd.LoginGoogleClientSecret = fields[2]
+ } else if initcmd.Login == "test" {
+ initcmd.LoginTest = true
+ if initcmd.AdminEmail == "" {
+ initcmd.AdminEmail = "admin@example.com"
+ }
+ } else if initcmd.Login == "pam" {
+ initcmd.LoginPAM = true
+ } else if initcmd.Login == "" {
+ // none; login will show an error page
+ } else {
+ err = fmt.Errorf("invalid argument to -login: %q: should be 'test', 'pam', 'google {client-id} {client-secret}', or empty", initcmd.Login)
+ return 1
+ }
+
+ switch initcmd.TLS {
+ case "none", "acme", "insecure":
+ default:
+ if !strings.HasPrefix(initcmd.TLS, "/") {
+ err = fmt.Errorf("invalid argument to -tls: %q; see %s -help", initcmd.TLS, prog)
+ return 1
+ }
+ initcmd.TLSDir = initcmd.TLS
+ }
+
+ confdir := "/etc/arvados"
+ conffile := confdir + "/config.yml"
+ if _, err = os.Stat(conffile); err == nil {
+ err = fmt.Errorf("config file %s already exists; delete it first if you really want to start over", conffile)
+ return 1
+ }
+
+ ports := []int{443}
+ for i := 4440; i < 4460; i++ {
+ ports = append(ports, i)
+ }
+ if initcmd.TLS == "acme" {
+ ports = append(ports, 80)
+ }
+ for _, port := range ports {
+ err = initcmd.checkPort(ctx, fmt.Sprintf("%d", port))
+ if err != nil {
+ return 1
+ }
+ }
+
+ if initcmd.CreateDB {
+ // Do the "create extension" thing early. This way, if
+ // there's no local postgresql server (a likely
+ // failure mode), we can bail out without any side
+ // effects, and the user can start over easily.
+ fmt.Fprintln(stderr, "installing pg_trgm postgresql extension...")
+ cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "--quiet",
+ "-c", `CREATE EXTENSION IF NOT EXISTS pg_trgm`)
+ cmd.Dir = "/"
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("error preparing postgresql server: %w", err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "...done")
+ initcmd.PostgreSQL.Host = "localhost"
+ initcmd.PostgreSQL.User = "arvados"
+ initcmd.PostgreSQL.Password = initcmd.RandomHex(32)
+ initcmd.PostgreSQL.DB = "arvados"
+ } else {
+ initcmd.PostgreSQL.Host = os.Getenv("POSTGRES_HOST")
+ initcmd.PostgreSQL.User = os.Getenv("POSTGRES_USER")
+ initcmd.PostgreSQL.Password = os.Getenv("POSTGRES_PASSWORD")
+ initcmd.PostgreSQL.DB = os.Getenv("POSTGRES_DB")
+ if initcmd.PostgreSQL.Host == "" || initcmd.PostgreSQL.User == "" || initcmd.PostgreSQL.Password == "" || initcmd.PostgreSQL.DB == "" {
+ err = fmt.Errorf("missing $POSTGRES_* env var(s) for -create-db=false; see %s -help", prog)
+ return 1
+ }
+ }
+
wwwuser, err := user.Lookup("www-data")
if err != nil {
err = fmt.Errorf("user.Lookup(%q): %w", "www-data", err)
if err != nil {
return 1
}
- initcmd.PostgreSQLPassword = initcmd.RandomHex(32)
+ fmt.Fprintln(stderr, "creating data storage directory /var/lib/arvados/keep ...")
err = os.Mkdir("/var/lib/arvados/keep", 0600)
if err != nil && !os.IsExist(err) {
err = fmt.Errorf("mkdir /var/lib/arvados/keep: %w", err)
return 1
}
- fmt.Fprintln(stderr, "created /var/lib/arvados/keep")
+ fmt.Fprintln(stderr, "...done")
- err = os.Mkdir("/etc/arvados", 0750)
+ fmt.Fprintln(stderr, "creating config file", conffile, "...")
+ err = os.Mkdir(confdir, 0750)
if err != nil && !os.IsExist(err) {
- err = fmt.Errorf("mkdir /etc/arvados: %w", err)
+ err = fmt.Errorf("mkdir %s: %w", confdir, err)
return 1
}
- err = os.Chown("/etc/arvados", 0, wwwgid)
- f, err := os.OpenFile("/etc/arvados/config.yml", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
+ err = os.Chown(confdir, 0, wwwgid)
if err != nil {
- err = fmt.Errorf("open /etc/arvados/config.yml: %w", err)
+ err = fmt.Errorf("chown 0:%d %s: %w", wwwgid, confdir, err)
+ return 1
+ }
+ f, err := os.OpenFile(conffile+".tmp", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
+ if err != nil {
+ err = fmt.Errorf("open %s: %w", conffile+".tmp", err)
return 1
}
tmpl, err := template.New("config").Parse(`Clusters:
"http://0.0.0.0:9001/": {}
Websocket:
InternalURLs:
- "http://0.0.0.0:9004/": {}
- ExternalURL: {{printf "%q" ( print "wss://" .Domain ":4444/websocket" ) }}
+ "http://0.0.0.0:8005/": {}
+ ExternalURL: {{printf "%q" ( print "wss://" .Domain ":4446/" ) }}
Keepbalance:
InternalURLs:
"http://0.0.0.0:9019/": {}
"http://0.0.0.0:9011/": {}
Collections:
BlobSigningKey: {{printf "%q" ( .RandomHex 50 )}}
- {{if .Insecure}}
+ {{if eq .TLS "insecure"}}
TrustAllContent: true
{{end}}
Containers:
ManagementToken: {{printf "%q" ( .RandomHex 50 )}}
PostgreSQL:
Connection:
- dbname: arvados_production
- host: localhost
- user: arvados
- password: {{printf "%q" .PostgreSQLPassword}}
+ dbname: {{printf "%q" .PostgreSQL.DB}}
+ host: {{printf "%q" .PostgreSQL.Host}}
+ user: {{printf "%q" .PostgreSQL.User}}
+ password: {{printf "%q" .PostgreSQL.Password}}
SystemRootToken: {{printf "%q" ( .RandomHex 50 )}}
- {{if .Insecure}}
TLS:
+ {{if eq .TLS "insecure"}}
Insecure: true
- {{end}}
+ {{else if eq .TLS "acme"}}
+ ACME:
+ Server: LE
+ {{else if ne .TLSDir ""}}
+ Certificate: {{printf "%q" (print .TLSDir "/cert")}}
+ Key: {{printf "%q" (print .TLSDir "/privkey")}}
+ {{else}}
+ {}
+ {{end}}
Volumes:
{{.ClusterID}}-nyw5e-000000000000000:
Driver: Directory
Replication: 2
Workbench:
SecretKeyBase: {{printf "%q" ( .RandomHex 50 )}}
+ {{if .LoginPAM}}
Login:
- {{if eq .Login "pam"}}
PAM:
Enable: true
- {{else if eq .Login "test"}}
+ {{else if .LoginTest}}
+ Login:
Test:
Enable: true
Users:
admin:
- Email: admin@example.com
+ Email: {{printf "%q" .AdminEmail}}
Password: admin
- {{else}}
- {}
- {{end}}
+ {{else if .LoginGoogle}}
+ Login:
+ Google:
+ Enable: true
+ ClientID: {{printf "%q" .LoginGoogleClientID}}
+ ClientSecret: {{printf "%q" .LoginGoogleClientSecret}}
+ {{end}}
Users:
- {{if eq .Login "test"}}
- AutoAdminUserWithEmail: admin@example.com
- {{else}}
- {}
- {{end}}
+ AutoAdminUserWithEmail: {{printf "%q" .AdminEmail}}
`)
if err != nil {
return 1
}
err = tmpl.Execute(f, initcmd)
if err != nil {
- err = fmt.Errorf("/etc/arvados/config.yml: tmpl.Execute: %w", err)
+ err = fmt.Errorf("%s: tmpl.Execute: %w", conffile+".tmp", err)
return 1
}
err = f.Close()
if err != nil {
- err = fmt.Errorf("/etc/arvados/config.yml: close: %w", err)
+ err = fmt.Errorf("%s: close: %w", conffile+".tmp", err)
+ return 1
+ }
+ err = os.Rename(conffile+".tmp", conffile)
+ if err != nil {
+ err = fmt.Errorf("rename %s -> %s: %w", conffile+".tmp", conffile, err)
return 1
}
- fmt.Fprintln(stderr, "created /etc/arvados/config.yml")
+ fmt.Fprintln(stderr, "...done")
ldr := config.NewLoader(nil, logger)
ldr.SkipLegacy = true
+ ldr.Path = conffile // load the file we just wrote, even if $ARVADOS_CONFIG is set
cfg, err := ldr.Load()
if err != nil {
- err = fmt.Errorf("/etc/arvados/config.yml: %w", err)
+ err = fmt.Errorf("%s: %w", conffile, err)
return 1
}
cluster, err := cfg.GetCluster("")
return 1
}
+ fmt.Fprintln(stderr, "creating postresql user and database...")
err = initcmd.createDB(ctx, cluster.PostgreSQL.Connection, stderr)
if err != nil {
return 1
}
+ fmt.Fprintln(stderr, "...done")
+ fmt.Fprintln(stderr, "initializing database...")
cmd := exec.CommandContext(ctx, "sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:"+os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "db:setup")
cmd.Dir = "/var/lib/arvados/railsapi"
cmd.Stdout = stderr
cmd.Stderr = stderr
err = cmd.Run()
if err != nil {
- err = fmt.Errorf("rake db:setup: %w", err)
+ err = fmt.Errorf("rake db:setup failed: %w", err)
return 1
}
- fmt.Fprintln(stderr, "initialized database")
+ fmt.Fprintln(stderr, "...done")
+
+ if initcmd.Start {
+ fmt.Fprintln(stderr, "starting systemd service...")
+ cmd := exec.CommandContext(ctx, "systemctl", "start", "arvados")
+ cmd.Dir = "/"
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("%v: %w", cmd.Args, err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "...done")
+
+ fmt.Fprintln(stderr, "checking controller API endpoint...")
+ u := url.URL(cluster.Services.Controller.ExternalURL)
+ conn := rpc.NewConn(cluster.ClusterID, &u, cluster.TLS.Insecure, rpc.PassthroughTokenProvider)
+ ctx := auth.NewContext(context.Background(), auth.NewCredentials(cluster.SystemRootToken))
+ _, err = conn.UserGetCurrent(ctx, arvados.GetOptions{})
+ if err != nil {
+ err = fmt.Errorf("API request failed: %w", err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "...looks good")
+ }
+
+ if out, err := exec.CommandContext(ctx, "docker", "version").CombinedOutput(); err == nil && strings.Contains(string(out), "\nServer:\n") {
+ fmt.Fprintln(stderr, "loading alpine docker image for diagnostics...")
+ cmd := exec.CommandContext(ctx, "docker", "pull", "alpine")
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("%v: %w", cmd.Args, err)
+ return 1
+ }
+ cmd = exec.CommandContext(ctx, "arv", "sudo", "keep", "docker", "alpine")
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("%v: %w", cmd.Args, err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "...done")
+ } else {
+ fmt.Fprintln(stderr, "docker is not installed -- skipping step of downloading 'alpine' image")
+ }
+
+ fmt.Fprintf(stderr, `
+Setup complete. Next steps:
+* run 'arv sudo diagnostics'
+* log in to workbench2 at %s
+* see documentation at https://doc.arvados.org/install/automatic.html
+`, cluster.Services.Workbench2.ExternalURL.String())
return 0
}
}
func (initcmd *initCommand) createDB(ctx context.Context, dbconn arvados.PostgreSQLConnection, stderr io.Writer) error {
- for _, sql := range []string{
- `CREATE USER ` + pq.QuoteIdentifier(dbconn["user"]) + ` WITH SUPERUSER ENCRYPTED PASSWORD ` + pq.QuoteLiteral(dbconn["password"]),
- `CREATE DATABASE ` + pq.QuoteIdentifier(dbconn["dbname"]) + ` WITH TEMPLATE template0 ENCODING 'utf8'`,
- `CREATE EXTENSION IF NOT EXISTS pg_trgm`,
- } {
- cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "-c", sql)
- cmd.Stdout = stderr
- cmd.Stderr = stderr
- err := cmd.Run()
- if err != nil {
- return fmt.Errorf("error setting up arvados user/database: %w", err)
- }
+ cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "--quiet",
+ "-c", `CREATE USER `+pq.QuoteIdentifier(dbconn["user"])+` WITH SUPERUSER ENCRYPTED PASSWORD `+pq.QuoteLiteral(dbconn["password"]),
+ "-c", `CREATE DATABASE `+pq.QuoteIdentifier(dbconn["dbname"])+` WITH TEMPLATE template0 ENCODING 'utf8'`,
+ )
+ cmd.Dir = "/"
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err := cmd.Run()
+ if err != nil {
+ return fmt.Errorf("error setting up arvados user/database: %w", err)
+ }
+ return nil
+}
+
+// Confirm that http://{initcmd.Domain}:{port} reaches a server that
+// we run on {port}.
+//
+// If port is "80", listening fails, and Nginx appears to be using the
+// debian-packaged default configuration that listens on port 80,
+// disable that Nginx config and try again.
+//
+// (Typically, the reason Nginx is installed is so that Arvados can
+// run an Nginx child process; the default Nginx service using config
+// from /etc/nginx is just an unfortunate side effect of installing
+// Nginx by way of the Debian package.)
+func (initcmd *initCommand) checkPort(ctx context.Context, port string) error {
+ err := initcmd.checkPortOnce(ctx, port)
+ if err == nil || port != "80" {
+ // success, or poking Nginx in the eye won't help
+ return err
+ }
+ d, err2 := os.Open("/etc/nginx/sites-enabled/.")
+ if err2 != nil {
+ return err
+ }
+ fis, err2 := d.Readdir(-1)
+ if err2 != nil || len(fis) != 1 {
+ return err
+ }
+ if target, err2 := os.Readlink("/etc/nginx/sites-enabled/default"); err2 != nil || target != "/etc/nginx/sites-available/default" {
+ return err
+ }
+ err2 = os.Remove("/etc/nginx/sites-enabled/default")
+ if err2 != nil {
+ return err
+ }
+ exec.CommandContext(ctx, "nginx", "-s", "reload").Run()
+ time.Sleep(time.Second)
+ return initcmd.checkPortOnce(ctx, port)
+}
+
+// Start an http server on 0.0.0.0:{port} and confirm that
+// http://{initcmd.Domain}:{port} reaches that server.
+func (initcmd *initCommand) checkPortOnce(ctx context.Context, port string) error {
+ b := make([]byte, 128)
+ _, err := rand.Read(b)
+ if err != nil {
+ return err
+ }
+ token := fmt.Sprintf("%x", b)
+
+ srv := http.Server{
+ Addr: net.JoinHostPort("", port),
+ Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprint(w, token)
+ })}
+ var errServe atomic.Value
+ go func() {
+ errServe.Store(srv.ListenAndServe())
+ }()
+ defer srv.Close()
+ url := "http://" + net.JoinHostPort(initcmd.Domain, port) + "/probe"
+ req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+ if err != nil {
+ return err
+ }
+ resp, err := http.DefaultClient.Do(req)
+ if err == nil {
+ defer resp.Body.Close()
+ }
+ if errServe, _ := errServe.Load().(error); errServe != nil {
+ // If server already exited, return that error
+ // (probably "can't listen"), not the request error.
+ return errServe
+ }
+ if err != nil {
+ return err
+ }
+ buf := make([]byte, len(token))
+ n, err := io.ReadFull(resp.Body, buf)
+ if string(buf[:n]) != token {
+ return fmt.Errorf("listened on port %s but %s connected to something else, returned %q, err %v", port, url, buf[:n], err)
}
return nil
}
})
ctx := ctxlog.Context(c.ctx, logger)
- listenURL, err := getListenAddr(cluster.Services, c.svcName, log)
+ listenURL, internalURL, err := getListenAddr(cluster.Services, c.svcName, log)
if err != nil {
return 1
}
- ctx = context.WithValue(ctx, contextKeyURL{}, listenURL)
+ ctx = context.WithValue(ctx, contextKeyURL{}, internalURL)
reg := prometheus.NewRegistry()
loader.RegisterMetrics(reg)
instrumented := httpserver.Instrument(reg, log,
httpserver.HandlerWithDeadline(cluster.API.RequestTimeout.Duration(),
httpserver.AddRequestIDs(
- httpserver.LogRequests(
- interceptHealthReqs(cluster.ManagementToken, handler.CheckHealth,
- httpserver.NewRequestLimiter(cluster.API.MaxConcurrentRequests, handler, reg))))))
+ httpserver.Inspect(reg, cluster.ManagementToken,
+ httpserver.LogRequests(
+ interceptHealthReqs(cluster.ManagementToken, handler.CheckHealth,
+ httpserver.NewRequestLimiter(cluster.API.MaxConcurrentRequests, handler, reg)))))))
srv := &httpserver.Server{
Server: http.Server{
Handler: ifCollectionInHost(instrumented, instrumented.ServeAPI(cluster.ManagementToken, instrumented)),
},
Addr: listenURL.Host,
}
- if listenURL.Scheme == "https" {
- tlsconfig, err := tlsConfigWithCertUpdater(cluster, logger)
+ if listenURL.Scheme == "https" || listenURL.Scheme == "wss" {
+ tlsconfig, err := makeTLSConfig(cluster, logger)
if err != nil {
logger.WithError(err).Errorf("cannot start %s service on %s", c.svcName, listenURL.String())
return 1
return ifCollectionInHost(next, mux)
}
-func getListenAddr(svcs arvados.Services, prog arvados.ServiceName, log logrus.FieldLogger) (arvados.URL, error) {
+// Determine listenURL (addr:port where server should bind) and
+// internalURL (target url that client should connect to) for a
+// service.
+//
+// If the config does not specify ListenURL, we check all of the
+// configured InternalURLs. If there is exactly one that matches our
+// hostname, or exactly one that matches a local interface address,
+// then we use that as listenURL.
+//
+// Note that listenURL and internalURL may use different protocols
+// (e.g., listenURL is http, but the service sits behind a proxy, so
+// clients connect using https).
+func getListenAddr(svcs arvados.Services, prog arvados.ServiceName, log logrus.FieldLogger) (arvados.URL, arvados.URL, error) {
svc, ok := svcs.Map()[prog]
if !ok {
- return arvados.URL{}, fmt.Errorf("unknown service name %q", prog)
+ return arvados.URL{}, arvados.URL{}, fmt.Errorf("unknown service name %q", prog)
}
- if want := os.Getenv("ARVADOS_SERVICE_INTERNAL_URL"); want == "" {
- } else if url, err := url.Parse(want); err != nil {
- return arvados.URL{}, fmt.Errorf("$ARVADOS_SERVICE_INTERNAL_URL (%q): %s", want, err)
- } else {
+ if want := os.Getenv("ARVADOS_SERVICE_INTERNAL_URL"); want != "" {
+ url, err := url.Parse(want)
+ if err != nil {
+ return arvados.URL{}, arvados.URL{}, fmt.Errorf("$ARVADOS_SERVICE_INTERNAL_URL (%q): %s", want, err)
+ }
if url.Path == "" {
url.Path = "/"
}
- return arvados.URL(*url), nil
+ for internalURL, conf := range svc.InternalURLs {
+ if internalURL.String() == url.String() {
+ listenURL := conf.ListenURL
+ if listenURL.Host == "" {
+ listenURL = internalURL
+ }
+ return listenURL, internalURL, nil
+ }
+ }
+ log.Warnf("possible configuration error: listening on %s (from $ARVADOS_SERVICE_INTERNAL_URL) even though configuration does not have a matching InternalURLs entry", url)
+ internalURL := arvados.URL(*url)
+ return internalURL, internalURL, nil
}
errors := []string{}
- for url := range svc.InternalURLs {
- listener, err := net.Listen("tcp", url.Host)
+ for internalURL, conf := range svc.InternalURLs {
+ listenURL := conf.ListenURL
+ if listenURL.Host == "" {
+ // If ListenURL is not specified, assume
+ // InternalURL is also usable as the listening
+ // proto/addr/port (i.e., simple case with no
+ // intermediate proxy/routing)
+ listenURL = internalURL
+ }
+ listenAddr := listenURL.Host
+ if _, _, err := net.SplitHostPort(listenAddr); err != nil {
+ // url "https://foo.example/" (with no
+ // explicit port name/number) means listen on
+ // the well-known port for the specified
+ // protocol, "foo.example:https".
+ port := listenURL.Scheme
+ if port == "ws" || port == "wss" {
+ port = "http" + port[2:]
+ }
+ listenAddr = net.JoinHostPort(listenAddr, port)
+ }
+ listener, err := net.Listen("tcp", listenAddr)
if err == nil {
listener.Close()
- return url, nil
+ return listenURL, internalURL, nil
} else if strings.Contains(err.Error(), "cannot assign requested address") {
// If 'Host' specifies a different server than
// the current one, it'll resolve the hostname
// can't bind an IP address it doesn't own.
continue
} else {
- errors = append(errors, fmt.Sprintf("tried %v, got %v", url, err))
+ errors = append(errors, fmt.Sprintf("%s: %s", listenURL, err))
}
}
if len(errors) > 0 {
- return arvados.URL{}, fmt.Errorf("could not enable the %q service on this host: %s", prog, strings.Join(errors, "; "))
+ return arvados.URL{}, arvados.URL{}, fmt.Errorf("could not enable the %q service on this host: %s", prog, strings.Join(errors, "; "))
}
- return arvados.URL{}, fmt.Errorf("configuration does not enable the %q service on this host", prog)
+ return arvados.URL{}, arvados.URL{}, fmt.Errorf("configuration does not enable the %q service on this host", prog)
}
type contextKeyURL struct{}
"crypto/tls"
"fmt"
"io/ioutil"
+ "net"
"net/http"
+ "net/url"
"os"
"testing"
"time"
contextKey key = iota
)
+func (*Suite) TestGetListenAddress(c *check.C) {
+ // Find an available port on the testing host, so the test
+ // cases don't get confused by "already in use" errors.
+ listener, err := net.Listen("tcp", ":")
+ c.Assert(err, check.IsNil)
+ _, unusedPort, err := net.SplitHostPort(listener.Addr().String())
+ c.Assert(err, check.IsNil)
+ listener.Close()
+
+ defer os.Unsetenv("ARVADOS_SERVICE_INTERNAL_URL")
+ for idx, trial := range []struct {
+ // internalURL => listenURL, both with trailing "/"
+ // because config loader always adds it
+ internalURLs map[string]string
+ envVar string
+ expectErrorMatch string
+ expectLogsMatch string
+ expectListen string
+ expectInternal string
+ }{
+ {
+ internalURLs: map[string]string{"http://localhost:" + unusedPort + "/": ""},
+ expectListen: "http://localhost:" + unusedPort + "/",
+ expectInternal: "http://localhost:" + unusedPort + "/",
+ },
+ { // implicit port 80 in InternalURLs
+ internalURLs: map[string]string{"http://localhost/": ""},
+ expectErrorMatch: `.*:80: bind: permission denied`,
+ },
+ { // implicit port 443 in InternalURLs
+ internalURLs: map[string]string{"https://host.example/": "http://localhost:" + unusedPort + "/"},
+ expectListen: "http://localhost:" + unusedPort + "/",
+ expectInternal: "https://host.example/",
+ },
+ { // implicit port 443 in ListenURL
+ internalURLs: map[string]string{"wss://host.example/": "wss://localhost/"},
+ expectErrorMatch: `.*:443: bind: permission denied`,
+ },
+ {
+ internalURLs: map[string]string{"https://hostname.example/": "http://localhost:8000/"},
+ expectListen: "http://localhost:8000/",
+ expectInternal: "https://hostname.example/",
+ },
+ {
+ internalURLs: map[string]string{
+ "https://hostname1.example/": "http://localhost:12435/",
+ "https://hostname2.example/": "http://localhost:" + unusedPort + "/",
+ },
+ envVar: "https://hostname2.example", // note this works despite missing trailing "/"
+ expectListen: "http://localhost:" + unusedPort + "/",
+ expectInternal: "https://hostname2.example/",
+ },
+ { // cannot listen on any of the ListenURLs
+ internalURLs: map[string]string{
+ "https://hostname1.example/": "http://1.2.3.4:" + unusedPort + "/",
+ "https://hostname2.example/": "http://1.2.3.4:" + unusedPort + "/",
+ },
+ expectErrorMatch: "configuration does not enable the \"arvados-controller\" service on this host",
+ },
+ { // cannot listen on any of the (implied) ListenURLs
+ internalURLs: map[string]string{
+ "https://1.2.3.4/": "",
+ "https://1.2.3.5/": "",
+ },
+ expectErrorMatch: "configuration does not enable the \"arvados-controller\" service on this host",
+ },
+ { // impossible port number
+ internalURLs: map[string]string{
+ "https://host.example/": "http://0.0.0.0:1234567",
+ },
+ expectErrorMatch: `.*:1234567: listen tcp: address 1234567: invalid port`,
+ },
+ {
+ // env var URL not mentioned in config = obey env var, with warning
+ internalURLs: map[string]string{"https://hostname1.example/": "http://localhost:8000/"},
+ envVar: "https://hostname2.example",
+ expectListen: "https://hostname2.example/",
+ expectInternal: "https://hostname2.example/",
+ expectLogsMatch: `.*\Qpossible configuration error: listening on https://hostname2.example/ (from $ARVADOS_SERVICE_INTERNAL_URL) even though configuration does not have a matching InternalURLs entry\E.*\n`,
+ },
+ {
+ // env var + empty config = obey env var, with warning
+ envVar: "https://hostname.example",
+ expectListen: "https://hostname.example/",
+ expectInternal: "https://hostname.example/",
+ expectLogsMatch: `.*\Qpossible configuration error: listening on https://hostname.example/ (from $ARVADOS_SERVICE_INTERNAL_URL) even though configuration does not have a matching InternalURLs entry\E.*\n`,
+ },
+ } {
+ c.Logf("trial %d %+v", idx, trial)
+ os.Setenv("ARVADOS_SERVICE_INTERNAL_URL", trial.envVar)
+ var logbuf bytes.Buffer
+ log := ctxlog.New(&logbuf, "text", "info")
+ services := arvados.Services{Controller: arvados.Service{InternalURLs: map[arvados.URL]arvados.ServiceInstance{}}}
+ for k, v := range trial.internalURLs {
+ u, err := url.Parse(k)
+ c.Assert(err, check.IsNil)
+ si := arvados.ServiceInstance{}
+ if v != "" {
+ u, err := url.Parse(v)
+ c.Assert(err, check.IsNil)
+ si.ListenURL = arvados.URL(*u)
+ }
+ services.Controller.InternalURLs[arvados.URL(*u)] = si
+ }
+ listenURL, internalURL, err := getListenAddr(services, "arvados-controller", log)
+ if trial.expectLogsMatch != "" {
+ c.Check(logbuf.String(), check.Matches, trial.expectLogsMatch)
+ }
+ if trial.expectErrorMatch != "" {
+ c.Check(err, check.ErrorMatches, trial.expectErrorMatch)
+ continue
+ }
+ if !c.Check(err, check.IsNil) {
+ continue
+ }
+ c.Check(listenURL.String(), check.Equals, trial.expectListen)
+ c.Check(internalURL.String(), check.Equals, trial.expectInternal)
+ }
+}
+
func (*Suite) TestCommand(c *check.C) {
cf, err := ioutil.TempFile("", "cmd_test.")
c.Assert(err, check.IsNil)
package service
import (
+ "context"
"crypto/tls"
"errors"
"fmt"
"os/signal"
"strings"
"syscall"
+ "time"
"git.arvados.org/arvados.git/sdk/go/arvados"
"github.com/sirupsen/logrus"
+ "golang.org/x/crypto/acme/autocert"
)
-func tlsConfigWithCertUpdater(cluster *arvados.Cluster, logger logrus.FieldLogger) (*tls.Config, error) {
+func makeTLSConfig(cluster *arvados.Cluster, logger logrus.FieldLogger) (*tls.Config, error) {
+ if cluster.TLS.ACME.Server != "" {
+ return makeAutocertConfig(cluster, logger)
+ } else {
+ return makeFileLoaderConfig(cluster, logger)
+ }
+}
+
+var errCertUnavailable = errors.New("certificate unavailable, waiting for supervisor to update cache")
+
+type readonlyDirCache autocert.DirCache
+
+func (c readonlyDirCache) Get(ctx context.Context, name string) ([]byte, error) {
+ data, err := autocert.DirCache(c).Get(ctx, name)
+ if err != nil {
+ // Returning an error other than autocert.ErrCacheMiss
+ // causes GetCertificate() to fail early instead of
+ // trying to obtain a certificate itself (which
+ // wouldn't work because we're not in a position to
+ // answer challenges).
+ return nil, errCertUnavailable
+ }
+ return data, nil
+}
+
+func (c readonlyDirCache) Put(ctx context.Context, name string, data []byte) error {
+ return fmt.Errorf("(bug?) (readonlyDirCache)Put(%s) called", name)
+}
+
+func (c readonlyDirCache) Delete(ctx context.Context, name string) error {
+ return nil
+}
+
+func makeAutocertConfig(cluster *arvados.Cluster, logger logrus.FieldLogger) (*tls.Config, error) {
+ mgr := &autocert.Manager{
+ Cache: readonlyDirCache("/var/lib/arvados/tmp/autocert"),
+ Prompt: autocert.AcceptTOS,
+ // HostPolicy accepts all names because this Manager
+ // doesn't request certs. Whoever writes certs to our
+ // cache is effectively responsible for HostPolicy.
+ HostPolicy: func(ctx context.Context, host string) error { return nil },
+ // Keep using whatever's in the cache as long as
+ // possible. Assume some other process (see lib/boot)
+ // handles renewals.
+ RenewBefore: time.Second,
+ }
+ return mgr.TLSConfig(), nil
+}
+
+func makeFileLoaderConfig(cluster *arvados.Cluster, logger logrus.FieldLogger) (*tls.Config, error) {
currentCert := make(chan *tls.Certificate, 1)
loaded := false
- key, cert := cluster.TLS.Key, cluster.TLS.Certificate
- if !strings.HasPrefix(key, "file://") || !strings.HasPrefix(cert, "file://") {
- return nil, errors.New("cannot use TLS certificate: TLS.Key and TLS.Certificate must be specified with a 'file://' prefix")
- }
- key, cert = key[7:], cert[7:]
+ key := strings.TrimPrefix(cluster.TLS.Key, "file://")
+ cert := strings.TrimPrefix(cluster.TLS.Certificate, "file://")
update := func() error {
cert, err := tls.LoadX509KeyPair(cert, key)
return nil, err
}
+ reload := make(chan os.Signal, 1)
+ signal.Notify(reload, syscall.SIGHUP)
+ go func() {
+ for range time.NewTicker(time.Hour).C {
+ reload <- nil
+ }
+ }()
go func() {
- reload := make(chan os.Signal, 1)
- signal.Notify(reload, syscall.SIGHUP)
for range reload {
err := update()
if err != nil {
Certificate string
Key string
Insecure bool
+ ACME struct {
+ Server string
+ }
}
Users struct {
ActivatedUsersAreVisibleToOthers bool
}
type ServiceInstance struct {
+ ListenURL URL
Rendezvous string `json:",omitempty"`
}
mode os.FileMode
size int64
modTime time.Time
+ // If not nil, sys() returns the source data structure, which
+ // can be a *Collection, *Group, or nil. Currently populated
+ // only for project dirs and top-level collection dirs. Does
+ // not stay up to date with upstream changes.
+ //
+ // Intended to support keep-web's properties-as-s3-metadata
+ // feature (https://dev.arvados.org/issues/19088).
+ sys func() interface{}
}
// Name implements os.FileInfo.
return fi.size
}
-// Sys implements os.FileInfo.
+// Sys implements os.FileInfo. See comment in fileinfo struct.
func (fi fileinfo) Sys() interface{} {
- return nil
+ if fi.sys == nil {
+ return nil
+ }
+ return fi.sys()
}
type nullnode struct{}
name: ".",
mode: os.ModeDir | 0755,
modTime: modTime,
+ sys: func() interface{} { return c },
},
inodes: make(map[string]inode),
},
name: coll.Name,
modTime: modTime,
mode: 0755 | os.ModeDir,
+ sys: func() interface{} { return &coll },
},
}
return &deferrednode{wrapped: placeholder, create: func() inode {
{"uuid", "is_a", []string{"arvados#collection", "arvados#group"}},
{"groups.group_class", "=", "project"},
},
+ Select: []string{"uuid", "name", "modified_at", "properties"},
})
if err != nil {
return nil, err
if strings.Contains(coll.UUID, "-j7d0g-") {
// Group item was loaded into a Collection var -- but
// we only need the Name and UUID anyway, so it's OK.
- return fs.newProjectNode(parent, coll.Name, coll.UUID), nil
+ return fs.newProjectNode(parent, coll.Name, coll.UUID, nil), nil
} else if strings.Contains(coll.UUID, "-4zz18-") {
return deferredCollectionFS(fs, parent, coll), nil
} else {
Count: "none",
Filters: filters,
Order: "uuid",
+ Select: []string{"uuid", "name", "modified_at", "properties"},
}
for {
continue
}
if strings.Contains(i.UUID, "-j7d0g-") {
- inodes = append(inodes, fs.newProjectNode(parent, i.Name, i.UUID))
+ inodes = append(inodes, fs.newProjectNode(parent, i.Name, i.UUID, &Group{
+ UUID: i.UUID,
+ Name: i.Name,
+ ModifiedAt: i.ModifiedAt,
+ Properties: i.Properties,
+ }))
} else if strings.Contains(i.UUID, "-4zz18-") {
inodes = append(inodes, deferredCollectionFS(fs, parent, i))
} else {
fs.root.treenode.Lock()
defer fs.root.treenode.Unlock()
fs.root.treenode.Child(mount, func(inode) (inode, error) {
- return fs.newProjectNode(fs.root, mount, uuid), nil
+ return fs.newProjectNode(fs.root, mount, uuid, nil), nil
})
}
if strings.Contains(id, "-4zz18-") || pdhRegexp.MatchString(id) {
return fs.mountCollection(parent, id)
} else if strings.Contains(id, "-j7d0g-") {
- return fs.newProjectNode(fs.root, id, id)
+ return fs.newProjectNode(fs.root, id, id, nil)
} else {
return nil
}
return cfs
}
-func (fs *customFileSystem) newProjectNode(root inode, name, uuid string) inode {
+func (fs *customFileSystem) newProjectNode(root inode, name, uuid string, proj *Group) inode {
+ var projLoading sync.Mutex
return &lookupnode{
stale: fs.Stale,
loadOne: func(parent inode, name string) (inode, error) { return fs.projectsLoadOne(parent, uuid, name) },
name: name,
modTime: time.Now(),
mode: 0755 | os.ModeDir,
+ sys: func() interface{} {
+ projLoading.Lock()
+ defer projLoading.Unlock()
+ if proj != nil {
+ return proj
+ }
+ var g Group
+ err := fs.RequestAndDecode(&g, "GET", "arvados/v1/groups/"+uuid, nil, nil)
+ if err != nil {
+ return err
+ }
+ proj = &g
+ return proj
+ },
},
},
}
return nil, os.ErrNotExist
}
user := resp.Items[0]
- return fs.newProjectNode(parent, user.Username, user.UUID), nil
+ return fs.newProjectNode(parent, user.Username, user.UUID, nil), nil
}
func (fs *customFileSystem) usersLoadAll(parent inode) ([]inode, error) {
if user.Username == "" {
continue
}
- inodes = append(inodes, fs.newProjectNode(parent, user.Username, user.UUID))
+ inodes = append(inodes, fs.newProjectNode(parent, user.Username, user.UUID, nil))
}
params.Filters = []Filter{{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}}
}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package httpserver
+
+import (
+ "encoding/json"
+ "net/http"
+ "sort"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/prometheus/client_golang/prometheus"
+)
+
+// Inspect serves a report of current requests at "GET
+// /_inspect/requests", and passes other requests through to the next
+// handler.
+//
+// If registry is not nil, Inspect registers metrics about current
+// requests.
+func Inspect(registry *prometheus.Registry, authToken string, next http.Handler) http.Handler {
+ type ent struct {
+ startTime time.Time
+ hangupTime atomic.Value
+ }
+ current := map[*http.Request]*ent{}
+ mtx := sync.Mutex{}
+ if registry != nil {
+ registry.MustRegister(prometheus.NewGaugeFunc(
+ prometheus.GaugeOpts{
+ Namespace: "arvados",
+ Name: "max_active_request_age_seconds",
+ Help: "Age of oldest active request",
+ },
+ func() float64 {
+ mtx.Lock()
+ defer mtx.Unlock()
+ earliest := time.Time{}
+ any := false
+ for _, e := range current {
+ if _, ok := e.hangupTime.Load().(time.Time); ok {
+ // Don't count abandoned requests here
+ continue
+ }
+ if !any || e.startTime.Before(earliest) {
+ any = true
+ earliest = e.startTime
+ }
+ }
+ if !any {
+ return 0
+ }
+ return float64(time.Since(earliest).Seconds())
+ },
+ ))
+ registry.MustRegister(prometheus.NewGaugeFunc(
+ prometheus.GaugeOpts{
+ Namespace: "arvados",
+ Name: "max_abandoned_request_age_seconds",
+ Help: "Maximum time since client hung up on a request whose processing thread is still running",
+ },
+ func() float64 {
+ mtx.Lock()
+ defer mtx.Unlock()
+ earliest := time.Time{}
+ any := false
+ for _, e := range current {
+ if hangupTime, ok := e.hangupTime.Load().(time.Time); ok {
+ if !any || hangupTime.Before(earliest) {
+ any = true
+ earliest = hangupTime
+ }
+ }
+ }
+ if !any {
+ return 0
+ }
+ return float64(time.Since(earliest).Seconds())
+ },
+ ))
+ }
+ return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ if req.Method == "GET" && req.URL.Path == "/_inspect/requests" {
+ if authToken == "" || req.Header.Get("Authorization") != "Bearer "+authToken {
+ Error(w, "unauthorized", http.StatusUnauthorized)
+ return
+ }
+ mtx.Lock()
+ defer mtx.Unlock()
+ type outrec struct {
+ RequestID string
+ Method string
+ Host string
+ URL string
+ RemoteAddr string
+ Elapsed float64
+ }
+ now := time.Now()
+ outrecs := []outrec{}
+ for req, e := range current {
+ outrecs = append(outrecs, outrec{
+ RequestID: req.Header.Get(HeaderRequestID),
+ Method: req.Method,
+ Host: req.Host,
+ URL: req.URL.String(),
+ RemoteAddr: req.RemoteAddr,
+ Elapsed: now.Sub(e.startTime).Seconds(),
+ })
+ }
+ sort.Slice(outrecs, func(i, j int) bool { return outrecs[i].Elapsed < outrecs[j].Elapsed })
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(outrecs)
+ } else {
+ e := ent{startTime: time.Now()}
+ mtx.Lock()
+ current[req] = &e
+ mtx.Unlock()
+ go func() {
+ <-req.Context().Done()
+ e.hangupTime.Store(time.Now())
+ }()
+ defer func() {
+ mtx.Lock()
+ defer mtx.Unlock()
+ delete(current, req)
+ }()
+ next.ServeHTTP(w, req)
+ }
+ })
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package httpserver
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "time"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+ check "gopkg.in/check.v1"
+)
+
+func (s *Suite) TestInspect(c *check.C) {
+ reg := prometheus.NewRegistry()
+ h := newTestHandler()
+ mh := Inspect(reg, "abcd", h)
+ handlerReturned := make(chan struct{})
+ reqctx, reqcancel := context.WithCancel(context.Background())
+ longreq := httptest.NewRequest("GET", "/test", nil).WithContext(reqctx)
+ go func() {
+ mh.ServeHTTP(httptest.NewRecorder(), longreq)
+ close(handlerReturned)
+ }()
+ <-h.inHandler
+
+ resp := httptest.NewRecorder()
+ req := httptest.NewRequest("GET", "/_inspect/requests", nil)
+ mh.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
+ c.Check(resp.Body.String(), check.Equals, `{"errors":["unauthorized"]}`+"\n")
+
+ resp = httptest.NewRecorder()
+ req.Header.Set("Authorization", "Bearer abcde")
+ mh.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
+
+ resp = httptest.NewRecorder()
+ req.Header.Set("Authorization", "Bearer abcd")
+ mh.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ reqs := []map[string]interface{}{}
+ err := json.NewDecoder(resp.Body).Decode(&reqs)
+ c.Check(err, check.IsNil)
+ c.Check(reqs, check.HasLen, 1)
+ c.Check(reqs[0]["URL"], check.Equals, "/test")
+
+ // Request is active, so we should see active request age > 0
+ resp = httptest.NewRecorder()
+ mreq := httptest.NewRequest("GET", "/metrics", nil)
+ promhttp.HandlerFor(reg, promhttp.HandlerOpts{}).ServeHTTP(resp, mreq)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*\narvados_max_active_request_age_seconds [0\.]*[1-9][-\d\.e]*\n.*`)
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*\narvados_max_abandoned_request_age_seconds 0\n.*`)
+
+ reqcancel()
+
+ // Request context is canceled but handler hasn't returned, so
+ // we should see max abandoned request age > 0 and active ==
+ // 0. We might need to wait a short time for the cancel to
+ // propagate.
+ for deadline := time.Now().Add(time.Second); time.Now().Before(deadline); time.Sleep(time.Second / 100) {
+ resp = httptest.NewRecorder()
+ promhttp.HandlerFor(reg, promhttp.HandlerOpts{}).ServeHTTP(resp, mreq)
+ c.Assert(resp.Code, check.Equals, http.StatusOK)
+ if strings.Contains(resp.Body.String(), "\narvados_max_active_request_age_seconds 0\n") {
+ break
+ }
+ }
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*\narvados_max_active_request_age_seconds 0\n.*`)
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*\narvados_max_abandoned_request_age_seconds [0\.]*[1-9][-\d\.e]*\n.*`)
+
+ h.okToProceed <- struct{}{}
+ <-handlerReturned
+
+ // Handler has returned, so we should see max abandoned
+ // request age == max active request age == 0
+ resp = httptest.NewRecorder()
+ promhttp.HandlerFor(reg, promhttp.HandlerOpts{}).ServeHTTP(resp, mreq)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*\narvados_max_active_request_age_seconds 0\n.*`)
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*\narvados_max_abandoned_request_age_seconds 0\n.*`)
+
+ // ...and no active requests at the /_monitor endpoint
+ resp = httptest.NewRecorder()
+ mh.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ reqs = nil
+ err = json.NewDecoder(resp.Body).Decode(&reqs)
+ c.Check(err, check.IsNil)
+ c.Assert(reqs, check.HasLen, 0)
+}
// HandlerWithDeadline cancels the request context if the request
// takes longer than the specified timeout without having its
// connection hijacked.
+//
+// If timeout is 0, there is no deadline: HandlerWithDeadline is a
+// no-op.
func HandlerWithDeadline(timeout time.Duration, next http.Handler) http.Handler {
+ if timeout == 0 {
+ return next
+ }
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithCancel(r.Context())
defer cancel()
<-h.okToProceed
}
-func newTestHandler(maxReqs int) *testHandler {
+func newTestHandler() *testHandler {
return &testHandler{
inHandler: make(chan struct{}),
okToProceed: make(chan struct{}),
}
func TestRequestLimiter1(t *testing.T) {
- h := newTestHandler(10)
+ h := newTestHandler()
l := NewRequestLimiter(1, h, nil)
var wg sync.WaitGroup
resps := make([]*httptest.ResponseRecorder, 10)
}
func TestRequestLimiter10(t *testing.T) {
- h := newTestHandler(10)
+ h := newTestHandler()
l := NewRequestLimiter(10, h, nil)
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
import okhttp3.HttpUrl;
import okhttp3.Request;
+import okhttp3.RequestBody;
import org.arvados.client.config.ConfigProvider;
+import java.io.File;
+
public class KeepWebApiClient extends BaseApiClient {
public KeepWebApiClient(ConfigProvider config) {
return newCall(request);
}
+ public String upload(String collectionUuid, File file, ProgressListener progressListener) {
+ RequestBody requestBody = new CountingFileRequestBody(file, progressListener);
+
+ Request request = getRequestBuilder()
+ .url(getUrlBuilder(collectionUuid, file.getName()).build())
+ .put(requestBody)
+ .build();
+ return newCall(request);
+ }
+
private HttpUrl.Builder getUrlBuilder(String collectionUuid, String filePathName) {
return new HttpUrl.Builder()
.scheme(config.getApiProtocol())
@FunctionalInterface
public interface ProgressListener {
- void updateProgress(long num);
+ void updateProgress(long uploadedBytes);
}
\ No newline at end of file
private List<KeepService> keepServices;
private List<KeepService> writableServices;
private Map<String, KeepService> gatewayServices;
- private final String apiToken;
private Integer maxReplicasPerService;
private final ConfigProvider config;
public KeepClient(ConfigProvider config) {
this.config = config;
keepServicesApiClient = new KeepServicesApiClient(config);
- apiToken = config.getApiToken();
}
public byte[] getDataChunk(KeepLocator keepLocator) {
private List<String> mapNewServices(Map<String, FileTransferHandler> rootsMap, KeepLocator locator,
boolean forceRebuild, boolean needWritable, Map<String, String> headers) {
- headers.putIfAbsent("Authorization", String.format("OAuth2 %s", apiToken));
+ headers.putIfAbsent("Authorization", String.format("OAuth2 %s", config.getApiToken()));
List<String> localRoots = weightedServiceRoots(locator, forceRebuild, needWritable);
for (String root : localRoots) {
FileTransferHandler keepServiceLocal = new FileTransferHandler(root, headers, config);
--- /dev/null
+/*
+ * Copyright (C) The Arvados Authors. All rights reserved.
+ *
+ * SPDX-License-Identifier: AGPL-3.0 OR Apache-2.0
+ *
+ */
+
+package org.arvados.client.api.client;
+
+import org.arvados.client.test.utils.ArvadosClientMockedWebServerTest;
+import org.junit.Test;
+
+import java.io.File;
+import java.nio.file.Files;
+
+import static org.arvados.client.test.utils.ApiClientTestUtils.getResponse;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class KeepWebApiClientTest extends ArvadosClientMockedWebServerTest {
+
+ private KeepWebApiClient client = new KeepWebApiClient(CONFIG);
+
+ @Test
+ public void uploadFile() throws Exception {
+ // given
+ String collectionUuid = "112ci-4zz18-p51w7z3fpopo6sm";
+ File file = Files.createTempFile("keep-upload-test", "txt").toFile();
+ Files.write(file.toPath(), "test data".getBytes());
+
+ server.enqueue(getResponse("keep-client-upload-response"));
+
+ // when
+ String uploadResponse = client.upload(collectionUuid, file, uploadedBytes -> System.out.printf("Uploaded bytes: %s/%s%n", uploadedBytes, file.length()));
+
+ // then
+ assertThat(uploadResponse).isEqualTo("Created");
+ }
+
+}
--- /dev/null
+Created
\ No newline at end of file
uwsgi_temp_path "{{TMPDIR}}";
scgi_temp_path "{{TMPDIR}}";
upstream controller {
- server {{LISTENHOST}}:{{CONTROLLERPORT}};
+ server {{UPSTREAMHOST}}:{{CONTROLLERPORT}};
}
server {
listen {{LISTENHOST}}:{{CONTROLLERSSLPORT}} ssl;
}
}
upstream arv-git-http {
- server {{LISTENHOST}}:{{GITPORT}};
+ server {{UPSTREAMHOST}}:{{GITPORT}};
}
server {
listen {{LISTENHOST}}:{{GITSSLPORT}} ssl;
}
}
upstream keepproxy {
- server {{LISTENHOST}}:{{KEEPPROXYPORT}};
+ server {{UPSTREAMHOST}}:{{KEEPPROXYPORT}};
}
server {
listen {{LISTENHOST}}:{{KEEPPROXYSSLPORT}} ssl;
}
}
upstream keep-web {
- server {{LISTENHOST}}:{{KEEPWEBPORT}};
+ server {{UPSTREAMHOST}}:{{KEEPWEBPORT}};
}
server {
listen {{LISTENHOST}}:{{KEEPWEBSSLPORT}} ssl;
}
}
upstream health {
- server {{LISTENHOST}}:{{HEALTHPORT}};
+ server {{UPSTREAMHOST}}:{{HEALTHPORT}};
}
server {
listen {{LISTENHOST}}:{{HEALTHSSLPORT}} ssl;
}
}
upstream ws {
- server {{LISTENHOST}}:{{WSPORT}};
+ server {{UPSTREAMHOST}}:{{WSPORT}};
}
server {
listen {{LISTENHOST}}:{{WSSSLPORT}} ssl;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_redirect off;
+
+ client_max_body_size 0;
+ proxy_http_version 1.1;
+ proxy_request_buffering off;
}
}
upstream workbench1 {
- server {{LISTENHOST}}:{{WORKBENCH1PORT}};
+ server {{UPSTREAMHOST}}:{{WORKBENCH1PORT}};
}
server {
listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl;
}
}
upstream workbench2 {
- server {{LISTENHOST}}:{{WORKBENCH2PORT}};
+ server {{UPSTREAMHOST}}:{{WORKBENCH2PORT}};
}
server {
listen {{LISTENHOST}}:{{WORKBENCH2SSLPORT}} ssl;
return
stop_nginx()
nginxconf = {}
+ nginxconf['UPSTREAMHOST'] = 'localhost'
nginxconf['LISTENHOST'] = 'localhost'
nginxconf['CONTROLLERPORT'] = internal_port_from_config("Controller")
nginxconf['ARVADOS_API_HOST'] = "0.0.0.0:" + str(external_port_from_config("Controller"))
GEM
remote: https://rubygems.org/
specs:
- actioncable (5.2.8)
- actionpack (= 5.2.8)
+ actioncable (5.2.8.1)
+ actionpack (= 5.2.8.1)
nio4r (~> 2.0)
websocket-driver (>= 0.6.1)
- actionmailer (5.2.8)
- actionpack (= 5.2.8)
- actionview (= 5.2.8)
- activejob (= 5.2.8)
+ actionmailer (5.2.8.1)
+ actionpack (= 5.2.8.1)
+ actionview (= 5.2.8.1)
+ activejob (= 5.2.8.1)
mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 2.0)
- actionpack (5.2.8)
- actionview (= 5.2.8)
- activesupport (= 5.2.8)
+ actionpack (5.2.8.1)
+ actionview (= 5.2.8.1)
+ activesupport (= 5.2.8.1)
rack (~> 2.0, >= 2.0.8)
rack-test (>= 0.6.3)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.2)
- actionview (5.2.8)
- activesupport (= 5.2.8)
+ actionview (5.2.8.1)
+ activesupport (= 5.2.8.1)
builder (~> 3.1)
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.3)
- activejob (5.2.8)
- activesupport (= 5.2.8)
+ activejob (5.2.8.1)
+ activesupport (= 5.2.8.1)
globalid (>= 0.3.6)
- activemodel (5.2.8)
- activesupport (= 5.2.8)
- activerecord (5.2.8)
- activemodel (= 5.2.8)
- activesupport (= 5.2.8)
+ activemodel (5.2.8.1)
+ activesupport (= 5.2.8.1)
+ activerecord (5.2.8.1)
+ activemodel (= 5.2.8.1)
+ activesupport (= 5.2.8.1)
arel (>= 9.0)
- activestorage (5.2.8)
- actionpack (= 5.2.8)
- activerecord (= 5.2.8)
+ activestorage (5.2.8.1)
+ actionpack (= 5.2.8.1)
+ activerecord (= 5.2.8.1)
marcel (~> 1.0.0)
- activesupport (5.2.8)
+ activesupport (5.2.8.1)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 0.7, < 2)
minitest (~> 5.1)
multi_json (1.15.0)
multipart-post (2.1.1)
nio4r (2.5.8)
- nokogiri (1.13.6)
+ nokogiri (1.13.7)
mini_portile2 (~> 2.8.0)
racc (~> 1.4)
oj (3.9.2)
power_assert (1.1.4)
public_suffix (4.0.6)
racc (1.6.0)
- rack (2.2.3.1)
- rack-test (1.1.0)
- rack (>= 1.0, < 3)
- rails (5.2.8)
- actioncable (= 5.2.8)
- actionmailer (= 5.2.8)
- actionpack (= 5.2.8)
- actionview (= 5.2.8)
- activejob (= 5.2.8)
- activemodel (= 5.2.8)
- activerecord (= 5.2.8)
- activestorage (= 5.2.8)
- activesupport (= 5.2.8)
+ rack (2.2.4)
+ rack-test (2.0.2)
+ rack (>= 1.3)
+ rails (5.2.8.1)
+ actioncable (= 5.2.8.1)
+ actionmailer (= 5.2.8.1)
+ actionpack (= 5.2.8.1)
+ actionview (= 5.2.8.1)
+ activejob (= 5.2.8.1)
+ activemodel (= 5.2.8.1)
+ activerecord (= 5.2.8.1)
+ activestorage (= 5.2.8.1)
+ activesupport (= 5.2.8.1)
bundler (>= 1.3.0)
- railties (= 5.2.8)
+ railties (= 5.2.8.1)
sprockets-rails (>= 2.0.0)
rails-controller-testing (1.0.4)
actionpack (>= 5.0.1.x)
rails-observers (0.1.5)
activemodel (>= 4.0)
rails-perftest (0.0.7)
- railties (5.2.8)
- actionpack (= 5.2.8)
- activesupport (= 5.2.8)
+ railties (5.2.8.1)
+ actionpack (= 5.2.8.1)
+ activesupport (= 5.2.8.1)
method_source
rake (>= 0.8.7)
thor (>= 0.19.0, < 2.0)
power_assert
thor (1.2.1)
thread_safe (0.3.6)
- tzinfo (1.2.9)
+ tzinfo (1.2.10)
thread_safe (~> 0.1)
websocket-driver (0.7.5)
websocket-extensions (>= 0.1.0)
skip_before_action :find_object_by_uuid
skip_before_action :render_404_if_no_object
before_action :admin_required
+ around_action :silence_logs, only: [:reset]
+
def reset
raise ArvadosModel::PermissionDeniedError unless Rails.env == 'test'
unexpected_uuids = user_uuids - fixture_uuids
if unexpected_uuids.any?
logger.error("Running in test environment, but non-fixture users exist: " +
- "#{unexpected_uuids}")
+ "#{unexpected_uuids}" + "\nMaybe test users without @example.com email addresses were created?")
raise ArvadosModel::PermissionDeniedError
end
# Done.
send_json success: true
end
+
+ protected
+
+ def silence_logs
+ Rails.logger.info("(logging level temporarily raised to :error, see #{__FILE__})")
+ orig = ActiveRecord::Base.logger.level
+ ActiveRecord::Base.logger.level = :error
+ begin
+ yield
+ ensure
+ ActiveRecord::Base.logger.level = orig
+ end
+ end
end
validate :identity_url_nil_if_empty
before_update :prevent_privilege_escalation
before_update :prevent_inactive_admin
+ before_update :prevent_nonadmin_system_root
before_update :verify_repositories_empty, :if => Proc.new {
username.nil? and username_changed?
}
# delete user signatures, login, repo, and vm perms, and mark as inactive
def unsetup
+ if self.uuid == system_user_uuid
+ raise "System root user cannot be deactivated"
+ end
+
# delete oid_login_perms for this user
#
# note: these permission links are obsolete, they have no effect
self.save!
end
+ # Called from ArvadosModel
+ def set_default_owner
+ self.owner_uuid = system_user_uuid
+ end
+
def must_unsetup_to_deactivate
if !self.new_record? &&
self.uuid[0..4] == Rails.configuration.Login.LoginCluster &&
true
end
+ def prevent_nonadmin_system_root
+ if self.uuid == system_user_uuid and self.is_admin_changed? and !self.is_admin
+ raise "System root user cannot be non-admin"
+ end
+ true
+ end
+
def search_permissions(start, graph, merged={}, upstream_mask=nil, upstream_path={})
nextpaths = graph[start]
return merged if !nextpaths
container_runtime_token_user:
owner_uuid: zzzzz-tpzed-000000000000000
uuid: zzzzz-tpzed-l3skomkti0c4vg4
- email: spectator@arvados.local
+ email: container_runtime_token_user@arvados.local
first_name: Spect
last_name: Ator
identity_url: https://container_runtime_token_user.openid.local
identity_url: https://inactive-uninvited-user.openid.local
is_active: false
is_admin: false
+ username: inactiveuninvited
prefs: {}
inactive:
identity_url: https://inactive-but-agreeable-user.openid.local
is_active: false
is_admin: false
+ username: inactiveusersignedua
prefs:
profile:
organization: example.com
identity_url: https://active_no_prefs.openid.local
is_active: true
is_admin: false
+ username: activenoprefs
prefs: {}
active_no_prefs_profile_no_getting_started_shown:
owner_uuid: zzzzz-tpzed-000000000000000
uuid: zzzzz-tpzed-a46c98d1td4aoj4
- email: active_no_prefs_profile@arvados.local
+ email: active_no_prefs_profile_no_gs@arvados.local
first_name: HasPrefs
last_name: NoProfile
identity_url: https://active_no_prefs_profile.openid.local
is_active: true
is_admin: false
+ username: activenoprefsprofilenogs
prefs:
test: abc
identity_url: https://active_no_prefs_profile_seen_gs.openid.local
is_active: true
is_admin: false
+ username: activenoprefsprofile
prefs:
test: abc
getting_started_shown: 2015-03-26 12:34:56.789000000 Z
last_name: NoGettingStartedShown
identity_url: https://active_nogettinstarted.openid.local
is_active: true
+ username: activenogettinstarted
prefs:
profile:
organization: example.com
permission_perftest:
owner_uuid: zzzzz-tpzed-000000000000000
uuid: zzzzz-tpzed-permissionptest
- email: fuse@arvados.local
+ email: permission_perftest@arvados.local
first_name: FUSE
last_name: User
identity_url: https://permission_perftest.openid.local
is_active: true
is_admin: false
modified_at: 2015-03-26 12:34:56.789000000 Z
- username: can-login-user
+ username: canLoginUser
assert_response 403
end
+ test "disabling system root user not permitted" do
+ put("/arvados/v1/users/#{users(:system_user).uuid}",
+ params: {
+ user: {is_admin: false}
+ },
+ headers: auth(:admin))
+ assert_response 422
+
+ post("/arvados/v1/users/#{users(:system_user).uuid}/unsetup",
+ params: {},
+ headers: auth(:admin))
+ assert_response 422
+ end
+
+ test "creating users only accepted for admins" do
+ assert_equal false, users(:active).is_admin
+ post '/arvados/v1/users',
+ params: {
+ "user" => {
+ "email" => 'foo@example.com',
+ "username" => "barney"
+ }
+ },
+ headers: auth(:active)
+ assert_response 403
+ end
+
+ test "create users assigns the system root user as their owner" do
+ post '/arvados/v1/users',
+ params: {
+ "user" => {
+ "email" => 'foo@example.com',
+ "username" => "barney"
+ }
+ },
+ headers: auth(:admin)
+ assert_response :success
+ assert_not_nil json_response["uuid"]
+ assert_equal users(:system_user).uuid, json_response["owner_uuid"]
+ end
+
+ test "create users ignores provided owner_uuid field" do
+ assert_equal false, users(:admin).uuid == users(:system_user).uuid
+ post '/arvados/v1/users',
+ params: {
+ "user" => {
+ "email" => 'foo@example.com',
+ "owner_uuid" => users(:admin).uuid,
+ "username" => "barney"
+ }
+ },
+ headers: auth(:admin)
+ assert_response :success
+ assert_not_nil json_response["uuid"]
+ assert_equal users(:system_user).uuid, json_response["owner_uuid"]
+ end
end
}
}
// The client's token was invalid (e.g., expired), or
- // the client didn't even provide one. Propagate the
- // 401 to encourage the client to use a [different]
- // token.
+ // the client didn't even provide one. Redirect to
+ // workbench2's login-and-redirect-to-download url if
+ // this is a browser navigation request. (The redirect
+ // flow can't preserve the original method if it's not
+ // GET, and doesn't make sense if the UA is a
+ // command-line tool, is trying to load an inline
+ // image, etc.; in these cases, there's nothing we can
+ // do, so return 401 unauthorized.)
+ //
+ // Note Sec-Fetch-Mode is sent by all non-EOL
+ // browsers, except Safari.
+ // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Sec-Fetch-Mode
//
// TODO(TC): This response would be confusing to
// someone trying (anonymously) to download public
// data that has been deleted. Allow a referrer to
// provide this context somehow?
- w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
- http.Error(w, unauthorizedMessage, http.StatusUnauthorized)
+ if r.Method == http.MethodGet && r.Header.Get("Sec-Fetch-Mode") == "navigate" {
+ target := url.URL(h.Cluster.Services.Workbench2.ExternalURL)
+ redirkey := "redirectToPreview"
+ if attachment {
+ redirkey = "redirectToDownload"
+ }
+ callback := "/c=" + collectionID + "/" + strings.Join(targetPath, "/")
+ // target.RawQuery = url.Values{redirkey:
+ // {target}}.Encode() would be the obvious
+ // thing to do here, but wb2 doesn't decode
+ // this as a query param -- it takes
+ // everything after "${redirkey}=" as the
+ // target URL. If we encode "/" as "%2F" etc.,
+ // the redirect won't work.
+ target.RawQuery = redirkey + "=" + callback
+ w.Header().Add("Location", target.String())
+ w.WriteHeader(http.StatusSeeOther)
+ } else {
+ w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
+ http.Error(w, unauthorizedMessage, http.StatusUnauthorized)
+ }
return
}
s.testVhostRedirectTokenToCookie(c, "GET",
arvadostest.FooCollection+".example.com/foo",
"?api_token="+arvadostest.ActiveToken,
- "",
+ nil,
"",
http.StatusOK,
"foo",
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
"",
- "",
+ nil,
"",
http.StatusOK,
"foo",
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
"",
- "",
+ nil,
"",
http.StatusNotFound,
notFoundMessage+"\n",
}
// Bad token in a cookie (even if it got there via our own
-// query-string-to-cookie redirect) is, in principle, retryable at the
-// same URL so it's 401 Unauthorized.
+// query-string-to-cookie redirect) is, in principle, retryable via
+// wb2-login-and-redirect flow.
func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
- s.testVhostRedirectTokenToCookie(c, "GET",
+ // Inline
+ resp := s.testVhostRedirectTokenToCookie(c, "GET",
+ arvadostest.FooCollection+".example.com/foo",
+ "?api_token=thisisabogustoken",
+ http.Header{"Sec-Fetch-Mode": {"navigate"}},
+ "",
+ http.StatusSeeOther,
+ "",
+ )
+ u, err := url.Parse(resp.Header().Get("Location"))
+ c.Assert(err, check.IsNil)
+ c.Logf("redirected to %s", u)
+ c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
+ c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
+ c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
+
+ // Download/attachment indicated by ?disposition=attachment
+ resp = s.testVhostRedirectTokenToCookie(c, "GET",
arvadostest.FooCollection+".example.com/foo",
+ "?api_token=thisisabogustoken&disposition=attachment",
+ http.Header{"Sec-Fetch-Mode": {"navigate"}},
+ "",
+ http.StatusSeeOther,
+ "",
+ )
+ u, err = url.Parse(resp.Header().Get("Location"))
+ c.Assert(err, check.IsNil)
+ c.Logf("redirected to %s", u)
+ c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
+ c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
+ c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
+
+ // Download/attachment indicated by vhost
+ resp = s.testVhostRedirectTokenToCookie(c, "GET",
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
"?api_token=thisisabogustoken",
+ http.Header{"Sec-Fetch-Mode": {"navigate"}},
"",
+ http.StatusSeeOther,
+ "",
+ )
+ u, err = url.Parse(resp.Header().Get("Location"))
+ c.Assert(err, check.IsNil)
+ c.Logf("redirected to %s", u)
+ c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
+ c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
+ c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
+
+ // Without "Sec-Fetch-Mode: navigate" header, just 401.
+ s.testVhostRedirectTokenToCookie(c, "GET",
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
+ "?api_token=thisisabogustoken",
+ http.Header{"Sec-Fetch-Mode": {"cors"}},
+ "",
+ http.StatusUnauthorized,
+ unauthorizedMessage+"\n",
+ )
+ s.testVhostRedirectTokenToCookie(c, "GET",
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
+ "?api_token=thisisabogustoken",
+ nil,
"",
http.StatusUnauthorized,
unauthorizedMessage+"\n",
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
- "",
+ nil,
"",
http.StatusBadRequest,
"cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
resp := s.testVhostRedirectTokenToCookie(c, "GET",
arvadostest.FooCollection+".example.com/foo",
"?disposition=attachment&api_token="+arvadostest.ActiveToken,
- "",
+ nil,
"",
http.StatusOK,
"foo",
resp := s.testVhostRedirectTokenToCookie(c, "GET",
"download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
- "",
+ nil,
"",
http.StatusOK,
"foo",
resp := s.testVhostRedirectTokenToCookie(c, "GET",
"download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
"?api_token="+arvadostest.ActiveToken,
- "",
+ nil,
"",
http.StatusOK,
"waz",
resp = s.testVhostRedirectTokenToCookie(c, "GET",
"download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
"?api_token="+arvadostest.ActiveToken,
- "",
+ nil,
"",
http.StatusOK,
"waz",
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
- "",
+ nil,
"",
http.StatusOK,
"foo",
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
- "",
+ nil,
"",
http.StatusBadRequest,
"cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
resp := s.testVhostRedirectTokenToCookie(c, "GET",
"example.com:1234/c="+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
- "",
+ nil,
"",
http.StatusOK,
"foo",
s.testVhostRedirectTokenToCookie(c, "POST",
arvadostest.FooCollection+".example.com/foo",
"",
- "application/x-www-form-urlencoded",
+ http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
http.StatusOK,
"foo",
s.testVhostRedirectTokenToCookie(c, "POST",
arvadostest.FooCollection+".example.com/foo",
"",
- "application/x-www-form-urlencoded",
+ http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
http.StatusNotFound,
notFoundMessage+"\n",
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
"",
- "",
+ nil,
"",
http.StatusOK,
"Hello world\n",
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
"",
- "",
+ nil,
"",
http.StatusNotFound,
notFoundMessage+"\n",
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
}
-func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
+func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
+ if reqHeader == nil {
+ reqHeader = http.Header{}
+ }
u, _ := url.Parse(`http://` + hostPath + queryString)
+ c.Logf("requesting %s", u)
req := &http.Request{
Method: method,
Host: u.Host,
URL: u,
RequestURI: u.RequestURI(),
- Header: http.Header{"Content-Type": {contentType}},
+ Header: reqHeader,
Body: ioutil.NopCloser(strings.NewReader(reqBody)),
}
return resp
}
c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
+ c.Check(strings.Split(resp.Header().Get("Location"), "?")[0], check.Equals, "http://"+hostPath)
cookies := (&http.Response{Header: resp.Header()}).Cookies()
- u, _ = u.Parse(resp.Header().Get("Location"))
+ u, err := u.Parse(resp.Header().Get("Location"))
+ c.Assert(err, check.IsNil)
+ c.Logf("following redirect to %s", u)
req = &http.Request{
Method: "GET",
Host: u.Host,
URL: u,
RequestURI: u.RequestURI(),
- Header: http.Header{},
+ Header: reqHeader,
}
for _, c := range cookies {
req.AddCookie(c)
resp = httptest.NewRecorder()
s.handler.ServeHTTP(resp, req)
- c.Check(resp.Header().Get("Location"), check.Equals, "")
+
+ if resp.Code != http.StatusSeeOther {
+ c.Check(resp.Header().Get("Location"), check.Equals, "")
+ }
return resp
}
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
+ "encoding/json"
"encoding/xml"
"errors"
"fmt"
"hash"
"io"
+ "mime"
"net/http"
+ "net/textproto"
"net/url"
"os"
"path/filepath"
if r.Method == "HEAD" && !objectNameGiven {
// HeadBucket
if err == nil && fi.IsDir() {
+ err = setFileInfoHeaders(w.Header(), fs, fspath)
+ if err != nil {
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusBadGateway)
+ return true
+ }
w.WriteHeader(http.StatusOK)
} else if os.IsNotExist(err) {
s3ErrorResponse(w, NoSuchBucket, "The specified bucket does not exist.", r.URL.Path, http.StatusNotFound)
return true
}
if err == nil && fi.IsDir() && objectNameGiven && strings.HasSuffix(fspath, "/") && h.Cluster.Collections.S3FolderObjects {
+ err = setFileInfoHeaders(w.Header(), fs, fspath)
+ if err != nil {
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusBadGateway)
+ return true
+ }
w.Header().Set("Content-Type", "application/x-directory")
w.WriteHeader(http.StatusOK)
return true
// shallow copy r, and change URL path
r := *r
r.URL.Path = fspath
+ err = setFileInfoHeaders(w.Header(), fs, fspath)
+ if err != nil {
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusBadGateway)
+ return true
+ }
http.FileServer(fs).ServeHTTP(w, &r)
return true
case r.Method == http.MethodPut:
}
}
+func setFileInfoHeaders(header http.Header, fs arvados.CustomFileSystem, path string) error {
+ maybeEncode := func(s string) string {
+ for _, c := range s {
+ if c > '\u007f' || c < ' ' {
+ return mime.BEncoding.Encode("UTF-8", s)
+ }
+ }
+ return s
+ }
+ path = strings.TrimSuffix(path, "/")
+ var props map[string]interface{}
+ for {
+ fi, err := fs.Stat(path)
+ if err != nil {
+ return err
+ }
+ switch src := fi.Sys().(type) {
+ case *arvados.Collection:
+ props = src.Properties
+ case *arvados.Group:
+ props = src.Properties
+ default:
+ if err, ok := src.(error); ok {
+ return err
+ }
+ // Try parent
+ cut := strings.LastIndexByte(path, '/')
+ if cut < 0 {
+ return nil
+ }
+ path = path[:cut]
+ continue
+ }
+ break
+ }
+ for k, v := range props {
+ if !validMIMEHeaderKey(k) {
+ continue
+ }
+ k = "x-amz-meta-" + k
+ if s, ok := v.(string); ok {
+ header.Set(k, maybeEncode(s))
+ } else if j, err := json.Marshal(v); err == nil {
+ header.Set(k, maybeEncode(string(j)))
+ }
+ }
+ return nil
+}
+
+func validMIMEHeaderKey(k string) bool {
+ check := "z-" + k
+ return check != textproto.CanonicalMIMEHeaderKey(check)
+}
+
// Call fn on the given path (directory) and its contents, in
// lexicographic order.
//
"crypto/sha256"
"fmt"
"io/ioutil"
+ "mime"
"net/http"
"net/http/httptest"
"net/url"
kc *keepclient.KeepClient
proj arvados.Group
projbucket *s3.Bucket
+ subproj arvados.Group
coll arvados.Collection
collbucket *s3.Bucket
}
func (s *IntegrationSuite) s3setup(c *check.C) s3stage {
- var proj arvados.Group
+ var proj, subproj arvados.Group
var coll arvados.Collection
arv := arvados.NewClientFromEnv()
arv.AuthToken = arvadostest.ActiveToken
"group": map[string]interface{}{
"group_class": "project",
"name": "keep-web s3 test",
+ "properties": map[string]interface{}{
+ "project-properties-key": "project properties value",
+ },
},
"ensure_unique_name": true,
})
c.Assert(err, check.IsNil)
+ err = arv.RequestAndDecode(&subproj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
+ "group": map[string]interface{}{
+ "owner_uuid": proj.UUID,
+ "group_class": "project",
+ "name": "keep-web s3 test subproject",
+ "properties": map[string]interface{}{
+ "subproject_properties_key": "subproject properties value",
+ "invalid header key": "this value will not be returned because key contains spaces",
+ },
+ },
+ })
+ c.Assert(err, check.IsNil)
err = arv.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{
"owner_uuid": proj.UUID,
"name": "keep-web s3 test collection",
"manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:emptyfile\n./emptydir d41d8cd98f00b204e9800998ecf8427e+0 0:0:.\n",
+ "properties": map[string]interface{}{
+ "string": "string value",
+ "array": []string{"element1", "element2"},
+ "object": map[string]interface{}{"key": map[string]interface{}{"key2": "value⛵"}},
+ "nonascii": "⛵",
+ "newline": "foo\r\nX-Bad: header",
+ // This key cannot be expressed as a MIME
+ // header key, so it will be silently skipped
+ // (see "Inject" in PropertiesAsMetadata test)
+ "a: a\r\nInject": "bogus",
+ },
}})
c.Assert(err, check.IsNil)
ac, err := arvadosclient.New(arv)
S3: client,
Name: proj.UUID,
},
- coll: coll,
+ subproj: subproj,
+ coll: coll,
collbucket: &s3.Bucket{
S3: client,
Name: coll.UUID,
c.Check(exists, check.Equals, true)
}
+func (s *IntegrationSuite) checkMetaEquals(c *check.C, hdr http.Header, expect map[string]string) {
+ got := map[string]string{}
+ for hk, hv := range hdr {
+ if k := strings.TrimPrefix(hk, "X-Amz-Meta-"); k != hk && len(hv) == 1 {
+ got[k] = hv[0]
+ }
+ }
+ c.Check(got, check.DeepEquals, expect)
+}
+
+func (s *IntegrationSuite) TestS3PropertiesAsMetadata(c *check.C) {
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+
+ expectCollectionTags := map[string]string{
+ "String": "string value",
+ "Array": `["element1","element2"]`,
+ "Object": mime.BEncoding.Encode("UTF-8", `{"key":{"key2":"value⛵"}}`),
+ "Nonascii": "=?UTF-8?b?4pu1?=",
+ "Newline": mime.BEncoding.Encode("UTF-8", "foo\r\nX-Bad: header"),
+ }
+ expectSubprojectTags := map[string]string{
+ "Subproject_properties_key": "subproject properties value",
+ }
+ expectProjectTags := map[string]string{
+ "Project-Properties-Key": "project properties value",
+ }
+
+ c.Log("HEAD object with metadata from collection")
+ resp, err := stage.collbucket.Head("sailboat.txt", nil)
+ c.Assert(err, check.IsNil)
+ s.checkMetaEquals(c, resp.Header, expectCollectionTags)
+
+ c.Log("GET object with metadata from collection")
+ rdr, hdr, err := stage.collbucket.GetReaderWithHeaders("sailboat.txt")
+ c.Assert(err, check.IsNil)
+ content, err := ioutil.ReadAll(rdr)
+ c.Check(err, check.IsNil)
+ rdr.Close()
+ c.Check(content, check.HasLen, 4)
+ s.checkMetaEquals(c, hdr, expectCollectionTags)
+ c.Check(hdr["Inject"], check.IsNil)
+
+ c.Log("HEAD bucket with metadata from collection")
+ resp, err = stage.collbucket.Head("/", nil)
+ c.Assert(err, check.IsNil)
+ s.checkMetaEquals(c, resp.Header, expectCollectionTags)
+
+ c.Log("HEAD directory placeholder with metadata from collection")
+ resp, err = stage.projbucket.Head("keep-web s3 test collection/", nil)
+ c.Assert(err, check.IsNil)
+ s.checkMetaEquals(c, resp.Header, expectCollectionTags)
+
+ c.Log("HEAD file with metadata from collection")
+ resp, err = stage.projbucket.Head("keep-web s3 test collection/sailboat.txt", nil)
+ c.Assert(err, check.IsNil)
+ s.checkMetaEquals(c, resp.Header, expectCollectionTags)
+
+ c.Log("HEAD directory placeholder with metadata from subproject")
+ resp, err = stage.projbucket.Head("keep-web s3 test subproject/", nil)
+ c.Assert(err, check.IsNil)
+ s.checkMetaEquals(c, resp.Header, expectSubprojectTags)
+
+ c.Log("HEAD bucket with metadata from project")
+ resp, err = stage.projbucket.Head("/", nil)
+ c.Assert(err, check.IsNil)
+ s.checkMetaEquals(c, resp.Header, expectProjectTags)
+}
+
func (s *IntegrationSuite) TestS3CollectionPutObjectSuccess(c *check.C) {
stage := s.s3setup(c)
defer stage.teardown(c)
require 'fileutils'
require 'yaml'
require 'optparse'
+require 'open3'
req_envs = %w(ARVADOS_API_HOST ARVADOS_API_TOKEN ARVADOS_VIRTUAL_MACHINE_UUID)
req_envs.each do |k|
unless pwnam[l[:username]]
STDERR.puts "Creating account #{l[:username]}"
# Create new user
- unless system("useradd", "-m",
+ out, st = Open3.capture2e("useradd", "-m",
"-c", username,
"-s", "/bin/bash",
username)
- STDERR.puts "Account creation failed for #{l[:username]}: #{$?}"
+ if st.exitstatus != 0
+ STDERR.puts "Account creation failed for #{l[:username]}:\n#{out}"
next
end
begin
if existing_groups.index(addgroup).nil?
# User should be in group, but isn't, so add them.
STDERR.puts "Add user #{username} to #{addgroup} group"
- system("usermod", "-aG", addgroup, username)
+ out, st = Open3.capture2e("usermod", "-aG", addgroup, username)
+ if st.exitstatus != 0
+ STDERR.puts "Failed to add #{username} to #{addgroup} group:\n#{out}"
+ end
end
end
if groups.index(removegroup).nil?
# User is in a group, but shouldn't be, so remove them.
STDERR.puts "Remove user #{username} from #{removegroup} group"
- system("gpasswd", "-d", username, removegroup)
+ out, st = Open3.capture2e("gpasswd", "-d", username, removegroup)
+ if st.exitstatus != 0
+ STDERR.puts "Failed to remove user #{username} from #{removegroup} group:\n#{out}"
+ end
end
end
resources:
virtual_machines:
shell:
- name: shell
+ name: shell.__CLUSTER__.__DOMAIN__
backend: __SHELL_INT_IP__
port: 4200
### STREAMS
http:
upstream webshell_upstream:
- - server: 'localhost:4200 fail_timeout=10s'
+ - server: 'shell.__CLUSTER__.__DOMAIN__:4200 fail_timeout=10s'
### SITES
servers:
users:
__CLUSTER___arvados:
ensure: present
- password: __DATABASE_PASSWORD__
+ password: "__DATABASE_PASSWORD__"
# tablespaces:
# arvados_tablespace:
- onlyif:
- /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+extra_shell_cron_add_login_sync_add_{{ vm }}_sbin_to_path_cron_env_present:
+ cron.env_present:
+ - name: PATH
+ - value: "/bin:/usr/bin:/usr/sbin"
+ - onlyif:
+ - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_login_sync_cron_present:
cron.present:
- name: /usr/local/bin/arvados-login-sync
- add_header: 'Strict-Transport-Security "max-age=63072000" always'
# OCSP stapling
- # FIXME! Stapling does not work with self-signed certificates, so disabling for tests
+ # NOTE! Stapling does not work with self-signed certificates, so disabling for tests
# - ssl_stapling: 'on'
# - ssl_stapling_verify: 'on'
users:
__CLUSTER___arvados:
ensure: present
- password: __DATABASE_PASSWORD__
+ password: "__DATABASE_PASSWORD__"
# tablespaces:
# arvados_tablespace:
- ip: 127.0.1.1
- names:
- {{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
- # FIXME! This just works for our testings.
+ # NOTE! This just works for our testings.
# Won't work if the cluster name != host name
{%- for entry in [
'api',
- add_header: 'Strict-Transport-Security "max-age=63072000" always'
# OCSP stapling
- # FIXME! Stapling does not work with self-signed certificates, so disabling for tests
+ # NOTE! Stapling does not work with self-signed certificates, so disabling for tests
# - ssl_stapling: 'on'
# - ssl_stapling_verify: 'on'
users:
__CLUSTER___arvados:
ensure: present
- password: __DATABASE_PASSWORD__
+ password: "__DATABASE_PASSWORD__"
# tablespaces:
# arvados_tablespace:
- ip: 127.0.1.1
- names:
- {{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
- # FIXME! This just works for our testing.
+ # NOTE! This just works for our testing.
# Won't work if the cluster name != host name
{%- for entry in [
'api',
--- /dev/null
+#!/bin/bash
+
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+#
+# installer.sh
+#
+# Helps manage the configuration in a git repository, and then deploy
+# nodes by pushing a copy of the git repository to each node and
+# running the provision script to do the actual installation and
+# configuration.
+#
+
+set -eu
+
+# The parameter file
+declare CONFIG_FILE=local.params
+
+# The salt template directory
+declare CONFIG_DIR=local_config_dir
+
+# The 5-character Arvados cluster id
+# This will be populated by loadconfig()
+declare CLUSTER
+
+# The parent domain (not including the cluster id)
+# This will be populated by loadconfig()
+declare DOMAIN
+
+# A bash associative array listing each node and mapping to the roles
+# that should be provisioned on those nodes.
+# This will be populated by loadconfig()
+declare -A NODES
+
+# The ssh user we'll use
+# This will be populated by loadconfig()
+declare DEPLOY_USER
+
+# The git repository that we'll push to on all the nodes
+# This will be populated by loadconfig()
+declare GITTARGET
+
+sync() {
+ local NODE=$1
+ local BRANCH=$2
+
+ # Synchronizes the configuration by creating a git repository on
+ # each node, pushing our branch, and updating the checkout.
+
+ if [[ "$NODE" != localhost ]] ; then
+ if ! ssh $NODE test -d ${GITTARGET}.git ; then
+
+ # Initialize the git repository (1st time case). We're
+ # actually going to make two repositories here because git
+ # will complain if you try to push to a repository with a
+ # checkout. So we're going to create a "bare" repository
+ # and then clone a regular repository (with a checkout)
+ # from that.
+
+ ssh $NODE git init --bare ${GITTARGET}.git
+ if ! git remote add $NODE $DEPLOY_USER@$NODE:${GITTARGET}.git ; then
+ git remote set-url $NODE $DEPLOY_USER@$NODE:${GITTARGET}.git
+ fi
+ git push $NODE $BRANCH
+ ssh $NODE git clone ${GITTARGET}.git ${GITTARGET}
+ fi
+
+ # The update case.
+ #
+ # Push to the bare repository on the remote node, then in the
+ # remote node repository with the checkout, pull the branch
+ # from the bare repository.
+
+ git push $NODE $BRANCH
+ ssh $NODE "git -C ${GITTARGET} checkout ${BRANCH} && git -C ${GITTARGET} pull"
+ fi
+}
+
+deploynode() {
+ local NODE=$1
+ local ROLES=$2
+
+ # Deploy a node. This runs the provision script on the node, with
+ # the appropriate roles.
+
+ if [[ -z "$ROLES" ]] ; then
+ echo "No roles declared for '$NODE' in ${CONFIG_FILE}"
+ exit 1
+ fi
+
+ if [[ "$NODE" = localhost ]] ; then
+ sudo ./provision.sh --config ${CONFIG_FILE} --roles ${ROLES}
+ else
+ ssh $DEPLOY_USER@$NODE "cd ${GITTARGET} && sudo ./provision.sh --config ${CONFIG_FILE} --roles ${ROLES}"
+ fi
+}
+
+loadconfig() {
+ if [[ ! -s $CONFIG_FILE ]] ; then
+ echo "Must be run from initialized setup dir, maybe you need to 'initialize' first?"
+ fi
+ source ${CONFIG_FILE}
+ GITTARGET=arvados-deploy-config-${CLUSTER}
+}
+
+subcmd="$1"
+if [[ -n "$subcmd" ]] ; then
+ shift
+fi
+case "$subcmd" in
+ initialize)
+ if [[ ! -f provision.sh ]] ; then
+ echo "Must be run from arvados/tools/salt-install"
+ exit
+ fi
+
+ set +u
+ SETUPDIR=$1
+ PARAMS=$2
+ SLS=$3
+ set -u
+
+ err=
+ if [[ -z "$PARAMS" || ! -f local.params.example.$PARAMS ]] ; then
+ echo "Not found: local.params.example.$PARAMS"
+ echo "Expected one of multiple_hosts, single_host_multiple_hostnames, single_host_single_hostname"
+ err=1
+ fi
+
+ if [[ -z "$SLS" || ! -d config_examples/$SLS ]] ; then
+ echo "Not found: config_examples/$SLS"
+ echo "Expected one of multi_host/aws, single_host/multiple_hostnames, single_host/single_hostname"
+ err=1
+ fi
+
+ if [[ -z "$SETUPDIR" || -z "$PARAMS" || -z "$SLS" ]]; then
+ echo "installer.sh <setup dir to initialize> <params template> <config template>"
+ err=1
+ fi
+
+ if [[ -n "$err" ]] ; then
+ exit 1
+ fi
+
+ echo "Initializing $SETUPDIR"
+ git init $SETUPDIR
+ cp -r *.sh tests $SETUPDIR
+
+ cp local.params.example.$PARAMS $SETUPDIR/${CONFIG_FILE}
+ cp -r config_examples/$SLS $SETUPDIR/${CONFIG_DIR}
+
+ cd $SETUPDIR
+ git add *.sh ${CONFIG_FILE} ${CONFIG_DIR} tests
+ git commit -m"initial commit"
+
+ echo "setup directory initialized, now go to $SETUPDIR, edit '${CONFIG_FILE}' and '${CONFIG_DIR}' as needed, then run 'installer.sh deploy'"
+ ;;
+ deploy)
+ set +u
+ NODE=$1
+ set -u
+
+ loadconfig
+
+ if grep -rni 'fixme' ${CONFIG_FILE} ${CONFIG_DIR} ; then
+ echo
+ echo "Some parameters still need to be updated. Please fix them and then re-run deploy."
+ exit 1
+ fi
+
+ BRANCH=$(git branch --show-current)
+
+ set -x
+
+ git add -A
+ if ! git diff --cached --exit-code ; then
+ git commit -m"prepare for deploy"
+ fi
+
+ if [[ -z "$NODE" ]]; then
+ for NODE in "${!NODES[@]}"
+ do
+ # First, push the git repo to each node. This also
+ # confirms that we have git and can log into each
+ # node.
+ sync $NODE $BRANCH
+ done
+
+ for NODE in "${!NODES[@]}"
+ do
+ # Do 'database' role first,
+ if [[ "${NODES[$NODE]}" =~ database ]] ; then
+ deploynode $NODE ${NODES[$NODE]}
+ unset NODES[$NODE]
+ fi
+ done
+
+ for NODE in "${!NODES[@]}"
+ do
+ # then 'api' or 'controller' roles
+ if [[ "${NODES[$NODE]}" =~ (api|controller) ]] ; then
+ deploynode $NODE ${NODES[$NODE]}
+ unset NODES[$NODE]
+ fi
+ done
+
+ for NODE in "${!NODES[@]}"
+ do
+ # Everything else (we removed the nodes that we
+ # already deployed from the list)
+ deploynode $NODE ${NODES[$NODE]}
+ done
+ else
+ # Just deploy the node that was supplied on the command line.
+ sync $NODE $BRANCH
+ deploynode $NODE
+ fi
+
+ echo
+ echo "Completed deploy, run 'installer.sh diagnostics' to verify the install"
+
+ ;;
+ diagnostics)
+ loadconfig
+
+ set +u
+ declare LOCATION=$1
+ set -u
+
+ if ! which arvados-client ; then
+ echo "arvados-client not found, install 'arvados-client' package with 'apt-get' or 'yum'"
+ exit 1
+ fi
+
+ if [[ -z "$LOCATION" ]] ; then
+ echo "Need to provide '-internal-client' or '-external-client'"
+ echo
+ echo "-internal-client You are running this on the same private network as the Arvados cluster (e.g. on one of the Arvados nodes)"
+ echo "-external-client You are running this outside the private network of the Arvados cluster (e.g. your workstation)"
+ exit 1
+ fi
+
+ export ARVADOS_API_HOST="${CLUSTER}.${DOMAIN}"
+ export ARVADOS_API_TOKEN="$SYSTEM_ROOT_TOKEN"
+
+ arvados-client diagnostics $LOCATION
+ ;;
+ *)
+ echo "Arvados installer"
+ echo ""
+ echo "initialize initialize the setup directory for configuration"
+ echo "deploy deploy the configuration from the setup directory"
+ echo "diagnostics check your install using diagnostics"
+ ;;
+esac
# The Arvados cluster ID, needs to be 5 lowercase alphanumeric characters.
CLUSTER="cluster_fixme_or_this_wont_work"
-# The domainname you want tou give to your cluster's hosts
+# The domain name you want to give to your cluster's hosts
+# the end result hostnames will be $SERVICE.$CLUSTER.$DOMAIN
DOMAIN="domain_fixme_or_this_wont_work"
+# For multi-node installs, the ssh log in for each node
+# must be root or able to sudo
+DEPLOY_USER=root
+
+# The mapping of nodes to roles
+# installer.sh will log in to each of these nodes and then provision
+# it for the specified roles.
+NODES=(
+ [controller.${CLUSTER}.${DOMAIN}]=api,controller,websocket,dispatcher,keepbalance
+ [keep0.${CLUSTER}.${DOMAIN}]=keepstore
+ [keep1.${CLUSTER}.${DOMAIN}]=keepstore
+ [keep.${CLUSTER}.${DOMAIN}]=keepproxy,keepweb
+ [workbench.${CLUSTER}.${DOMAIN}]=workbench,workbench2,webshell
+ [shell.${CLUSTER}.${DOMAIN}]=shell
+)
+
# Host SSL port where you want to point your browser to access Arvados
# Defaults to 443 for regular runs, and to 8443 when called in Vagrant.
# You can point it to another port if desired
# The domainname you want tou give to your cluster's hosts
DOMAIN="domain_fixme_or_this_wont_work"
+# For multi-node installs, the ssh log in for each node
+# must be root or able to sudo
+DEPLOY_USER=root
+
+# The mapping of nodes to roles
+# installer.sh will log in to each of these nodes and then provision
+# it for the specified roles.
+NODES=(
+ [localhost]=api,controller,websocket,dispatcher,keepbalance,keepstore,keepproxy,keepweb,workbench,workbench2,webshell
+)
+
# External ports used by the Arvados services
CONTROLLER_EXT_SSL_PORT=443
KEEP_EXT_SSL_PORT=25101
# The domainname for your cluster's hosts
DOMAIN="domain_fixme_or_this_wont_work"
+# For multi-node installs, the ssh log in for each node
+# must be root or able to sudo
+DEPLOY_USER=root
+
+# The mapping of nodes to roles
+# installer.sh will log in to each of these nodes and then provision
+# it for the specified roles.
+NODES=(
+ [localhost]=api,controller,websocket,dispatcher,keepbalance,keepstore,keepproxy,keepweb,workbench,workbench2,webshell
+)
+
# Set this value when installing a cluster in a single host with a single
# hostname to access all the instances. HOSTNAME_EXT should be set to the
# external hostname for the instance.
arguments ${@}
+declare -A NODES
+
if [ -s ${CONFIG_FILE} ]; then
source ${CONFIG_FILE}
else
exit 1
fi
-if grep -q 'fixme_or_this_wont_work' ${CONFIG_FILE} ; then
+if grep -rni 'fixme' ${CONFIG_FILE} ${CONFIG_DIR} ; then
echo >&2 "The config file ${CONFIG_FILE} has some parameters that need to be modified."
echo >&2 "Please, fix them and re-run the provision script."
exit 1
// Input file as a required positional argument
if flags.NArg() == 0 {
return fmt.Errorf("please provide a path to an input file")
+ } else if flags.NArg() > 1 {
+ return fmt.Errorf("please provide just one input file argument")
}
- srcPath := &os.Args[flags.NFlag()+1]
+ srcPath := &os.Args[len(os.Args)-1]
// Validations
if *srcPath == "" {
--- /dev/null
+sync-users
\ No newline at end of file
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "bytes"
+ "encoding/csv"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "io"
+ "log"
+ "net/url"
+ "os"
+ "regexp"
+ "strconv"
+ "strings"
+
+ "git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+var version = "dev"
+
+type resourceList interface {
+ Len() int
+ GetItems() []interface{}
+}
+
+// UserList implements resourceList interface
+type UserList struct {
+ arvados.UserList
+}
+
+// Len returns the amount of items this list holds
+func (l UserList) Len() int {
+ return len(l.Items)
+}
+
+// GetItems returns the list of items
+func (l UserList) GetItems() (out []interface{}) {
+ for _, item := range l.Items {
+ out = append(out, item)
+ }
+ return
+}
+
+func main() {
+ cfg, err := GetConfig()
+ if err != nil {
+ log.Fatalf("%v", err)
+ }
+
+ if err := doMain(&cfg); err != nil {
+ log.Fatalf("%v", err)
+ }
+}
+
+type ConfigParams struct {
+ CaseInsensitive bool
+ Client *arvados.Client
+ ClusterID string
+ CurrentUser arvados.User
+ DeactivateUnlisted bool
+ Path string
+ UserID string
+ SysUserUUID string
+ AnonUserUUID string
+ Verbose bool
+}
+
+func ParseFlags(cfg *ConfigParams) error {
+ // Acceptable attributes to identify a user on the CSV file
+ userIDOpts := map[string]bool{
+ "email": true, // default
+ "username": true,
+ }
+
+ flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+ flags.Usage = func() {
+ usageStr := `Synchronize remote users into Arvados from a CSV format file with 5 columns:
+ * 1st: User Identifier (email or username)
+ * 2nd: First name
+ * 3rd: Last name
+ * 4th: Active status (0 or 1)
+ * 5th: Admin status (0 or 1)`
+ fmt.Fprintf(flags.Output(), "%s\n\n", usageStr)
+ fmt.Fprintf(flags.Output(), "Usage:\n%s [OPTIONS] <input-file.csv>\n\n", os.Args[0])
+ fmt.Fprintf(flags.Output(), "Options:\n")
+ flags.PrintDefaults()
+ }
+
+ caseInsensitive := flags.Bool(
+ "case-insensitive",
+ false,
+ "Performs case insensitive matching on user IDs. Always ON when using 'email' user IDs.")
+ deactivateUnlisted := flags.Bool(
+ "deactivate-unlisted",
+ false,
+ "Deactivate users that are not in the input file.")
+ userID := flags.String(
+ "user-id",
+ "email",
+ "Attribute by which every user is identified. Valid values are: email and username.")
+ verbose := flags.Bool(
+ "verbose",
+ false,
+ "Log informational messages.")
+ getVersion := flags.Bool(
+ "version",
+ false,
+ "Print version information and exit.")
+
+ if ok, code := cmd.ParseFlags(flags, os.Args[0], os.Args[1:], "input-file.csv", os.Stderr); !ok {
+ os.Exit(code)
+ } else if *getVersion {
+ fmt.Printf("%s %s\n", os.Args[0], version)
+ os.Exit(0)
+ }
+
+ // Input file as a required positional argument
+ if flags.NArg() == 0 {
+ return fmt.Errorf("please provide a path to an input file")
+ } else if flags.NArg() > 1 {
+ return fmt.Errorf("please provide just one input file argument")
+ }
+ srcPath := &os.Args[len(os.Args)-1]
+
+ // Validations
+ if *srcPath == "" {
+ return fmt.Errorf("input file path invalid")
+ }
+ if !userIDOpts[*userID] {
+ var options []string
+ for opt := range userIDOpts {
+ options = append(options, opt)
+ }
+ return fmt.Errorf("user ID must be one of: %s", strings.Join(options, ", "))
+ }
+ if *userID == "email" {
+ // Always do case-insensitive email addresses matching
+ *caseInsensitive = true
+ }
+
+ cfg.CaseInsensitive = *caseInsensitive
+ cfg.DeactivateUnlisted = *deactivateUnlisted
+ cfg.Path = *srcPath
+ cfg.UserID = *userID
+ cfg.Verbose = *verbose
+
+ return nil
+}
+
+// GetConfig sets up a ConfigParams struct
+func GetConfig() (cfg ConfigParams, err error) {
+ err = ParseFlags(&cfg)
+ if err != nil {
+ return
+ }
+
+ cfg.Client = arvados.NewClientFromEnv()
+
+ // Check current user permissions
+ u, err := cfg.Client.CurrentUser()
+ if err != nil {
+ return cfg, fmt.Errorf("error getting the current user: %s", err)
+ }
+ if !u.IsAdmin {
+ return cfg, fmt.Errorf("current user %q is not an admin user", u.UUID)
+ }
+ if cfg.Verbose {
+ log.Printf("Running as admin user %q (%s)", u.Email, u.UUID)
+ }
+ cfg.CurrentUser = u
+
+ var ac struct {
+ ClusterID string
+ Login struct {
+ LoginCluster string
+ }
+ }
+ err = cfg.Client.RequestAndDecode(&ac, "GET", "arvados/v1/config", nil, nil)
+ if err != nil {
+ return cfg, fmt.Errorf("error getting the exported config: %s", err)
+ }
+ if ac.Login.LoginCluster != "" && ac.Login.LoginCluster != ac.ClusterID {
+ return cfg, fmt.Errorf("cannot run on a cluster other than the login cluster")
+ }
+ cfg.SysUserUUID = ac.ClusterID + "-tpzed-000000000000000"
+ cfg.AnonUserUUID = ac.ClusterID + "-tpzed-anonymouspublic"
+ cfg.ClusterID = ac.ClusterID
+
+ return cfg, nil
+}
+
+// GetUserID returns the correct user id value depending on the selector
+func GetUserID(u arvados.User, idSelector string) (string, error) {
+ switch idSelector {
+ case "email":
+ return u.Email, nil
+ case "username":
+ return u.Username, nil
+ default:
+ return "", fmt.Errorf("cannot identify user by %q selector", idSelector)
+ }
+}
+
+func doMain(cfg *ConfigParams) error {
+ // Try opening the input file early, just in case there's a problem.
+ f, err := os.Open(cfg.Path)
+ if err != nil {
+ return fmt.Errorf("error opening input file: %s", err)
+ }
+ defer f.Close()
+
+ iCaseLog := ""
+ if cfg.UserID == "username" && cfg.CaseInsensitive {
+ iCaseLog = " - username matching requested to be case-insensitive"
+ }
+ log.Printf("%s %s started. Using %q as users id%s", os.Args[0], version, cfg.UserID, iCaseLog)
+
+ allUsers := make(map[string]arvados.User)
+ userIDToUUID := make(map[string]string) // Index by email or username
+ dupedEmails := make(map[string][]arvados.User)
+ emptyUserIDs := []string{}
+ processedUsers := make(map[string]bool)
+ results, err := GetAll(cfg.Client, "users", arvados.ResourceListParams{}, &UserList{})
+ if err != nil {
+ return fmt.Errorf("error getting all users: %s", err)
+ }
+ log.Printf("Found %d users in cluster %q", len(results), cfg.ClusterID)
+ localUserUuidRegex := regexp.MustCompile(fmt.Sprintf("^%s-tpzed-[0-9a-z]{15}$", cfg.ClusterID))
+ for _, item := range results {
+ u := item.(arvados.User)
+
+ // Remote user check
+ if !localUserUuidRegex.MatchString(u.UUID) {
+ if cfg.Verbose {
+ log.Printf("Remote user %q (%s) won't be considered for processing", u.Email, u.UUID)
+ }
+ continue
+ }
+
+ // Duplicated user id check
+ uID, err := GetUserID(u, cfg.UserID)
+ if err != nil {
+ return err
+ }
+ if uID == "" {
+ if u.UUID != cfg.AnonUserUUID && u.UUID != cfg.SysUserUUID {
+ emptyUserIDs = append(emptyUserIDs, u.UUID)
+ log.Printf("Empty %s found in user %s - ignoring", cfg.UserID, u.UUID)
+ }
+ continue
+ }
+ if cfg.CaseInsensitive {
+ uID = strings.ToLower(uID)
+ }
+ if alreadySeenUUID, found := userIDToUUID[uID]; found {
+ if cfg.UserID == "username" && uID != "" {
+ return fmt.Errorf("case insensitive collision for username %q between %q and %q", uID, u.UUID, alreadySeenUUID)
+ } else if cfg.UserID == "email" && uID != "" {
+ log.Printf("Duplicated email %q found in user %s - ignoring", uID, u.UUID)
+ if len(dupedEmails[uID]) == 0 {
+ dupedEmails[uID] = []arvados.User{allUsers[alreadySeenUUID]}
+ }
+ dupedEmails[uID] = append(dupedEmails[uID], u)
+ delete(allUsers, alreadySeenUUID) // Skip even the first occurrence,
+ // for security purposes.
+ continue
+ }
+ }
+ if cfg.Verbose {
+ log.Printf("Seen user %q (%s)", uID, u.UUID)
+ }
+ userIDToUUID[uID] = u.UUID
+ allUsers[u.UUID] = u
+ processedUsers[u.UUID] = false
+ }
+
+ loadedRecords, err := LoadInputFile(f)
+ if err != nil {
+ return fmt.Errorf("reading input file %q: %s", cfg.Path, err)
+ }
+ log.Printf("Loaded %d records from input file", len(loadedRecords))
+
+ updatesSucceeded := map[string]bool{}
+ updatesFailed := map[string]bool{}
+ updatesSkipped := map[string]bool{}
+
+ for _, record := range loadedRecords {
+ if cfg.CaseInsensitive {
+ record.UserID = strings.ToLower(record.UserID)
+ }
+ recordUUID := userIDToUUID[record.UserID]
+ processedUsers[recordUUID] = true
+ if cfg.UserID == "email" && record.UserID == cfg.CurrentUser.Email {
+ updatesSkipped[recordUUID] = true
+ log.Printf("Skipping current user %q (%s) from processing", record.UserID, cfg.CurrentUser.UUID)
+ continue
+ }
+ if updated, err := ProcessRecord(cfg, record, userIDToUUID, allUsers); err != nil {
+ log.Printf("error processing record %q: %s", record.UserID, err)
+ updatesFailed[recordUUID] = true
+ } else if updated {
+ updatesSucceeded[recordUUID] = true
+ }
+ }
+
+ if cfg.DeactivateUnlisted {
+ for userUUID, user := range allUsers {
+ if shouldSkip(cfg, user) {
+ updatesSkipped[userUUID] = true
+ log.Printf("Skipping unlisted user %q (%s) from deactivating", user.Email, user.UUID)
+ continue
+ }
+ if !processedUsers[userUUID] && allUsers[userUUID].IsActive {
+ if cfg.Verbose {
+ log.Printf("Deactivating unlisted user %q (%s)", user.Username, user.UUID)
+ }
+ var updatedUser arvados.User
+ if err := UnsetupUser(cfg.Client, user.UUID, &updatedUser); err != nil {
+ log.Printf("error deactivating unlisted user %q: %s", user.UUID, err)
+ updatesFailed[userUUID] = true
+ } else {
+ allUsers[userUUID] = updatedUser
+ updatesSucceeded[userUUID] = true
+ }
+ }
+ }
+ }
+
+ log.Printf("User update successes: %d, skips: %d, failures: %d", len(updatesSucceeded), len(updatesSkipped), len(updatesFailed))
+
+ var errors []string
+ if len(dupedEmails) > 0 {
+ emails := make([]string, len(dupedEmails))
+ i := 0
+ for e := range dupedEmails {
+ emails[i] = e
+ i++
+ }
+ errors = append(errors, fmt.Sprintf("skipped %d duplicated email address(es) in the cluster's local user list: %v", len(dupedEmails), emails))
+ }
+ if len(emptyUserIDs) > 0 {
+ errors = append(errors, fmt.Sprintf("skipped %d user account(s) with empty %s: %v", len(emptyUserIDs), cfg.UserID, emptyUserIDs))
+ }
+ if len(errors) > 0 {
+ return fmt.Errorf("%s", strings.Join(errors, "\n"))
+ }
+
+ return nil
+}
+
+func shouldSkip(cfg *ConfigParams, user arvados.User) bool {
+ switch user.UUID {
+ case cfg.SysUserUUID, cfg.AnonUserUUID:
+ return true
+ case cfg.CurrentUser.UUID:
+ return true
+ }
+ return false
+}
+
+type userRecord struct {
+ UserID string
+ FirstName string
+ LastName string
+ Active bool
+ Admin bool
+}
+
+func needsUpdating(user arvados.User, record userRecord) bool {
+ userData := userRecord{"", user.FirstName, user.LastName, user.IsActive, user.IsAdmin}
+ recordData := userRecord{"", record.FirstName, record.LastName, record.Active, record.Active && record.Admin}
+ return userData != recordData
+}
+
+// ProcessRecord creates or updates a user based on the given record
+func ProcessRecord(cfg *ConfigParams, record userRecord, userIDToUUID map[string]string, allUsers map[string]arvados.User) (bool, error) {
+ if cfg.Verbose {
+ log.Printf("Processing record for user %q", record.UserID)
+ }
+
+ wantedActiveStatus := strconv.FormatBool(record.Active)
+ wantedAdminStatus := strconv.FormatBool(record.Active && record.Admin)
+ createRequired := false
+ updateRequired := false
+ // Check if user exists, set its active & admin status.
+ var user arvados.User
+ recordUUID := userIDToUUID[record.UserID]
+ user, found := allUsers[recordUUID]
+ if !found {
+ if cfg.Verbose {
+ log.Printf("User %q does not exist, creating", record.UserID)
+ }
+ createRequired = true
+ err := CreateUser(cfg.Client, &user, map[string]string{
+ cfg.UserID: record.UserID,
+ "first_name": record.FirstName,
+ "last_name": record.LastName,
+ "is_active": wantedActiveStatus,
+ "is_admin": wantedAdminStatus,
+ })
+ if err != nil {
+ return false, fmt.Errorf("error creating user %q: %s", record.UserID, err)
+ }
+ } else if needsUpdating(user, record) {
+ updateRequired = true
+ if record.Active {
+ if !user.IsActive && cfg.Verbose {
+ log.Printf("User %q (%s) is inactive, activating", record.UserID, user.UUID)
+ }
+ // Here we assume the 'setup' is done elsewhere if needed.
+ err := UpdateUser(cfg.Client, user.UUID, &user, map[string]string{
+ "first_name": record.FirstName,
+ "last_name": record.LastName,
+ "is_active": wantedActiveStatus,
+ "is_admin": wantedAdminStatus,
+ })
+ if err != nil {
+ return false, fmt.Errorf("error updating user %q: %s", record.UserID, err)
+ }
+ } else {
+ fnChanged := user.FirstName != record.FirstName
+ lnChanged := user.LastName != record.LastName
+ if fnChanged || lnChanged {
+ err := UpdateUser(cfg.Client, user.UUID, &user, map[string]string{
+ "first_name": record.FirstName,
+ "last_name": record.LastName,
+ })
+ if err != nil {
+ return false, fmt.Errorf("error updating user %q: %s", record.UserID, err)
+ }
+ }
+ if user.IsActive {
+ if cfg.Verbose {
+ log.Printf("User %q is active, deactivating", record.UserID)
+ }
+ err := UnsetupUser(cfg.Client, user.UUID, &user)
+ if err != nil {
+ return false, fmt.Errorf("error deactivating user %q: %s", record.UserID, err)
+ }
+ }
+ }
+ }
+ if createRequired {
+ log.Printf("Created user %q", record.UserID)
+ }
+ if updateRequired {
+ log.Printf("Updated user %q", record.UserID)
+ }
+
+ return createRequired || updateRequired, nil
+}
+
+// LoadInputFile reads the input file and returns a list of user records
+func LoadInputFile(f *os.File) (loadedRecords []userRecord, err error) {
+ lineNo := 0
+ csvReader := csv.NewReader(f)
+ loadedRecords = make([]userRecord, 0)
+
+ for {
+ record, e := csvReader.Read()
+ if e == io.EOF {
+ break
+ }
+ lineNo++
+ if e != nil {
+ err = fmt.Errorf("parsing error at line %d: %s", lineNo, e)
+ return
+ }
+ if len(record) != 5 {
+ err = fmt.Errorf("parsing error at line %d: expected 5 fields, found %d", lineNo, len(record))
+ return
+ }
+ userID := strings.ToLower(strings.TrimSpace(record[0]))
+ firstName := strings.TrimSpace(record[1])
+ lastName := strings.TrimSpace(record[2])
+ active := strings.TrimSpace(record[3])
+ admin := strings.TrimSpace(record[4])
+ if userID == "" || firstName == "" || lastName == "" || active == "" || admin == "" {
+ err = fmt.Errorf("parsing error at line %d: fields cannot be empty", lineNo)
+ return
+ }
+ activeBool, err := strconv.ParseBool(active)
+ if err != nil {
+ return nil, fmt.Errorf("parsing error at line %d: active status not recognized", lineNo)
+ }
+ adminBool, err := strconv.ParseBool(admin)
+ if err != nil {
+ return nil, fmt.Errorf("parsing error at line %d: admin status not recognized", lineNo)
+ }
+ loadedRecords = append(loadedRecords, userRecord{
+ UserID: userID,
+ FirstName: firstName,
+ LastName: lastName,
+ Active: activeBool,
+ Admin: adminBool,
+ })
+ }
+ return loadedRecords, nil
+}
+
+// GetAll adds all objects of type 'resource' to the 'allItems' list
+func GetAll(c *arvados.Client, res string, params arvados.ResourceListParams, page resourceList) (allItems []interface{}, err error) {
+ // Use the maximum page size the server allows
+ limit := 1<<31 - 1
+ params.Limit = &limit
+ params.Offset = 0
+ params.Order = "uuid"
+ for {
+ if err = GetResourceList(c, &page, res, params); err != nil {
+ return allItems, err
+ }
+ // Have we finished paging?
+ if page.Len() == 0 {
+ break
+ }
+ allItems = append(allItems, page.GetItems()...)
+ params.Offset += page.Len()
+ }
+ return allItems, nil
+}
+
+func jsonReader(rscName string, ob interface{}) io.Reader {
+ j, err := json.Marshal(ob)
+ if err != nil {
+ panic(err)
+ }
+ v := url.Values{}
+ v[rscName] = []string{string(j)}
+ return bytes.NewBufferString(v.Encode())
+}
+
+// GetResourceList fetches res list using params
+func GetResourceList(c *arvados.Client, dst *resourceList, res string, params interface{}) error {
+ return c.RequestAndDecode(dst, "GET", "/arvados/v1/"+res, nil, params)
+}
+
+// CreateUser creates a user with userData parameters, assigns it to dst
+func CreateUser(c *arvados.Client, dst *arvados.User, userData map[string]string) error {
+ return c.RequestAndDecode(dst, "POST", "/arvados/v1/users", jsonReader("user", userData), nil)
+}
+
+// UpdateUser updates a user with userData parameters
+func UpdateUser(c *arvados.Client, userUUID string, dst *arvados.User, userData map[string]string) error {
+ return c.RequestAndDecode(&dst, "PUT", "/arvados/v1/users/"+userUUID, jsonReader("user", userData), nil)
+}
+
+// UnsetupUser deactivates a user
+func UnsetupUser(c *arvados.Client, userUUID string, dst *arvados.User) error {
+ return c.RequestAndDecode(&dst, "POST", "/arvados/v1/users/"+userUUID+"/unsetup", nil, nil)
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "fmt"
+ "io/ioutil"
+ "os"
+ "regexp"
+ "strings"
+ "testing"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ . "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ TestingT(t)
+}
+
+type TestSuite struct {
+ cfg *ConfigParams
+ ac *arvados.Client
+ users map[string]arvados.User
+}
+
+func (s *TestSuite) SetUpTest(c *C) {
+ s.ac = arvados.NewClientFromEnv()
+ u, err := s.ac.CurrentUser()
+ c.Assert(err, IsNil)
+ c.Assert(u.IsAdmin, Equals, true)
+
+ s.users = make(map[string]arvados.User)
+ ul := arvados.UserList{}
+ s.ac.RequestAndDecode(&ul, "GET", "/arvados/v1/users", nil, arvados.ResourceListParams{})
+ c.Assert(ul.ItemsAvailable, Not(Equals), 0)
+ s.users = make(map[string]arvados.User)
+ for _, u := range ul.Items {
+ s.users[u.UUID] = u
+ }
+
+ // Set up command config
+ os.Args = []string{"cmd", "somefile.csv"}
+ config, err := GetConfig()
+ c.Assert(err, IsNil)
+ s.cfg = &config
+}
+
+func (s *TestSuite) TearDownTest(c *C) {
+ var dst interface{}
+ // Reset database to fixture state after every test run.
+ err := s.cfg.Client.RequestAndDecode(&dst, "POST", "/database/reset", nil, nil)
+ c.Assert(err, IsNil)
+}
+
+var _ = Suite(&TestSuite{})
+
+// MakeTempCSVFile creates a temp file with data as comma separated values
+func MakeTempCSVFile(data [][]string) (f *os.File, err error) {
+ f, err = ioutil.TempFile("", "test_sync_users")
+ if err != nil {
+ return
+ }
+ for _, line := range data {
+ fmt.Fprintf(f, "%s\n", strings.Join(line, ","))
+ }
+ err = f.Close()
+ return
+}
+
+// RecordsToStrings formats the input data suitable for MakeTempCSVFile
+func RecordsToStrings(records []userRecord) [][]string {
+ data := [][]string{}
+ for _, u := range records {
+ data = append(data, []string{
+ u.UserID,
+ u.FirstName,
+ u.LastName,
+ fmt.Sprintf("%t", u.Active),
+ fmt.Sprintf("%t", u.Admin)})
+ }
+ return data
+}
+
+func ListUsers(ac *arvados.Client) ([]arvados.User, error) {
+ var ul arvados.UserList
+ err := ac.RequestAndDecode(&ul, "GET", "/arvados/v1/users", nil, arvados.ResourceListParams{})
+ if err != nil {
+ return nil, err
+ }
+ return ul.Items, nil
+}
+
+func (s *TestSuite) TestParseFlagsWithoutPositionalArgument(c *C) {
+ os.Args = []string{"cmd", "-verbose"}
+ err := ParseFlags(&ConfigParams{})
+ c.Assert(err, NotNil)
+ c.Assert(err, ErrorMatches, ".*please provide a path to an input file.*")
+}
+
+func (s *TestSuite) TestParseFlagsWrongUserID(c *C) {
+ os.Args = []string{"cmd", "-user-id", "nickname", "/tmp/somefile.csv"}
+ err := ParseFlags(&ConfigParams{})
+ c.Assert(err, NotNil)
+ c.Assert(err, ErrorMatches, ".*user ID must be one of:.*")
+}
+
+func (s *TestSuite) TestParseFlagsWithPositionalArgument(c *C) {
+ cfg := ConfigParams{}
+ os.Args = []string{"cmd", "/tmp/somefile.csv"}
+ err := ParseFlags(&cfg)
+ c.Assert(err, IsNil)
+ c.Assert(cfg.Path, Equals, "/tmp/somefile.csv")
+ c.Assert(cfg.Verbose, Equals, false)
+ c.Assert(cfg.DeactivateUnlisted, Equals, false)
+ c.Assert(cfg.UserID, Equals, "email")
+ c.Assert(cfg.CaseInsensitive, Equals, true)
+}
+
+func (s *TestSuite) TestParseFlagsWithOptionalFlags(c *C) {
+ cfg := ConfigParams{}
+ os.Args = []string{"cmd", "-verbose", "-deactivate-unlisted", "-user-id", "username", "/tmp/somefile.csv"}
+ err := ParseFlags(&cfg)
+ c.Assert(err, IsNil)
+ c.Assert(cfg.Path, Equals, "/tmp/somefile.csv")
+ c.Assert(cfg.Verbose, Equals, true)
+ c.Assert(cfg.DeactivateUnlisted, Equals, true)
+ c.Assert(cfg.UserID, Equals, "username")
+ c.Assert(cfg.CaseInsensitive, Equals, false)
+}
+
+func (s *TestSuite) TestGetConfig(c *C) {
+ os.Args = []string{"cmd", "/tmp/somefile.csv"}
+ cfg, err := GetConfig()
+ c.Assert(err, IsNil)
+ c.Assert(cfg.AnonUserUUID, Not(Equals), "")
+ c.Assert(cfg.SysUserUUID, Not(Equals), "")
+ c.Assert(cfg.CurrentUser, Not(Equals), "")
+ c.Assert(cfg.ClusterID, Not(Equals), "")
+ c.Assert(cfg.Client, NotNil)
+}
+
+func (s *TestSuite) TestFailOnEmptyFields(c *C) {
+ records := [][]string{
+ {"", "first-name", "last-name", "1", "0"},
+ {"user@example", "", "last-name", "1", "0"},
+ {"user@example", "first-name", "", "1", "0"},
+ {"user@example", "first-name", "last-name", "", "0"},
+ {"user@example", "first-name", "last-name", "1", ""},
+ }
+ for _, record := range records {
+ data := [][]string{record}
+ tmpfile, err := MakeTempCSVFile(data)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name())
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, NotNil)
+ c.Assert(err, ErrorMatches, ".*fields cannot be empty.*")
+ }
+}
+
+func (s *TestSuite) TestIgnoreSpaces(c *C) {
+ // Make sure users aren't already there from fixtures
+ for _, user := range s.users {
+ e := user.Email
+ found := e == "user1@example.com" || e == "user2@example.com" || e == "user3@example.com"
+ c.Assert(found, Equals, false)
+ }
+ // Use CSV data with leading/trailing whitespaces, confirm that they get ignored
+ data := [][]string{
+ {" user1@example.com", " Example", " User1", "1", "0"},
+ {"user2@example.com ", "Example ", "User2 ", "1", "0"},
+ {" user3@example.com ", " Example ", " User3 ", "1", "0"},
+ }
+ tmpfile, err := MakeTempCSVFile(data)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name())
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+ users, err := ListUsers(s.cfg.Client)
+ c.Assert(err, IsNil)
+ for _, userNr := range []int{1, 2, 3} {
+ found := false
+ for _, user := range users {
+ if user.Email == fmt.Sprintf("user%d@example.com", userNr) &&
+ user.LastName == fmt.Sprintf("User%d", userNr) &&
+ user.FirstName == "Example" && user.IsActive == true {
+ found = true
+ break
+ }
+ }
+ c.Assert(found, Equals, true)
+ }
+}
+
+// Error out when records have != 5 records
+func (s *TestSuite) TestWrongNumberOfFields(c *C) {
+ for _, testCase := range [][][]string{
+ {{"user1@example.com", "Example", "User1", "1"}},
+ {{"user1@example.com", "Example", "User1", "1", "0", "extra data"}},
+ } {
+ tmpfile, err := MakeTempCSVFile(testCase)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name())
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, NotNil)
+ c.Assert(err, ErrorMatches, ".*expected 5 fields, found.*")
+ }
+}
+
+// Error out when records have incorrect data types
+func (s *TestSuite) TestWrongDataFields(c *C) {
+ for _, testCase := range [][][]string{
+ {{"user1@example.com", "Example", "User1", "yep", "0"}},
+ {{"user1@example.com", "Example", "User1", "1", "nope"}},
+ } {
+ tmpfile, err := MakeTempCSVFile(testCase)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name())
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, NotNil)
+ c.Assert(err, ErrorMatches, ".*parsing error at line.*[active|admin] status not recognized.*")
+ }
+}
+
+// Create, activate and deactivate users
+func (s *TestSuite) TestUserCreationAndUpdate(c *C) {
+ for _, tc := range []string{"email", "username"} {
+ uIDPrefix := tc
+ uIDSuffix := ""
+ if tc == "email" {
+ uIDSuffix = "@example.com"
+ }
+ s.cfg.UserID = tc
+ records := []userRecord{{
+ UserID: fmt.Sprintf("%suser1%s", uIDPrefix, uIDSuffix),
+ FirstName: "Example",
+ LastName: "User1",
+ Active: true,
+ Admin: false,
+ }, {
+ UserID: fmt.Sprintf("%suser2%s", uIDPrefix, uIDSuffix),
+ FirstName: "Example",
+ LastName: "User2",
+ Active: false, // initially inactive
+ Admin: false,
+ }, {
+ UserID: fmt.Sprintf("%sadmin1%s", uIDPrefix, uIDSuffix),
+ FirstName: "Example",
+ LastName: "Admin1",
+ Active: true,
+ Admin: true,
+ }, {
+ UserID: fmt.Sprintf("%sadmin2%s", uIDPrefix, uIDSuffix),
+ FirstName: "Example",
+ LastName: "Admin2",
+ Active: false, // initially inactive
+ Admin: true,
+ }}
+ // Make sure users aren't already there from fixtures
+ for _, user := range s.users {
+ uID, err := GetUserID(user, s.cfg.UserID)
+ c.Assert(err, IsNil)
+ found := false
+ for _, r := range records {
+ if uID == r.UserID {
+ found = true
+ break
+ }
+ }
+ c.Assert(found, Equals, false)
+ }
+ // User creation
+ tmpfile, err := MakeTempCSVFile(RecordsToStrings(records))
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name())
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+
+ users, err := ListUsers(s.cfg.Client)
+ c.Assert(err, IsNil)
+ for _, r := range records {
+ var foundUser arvados.User
+ for _, user := range users {
+ uID, err := GetUserID(user, s.cfg.UserID)
+ c.Assert(err, IsNil)
+ if uID == r.UserID {
+ // Add an @example.com email if missing
+ // (to avoid database reset errors)
+ if tc == "username" && user.Email == "" {
+ err := UpdateUser(s.cfg.Client, user.UUID, &user, map[string]string{
+ "email": fmt.Sprintf("%s@example.com", user.Username),
+ })
+ c.Assert(err, IsNil)
+ }
+ foundUser = user
+ break
+ }
+ }
+ c.Assert(foundUser, NotNil)
+ c.Logf("Checking creation for user %q", r.UserID)
+ c.Assert(foundUser.FirstName, Equals, r.FirstName)
+ c.Assert(foundUser.LastName, Equals, r.LastName)
+ c.Assert(foundUser.IsActive, Equals, r.Active)
+ c.Assert(foundUser.IsAdmin, Equals, (r.Active && r.Admin))
+ }
+ // User update
+ for idx := range records {
+ records[idx].Active = !records[idx].Active
+ records[idx].FirstName = records[idx].FirstName + "Updated"
+ records[idx].LastName = records[idx].LastName + "Updated"
+ }
+ tmpfile, err = MakeTempCSVFile(RecordsToStrings(records))
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name())
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+
+ users, err = ListUsers(s.cfg.Client)
+ c.Assert(err, IsNil)
+ for _, r := range records {
+ var foundUser arvados.User
+ for _, user := range users {
+ uID, err := GetUserID(user, s.cfg.UserID)
+ c.Assert(err, IsNil)
+ if uID == r.UserID {
+ foundUser = user
+ break
+ }
+ }
+ c.Assert(foundUser, NotNil)
+ c.Logf("Checking update for user %q", r.UserID)
+ c.Assert(foundUser.FirstName, Equals, r.FirstName)
+ c.Assert(foundUser.LastName, Equals, r.LastName)
+ c.Assert(foundUser.IsActive, Equals, r.Active)
+ c.Assert(foundUser.IsAdmin, Equals, (r.Active && r.Admin))
+ }
+ }
+}
+
+func (s *TestSuite) TestDeactivateUnlisted(c *C) {
+ localUserUuidRegex := regexp.MustCompile(fmt.Sprintf("^%s-tpzed-[0-9a-z]{15}$", s.cfg.ClusterID))
+
+ var user1 arvados.User
+ for _, nr := range []int{1, 2} {
+ var newUser arvados.User
+ err := CreateUser(s.cfg.Client, &newUser, map[string]string{
+ "email": fmt.Sprintf("user%d@example.com", nr),
+ "first_name": "Example",
+ "last_name": fmt.Sprintf("User%d", nr),
+ "is_active": "true",
+ "is_admin": "false",
+ })
+ c.Assert(err, IsNil)
+ c.Assert(newUser.IsActive, Equals, true)
+ if nr == 1 {
+ user1 = newUser // for later confirmation
+ }
+ }
+
+ users, err := ListUsers(s.cfg.Client)
+ c.Assert(err, IsNil)
+ previouslyActiveUsers := 0
+ for _, u := range users {
+ if u.UUID == fmt.Sprintf("%s-tpzed-anonymouspublic", s.cfg.ClusterID) && !u.IsActive {
+ // Make sure the anonymous user is active for this test
+ var au arvados.User
+ err := UpdateUser(s.cfg.Client, u.UUID, &au, map[string]string{"is_active": "true"})
+ c.Assert(err, IsNil)
+ c.Assert(au.IsActive, Equals, true)
+ }
+ if localUserUuidRegex.MatchString(u.UUID) && u.IsActive {
+ previouslyActiveUsers++
+ }
+ }
+ // Active users: System root, Anonymous, current user and the
+ // ones just created (other active users may exist from fixture).
+ c.Logf("Initial active users count: %d", previouslyActiveUsers)
+ c.Assert(previouslyActiveUsers > 5, Equals, true)
+
+ // Here we omit user2@example.com from the CSV file.
+ data := [][]string{
+ {"user1@example.com", "Example", "User1", "1", "0"},
+ }
+ tmpfile, err := MakeTempCSVFile(data)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name())
+
+ s.cfg.DeactivateUnlisted = true
+ s.cfg.Verbose = true
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+
+ users, err = ListUsers(s.cfg.Client)
+ c.Assert(err, IsNil)
+ currentlyActiveUsers := 0
+ acceptableActiveUUIDs := map[string]bool{
+ fmt.Sprintf("%s-tpzed-000000000000000", s.cfg.ClusterID): true,
+ fmt.Sprintf("%s-tpzed-anonymouspublic", s.cfg.ClusterID): true,
+ s.cfg.CurrentUser.UUID: true,
+ user1.UUID: true,
+ }
+ remainingActiveUUIDs := map[string]bool{}
+ seenUserEmails := map[string]bool{}
+ for _, u := range users {
+ if _, ok := seenUserEmails[u.Email]; ok {
+ c.Errorf("Duplicated email address %q in user list (probably from fixtures). This test requires unique email addresses.", u.Email)
+ }
+ seenUserEmails[u.Email] = true
+ if localUserUuidRegex.MatchString(u.UUID) && u.IsActive {
+ c.Logf("Found remaining active user %q (%s)", u.Email, u.UUID)
+ _, ok := acceptableActiveUUIDs[u.UUID]
+ c.Assert(ok, Equals, true)
+ remainingActiveUUIDs[u.UUID] = true
+ currentlyActiveUsers++
+ }
+ }
+ // 4 active users remaining: System root, Anonymous, the current user
+ // and user1@example.com
+ c.Logf("Active local users remaining: %v", remainingActiveUUIDs)
+ c.Assert(currentlyActiveUsers, Equals, 4)
+}
+
+func (s *TestSuite) TestFailOnDuplicatedEmails(c *C) {
+ for i := range []int{1, 2} {
+ isAdmin := i == 2
+ err := CreateUser(s.cfg.Client, &arvados.User{}, map[string]string{
+ "email": "somedupedemail@example.com",
+ "first_name": fmt.Sprintf("Duped %d", i),
+ "username": fmt.Sprintf("dupedemail%d", i),
+ "last_name": "User",
+ "is_active": "true",
+ "is_admin": fmt.Sprintf("%t", isAdmin),
+ })
+ c.Assert(err, IsNil)
+ }
+ s.cfg.Verbose = true
+ data := [][]string{
+ {"user1@example.com", "Example", "User1", "0", "0"},
+ }
+ tmpfile, err := MakeTempCSVFile(data)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name())
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, NotNil)
+ c.Assert(err, ErrorMatches, "skipped.*duplicated email address.*")
+}
+
+func (s *TestSuite) TestFailOnEmptyUsernames(c *C) {
+ for i := range []int{1, 2} {
+ var user arvados.User
+ err := CreateUser(s.cfg.Client, &user, map[string]string{
+ "email": fmt.Sprintf("johndoe%d@example.com", i),
+ "username": "",
+ "first_name": "John",
+ "last_name": "Doe",
+ "is_active": "true",
+ "is_admin": "false",
+ })
+ c.Assert(err, IsNil)
+ c.Assert(user.Username, Equals, fmt.Sprintf("johndoe%d", i))
+ if i == 1 {
+ err = UpdateUser(s.cfg.Client, user.UUID, &user, map[string]string{
+ "username": "",
+ })
+ c.Assert(err, IsNil)
+ c.Assert(user.Username, Equals, "")
+ }
+ }
+
+ s.cfg.Verbose = true
+ data := [][]string{
+ {"johndoe0", "John", "Doe", "0", "0"},
+ }
+ tmpfile, err := MakeTempCSVFile(data)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name())
+ s.cfg.Path = tmpfile.Name()
+ s.cfg.UserID = "username"
+ err = doMain(s.cfg)
+ c.Assert(err, NotNil)
+ c.Assert(err, ErrorMatches, "skipped 1 user account.*with empty username.*")
+}
+
+func (s *TestSuite) TestFailOnDupedUsernameAndCaseInsensitiveMatching(c *C) {
+ for _, i := range []int{1, 2} {
+ var user arvados.User
+ emailPrefix := "john"
+ if i == 1 {
+ emailPrefix = "JOHN"
+ }
+ err := CreateUser(s.cfg.Client, &user, map[string]string{
+ "email": fmt.Sprintf("%sdoe@example.com", emailPrefix),
+ "username": "",
+ "first_name": "John",
+ "last_name": "Doe",
+ "is_active": "true",
+ "is_admin": "false",
+ })
+ c.Assert(err, IsNil)
+ c.Assert(user.Username, Equals, fmt.Sprintf("%sdoe", emailPrefix))
+ }
+
+ s.cfg.Verbose = true
+ data := [][]string{
+ {"johndoe", "John", "Doe", "0", "0"},
+ }
+ tmpfile, err := MakeTempCSVFile(data)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name())
+ s.cfg.Path = tmpfile.Name()
+ s.cfg.UserID = "username"
+ s.cfg.CaseInsensitive = true
+ err = doMain(s.cfg)
+ c.Assert(err, NotNil)
+ c.Assert(err, ErrorMatches, "case insensitive collision for username.*between.*and.*")
+}
+
+func (s *TestSuite) TestSuccessOnUsernameAndCaseSensitiveMatching(c *C) {
+ for _, i := range []int{1, 2} {
+ var user arvados.User
+ emailPrefix := "john"
+ if i == 1 {
+ emailPrefix = "JOHN"
+ }
+ err := CreateUser(s.cfg.Client, &user, map[string]string{
+ "email": fmt.Sprintf("%sdoe@example.com", emailPrefix),
+ "username": "",
+ "first_name": "John",
+ "last_name": "Doe",
+ "is_active": "true",
+ "is_admin": "false",
+ })
+ c.Assert(err, IsNil)
+ c.Assert(user.Username, Equals, fmt.Sprintf("%sdoe", emailPrefix))
+ }
+
+ s.cfg.Verbose = true
+ data := [][]string{
+ {"johndoe", "John", "Doe", "0", "0"},
+ }
+ tmpfile, err := MakeTempCSVFile(data)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name())
+ s.cfg.Path = tmpfile.Name()
+ s.cfg.UserID = "username"
+ s.cfg.CaseInsensitive = false
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+}
projects / arvados.git / commitdiff