From: Tom Clegg Date: Tue, 12 Apr 2022 15:54:26 +0000 (-0400) Subject: 18700: Merge branch 'main' X-Git-Tag: 2.5.0~214^2~1 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/b53513423ab948804425424278ac554870864997?hp=b161916d6b97e8b7205670ecc972a590749dd93c 18700: Merge branch 'main' Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- diff --git a/apps/workbench/vendor/assets/javascripts/jquery.number.min.js b/apps/workbench/app/assets/javascripts/jquery.number.min.js similarity index 100% rename from apps/workbench/vendor/assets/javascripts/jquery.number.min.js rename to apps/workbench/app/assets/javascripts/jquery.number.min.js diff --git a/cmd/arvados-package/build.go b/cmd/arvados-package/build.go index 1437f4b772..27531d3bc3 100644 --- a/cmd/arvados-package/build.go +++ b/cmd/arvados-package/build.go @@ -24,13 +24,13 @@ import ( func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error { if opts.PackageVersion == "" { var buf bytes.Buffer - cmd := exec.CommandContext(ctx, "git", "describe", "--tag", "--dirty") + cmd := exec.CommandContext(ctx, "bash", "./build/version-at-commit.sh", "HEAD") cmd.Stdout = &buf cmd.Stderr = stderr cmd.Dir = opts.SourceDir err := cmd.Run() if err != nil { - return fmt.Errorf("git describe: %w", err) + return fmt.Errorf("%v: %w", cmd.Args, err) } opts.PackageVersion = strings.TrimSpace(buf.String()) ctxlog.FromContext(ctx).Infof("version not specified; using %s", opts.PackageVersion) @@ -53,6 +53,11 @@ func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Wr return err } defer os.RemoveAll(tmpdir) + if abs, err := filepath.Abs(tmpdir); err != nil { + return fmt.Errorf("error getting absolute path of tmpdir %s: %w", tmpdir, err) + } else { + tmpdir = abs + } selfbin, err := os.Readlink("/proc/self/exe") if err != nil { @@ -87,7 +92,7 @@ func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Wr cmd.Stderr = stderr err = cmd.Run() if err != nil { - return fmt.Errorf("docker run: %w", err) + return fmt.Errorf("%v: %w", cmd.Args, err) } cmd = exec.CommandContext(ctx, "docker", "commit", buildCtrName, buildImageName) @@ -120,7 +125,7 @@ func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Wr cmd.Stderr = stderr err = cmd.Run() if err != nil { - return fmt.Errorf("docker run: %w", err) + return fmt.Errorf("%v: %w", cmd.Args, err) } err = os.Rename(tmpdir+"/"+packageFilename, opts.PackageDir+"/"+packageFilename) diff --git a/cmd/arvados-package/cmd.go b/cmd/arvados-package/cmd.go index a64d440749..db3d63f277 100644 --- a/cmd/arvados-package/cmd.go +++ b/cmd/arvados-package/cmd.go @@ -65,6 +65,7 @@ type opts struct { TargetOS string Maintainer string Vendor string + Live string } func parseFlags(prog string, args []string, stderr io.Writer) (_ opts, ok bool, exitCode int) { @@ -76,12 +77,13 @@ func parseFlags(prog string, args []string, stderr io.Writer) (_ opts, ok bool, } flags := flag.NewFlagSet("", flag.ContinueOnError) flags.StringVar(&opts.PackageVersion, "package-version", opts.PackageVersion, "package version to build/test, like \"1.2.3\"") - flags.StringVar(&opts.SourceDir, "source", opts.SourceDir, "arvados source tree location") - flags.StringVar(&opts.PackageDir, "package-dir", opts.PackageDir, "destination directory for new package (default is cwd)") - flags.StringVar(&opts.PackageChown, "package-chown", opts.PackageChown, "desired uid:gid for new package (default is current user:group)") + flags.StringVar(&opts.SourceDir, "source", opts.SourceDir, "arvados source tree `directory`") + flags.StringVar(&opts.PackageDir, "package-dir", opts.PackageDir, "destination `directory` for new package (default is cwd)") + flags.StringVar(&opts.PackageChown, "package-chown", opts.PackageChown, "desired `uid:gid` for new package (default is current user:group)") flags.StringVar(&opts.TargetOS, "target-os", opts.TargetOS, "target operating system vendor:version") flags.StringVar(&opts.Maintainer, "package-maintainer", opts.Maintainer, "maintainer to be listed in package metadata") flags.StringVar(&opts.Vendor, "package-vendor", opts.Vendor, "vendor to be listed in package metadata") + flags.StringVar(&opts.Live, "live", opts.Live, "(for testinstall) advertise external URLs like https://`example.com`:44xx, use the host's /var/lib/acme/live certificates, listen on the host's external interfaces, and wait for ^C before shutting down") flags.BoolVar(&opts.RebuildImage, "rebuild-image", opts.RebuildImage, "rebuild docker image(s) instead of using existing") flags.Usage = func() { fmt.Fprint(flags.Output(), `Usage: arvados-package [options] @@ -92,7 +94,11 @@ Subcommands: out version of the arvados source tree testinstall use a docker container to install a package and confirm - the resulting installation is functional + the resulting installation is functional; optionally, + expose the test cluster's services using the host's + interfaces and ACME certificates, and leave it up to + facilitate interactive testing (see -live option + below) version show program version diff --git a/cmd/arvados-package/fpm.go b/cmd/arvados-package/fpm.go index ca63929e98..d81abab583 100644 --- a/cmd/arvados-package/fpm.go +++ b/cmd/arvados-package/fpm.go @@ -64,12 +64,12 @@ func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writ // Remove unneeded files. This is much faster than "fpm // --exclude X" because fpm copies everything into a staging // area before looking at the --exclude args. - cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git */bundler/gems/arvados-*/[^s]* */bundler/gems/arvados-*/s[^d]* */bundler/gems/arvados-*/sdk/[^cr]* */gems/passenger-*/src/cxx* ruby/*/gems/*/ext /var/lib/arvados/go") + cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git */bundler/gems/arvados-*/[^s]* */bundler/gems/arvados-*/s[^d]* */bundler/gems/arvados-*/sdk/[^cr]* */gems/passenger-*/src/cxx* ruby/*/gems/*/ext /var/lib/arvados/go /var/lib/arvados/arvados-workbench2 /var/lib/arvados/node-*") cmd.Stdout = stdout cmd.Stderr = stderr err = cmd.Run() if err != nil { - return fmt.Errorf("rm -rf [...]: %w", err) + return fmt.Errorf("%v: %w", cmd.Args, err) } format := "deb" // TODO: rpm @@ -97,6 +97,10 @@ func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writ "--verbose", "--deb-use-file-permissions", "--rpm-use-file-permissions", + "/etc/systemd/system/multi-user.target.wants/arvados.service", + "/lib/systemd/system/arvados.service", + "/usr/bin/arvados-client", + "/usr/bin/arvados-server", "/var/lib/arvados", "/var/www/.gem", "/var/www/.passenger", diff --git a/cmd/arvados-package/install.go b/cmd/arvados-package/install.go index 5fdb7a8756..0de99cabd2 100644 --- a/cmd/arvados-package/install.go +++ b/cmd/arvados-package/install.go @@ -21,6 +21,10 @@ import ( func testinstall(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error { depsImageName := "arvados-package-deps-" + opts.TargetOS depsCtrName := strings.Replace(depsImageName, ":", "-", -1) + absPackageDir, err := filepath.Abs(opts.PackageDir) + if err != nil { + return fmt.Errorf("error resolving PackageDir %q: %w", opts.PackageDir, err) + } _, prog := filepath.Split(os.Args[0]) tmpdir, err := ioutil.TempDir("", prog+".") @@ -40,7 +44,7 @@ func testinstall(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr cmd := exec.CommandContext(ctx, "docker", "run", "--name", depsCtrName, "--tmpfs", "/tmp:exec,mode=01777", - "-v", opts.PackageDir+":/pkg:ro", + "-v", absPackageDir+":/pkg:ro", "--env", "DEBIAN_FRONTEND=noninteractive", opts.TargetOS, "bash", "-c", ` @@ -64,7 +68,7 @@ rm /etc/apt/sources.list.d/arvados-local.list cmd.Stderr = stderr err = cmd.Run() if err != nil { - return fmt.Errorf("docker run: %w", err) + return fmt.Errorf("%v: %w", cmd.Args, err) } cmd = exec.CommandContext(ctx, "docker", "commit", depsCtrName, depsImageName) @@ -72,7 +76,7 @@ rm /etc/apt/sources.list.d/arvados-local.list cmd.Stderr = stderr err = cmd.Run() if err != nil { - return fmt.Errorf("docker commit: %w", err) + return fmt.Errorf("%v: %w", cmd.Args, err) } } @@ -81,9 +85,23 @@ rm /etc/apt/sources.list.d/arvados-local.list versionsuffix = "=" + opts.PackageVersion } cmd := exec.CommandContext(ctx, "docker", "run", "--rm", - "--tmpfs", "/tmp:exec,mode=01777", - "-v", opts.PackageDir+":/pkg:ro", - "--env", "DEBIAN_FRONTEND=noninteractive", + "--tmpfs=/tmp:exec,mode=01777", + "--volume="+absPackageDir+":/pkg:ro", + "--env=DEBIAN_FRONTEND=noninteractive") + if opts.Live != "" { + cmd.Args = append(cmd.Args, + "--env=domain="+opts.Live, + "--env=bootargs=", + "--publish=:4440-4460:4440-4460", + "--publish=:9000-9020:9000-9020", + "--add-host="+opts.Live+":0.0.0.0", + "--volume=/var/lib/acme:/var/lib/acme:ro") + } else { + cmd.Args = append(cmd.Args, + "--env=domain=localhost", + "--env=bootargs=-shutdown") + } + cmd.Args = append(cmd.Args, depsImageName, "bash", "-c", ` set -e -o pipefail @@ -102,14 +120,14 @@ eatmydata apt-get install --reinstall -y --no-install-recommends arvados-server- SUDO_FORCE_REMOVE=yes apt-get autoremove -y /etc/init.d/postgresql start -arvados-server init -cluster-id x1234 -exec arvados-server boot -listen-host 0.0.0.0 -shutdown +arvados-server init -cluster-id x1234 -domain=$domain -login=test -insecure +exec arvados-server boot -listen-host=0.0.0.0 -no-workbench2=false $bootargs `) cmd.Stdout = stdout cmd.Stderr = stderr err = cmd.Run() if err != nil { - return fmt.Errorf("docker run: %w", err) + return fmt.Errorf("%v: %w", cmd.Args, err) } return nil } diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go index 13b06d6795..26c3f485ea 100644 --- a/cmd/arvados-server/cmd.go +++ b/cmd/arvados-server/cmd.go @@ -5,6 +5,10 @@ package main import ( + "encoding/json" + "fmt" + "io" + "net/http" "os" "git.arvados.org/arvados.git/lib/boot" @@ -42,6 +46,7 @@ var ( "keepproxy": keepproxy.Command, "keepstore": keepstore.Command, "recover-collection": recovercollection.Command, + "workbench2": wb2command{}, "ws": ws.Command, }) ) @@ -49,3 +54,31 @@ var ( func main() { os.Exit(handler.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr)) } + +type wb2command struct{} + +func (wb2command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { + if len(args) != 3 { + fmt.Fprintf(stderr, "usage: %s api-host listen-addr app-dir\n", prog) + return 1 + } + configJSON, err := json.Marshal(map[string]string{"API_HOST": args[0]}) + if err != nil { + fmt.Fprintf(stderr, "json.Marshal: %s\n", err) + return 1 + } + mux := http.NewServeMux() + mux.Handle("/", http.FileServer(http.Dir(args[2]))) + mux.HandleFunc("/config.json", func(w http.ResponseWriter, _ *http.Request) { + w.Write(configJSON) + }) + mux.HandleFunc("/_health/ping", func(w http.ResponseWriter, _ *http.Request) { + io.WriteString(w, `{"health":"OK"}`) + }) + err = http.ListenAndServe(args[1], mux) + if err != nil { + fmt.Fprintln(stderr, err.Error()) + return 1 + } + return 0 +} diff --git a/lib/boot/cmd.go b/lib/boot/cmd.go index 96241d24b9..9c3047a7b6 100644 --- a/lib/boot/cmd.go +++ b/lib/boot/cmd.go @@ -68,7 +68,9 @@ func (bcmd bootCommand) run(ctx context.Context, prog string, args []string, std flags.StringVar(&super.ClusterType, "type", "production", "cluster `type`: development, test, or production") flags.StringVar(&super.ListenHost, "listen-host", "localhost", "host name or interface address for service listeners") flags.StringVar(&super.ControllerAddr, "controller-address", ":0", "desired controller address, `host:port` or `:port`") + flags.StringVar(&super.Workbench2Source, "workbench2-source", "../arvados-workbench2", "path to arvados-workbench2 source tree") flags.BoolVar(&super.NoWorkbench1, "no-workbench1", false, "do not run workbench1") + flags.BoolVar(&super.NoWorkbench2, "no-workbench2", true, "do not run workbench2") flags.BoolVar(&super.OwnTemporaryDatabase, "own-temporary-database", false, "bring up a postgres server and create a temporary database") timeout := flags.Duration("timeout", 0, "maximum time to wait for cluster to be ready") shutdown := flags.Bool("shutdown", false, "shut down when the cluster becomes ready") diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go index 5826e5c013..e67bc1d900 100644 --- a/lib/boot/nginx.go +++ b/lib/boot/nginx.go @@ -12,7 +12,6 @@ import ( "net/url" "os" "os/exec" - "os/user" "path/filepath" "regexp" @@ -33,12 +32,20 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er return err } vars := map[string]string{ - "LISTENHOST": super.ListenHost, - "SSLCERT": filepath.Join(super.tempdir, "server.crt"), - "SSLKEY": filepath.Join(super.tempdir, "server.key"), - "ACCESSLOG": filepath.Join(super.tempdir, "nginx_access.log"), - "ERRORLOG": filepath.Join(super.tempdir, "nginx_error.log"), - "TMPDIR": super.wwwtempdir, + "LISTENHOST": super.ListenHost, + "SSLCERT": filepath.Join(super.tempdir, "server.crt"), + "SSLKEY": filepath.Join(super.tempdir, "server.key"), + "ACCESSLOG": filepath.Join(super.tempdir, "nginx_access.log"), + "ERRORLOG": filepath.Join(super.tempdir, "nginx_error.log"), + "TMPDIR": super.wwwtempdir, + "ARVADOS_API_HOST": super.cluster.Services.Controller.ExternalURL.Host, + } + u := url.URL(super.cluster.Services.Controller.ExternalURL) + ctrlHost := u.Hostname() + if f, err := os.Open("/var/lib/acme/live/" + ctrlHost + "/privkey"); err == nil { + f.Close() + vars["SSLCERT"] = "/var/lib/acme/live/" + ctrlHost + "/cert" + vars["SSLKEY"] = "/var/lib/acme/live/" + ctrlHost + "/privkey" } for _, cmpt := range []struct { varname string @@ -51,6 +58,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er {"GIT", super.cluster.Services.GitHTTP}, {"HEALTH", super.cluster.Services.Health}, {"WORKBENCH1", super.cluster.Services.Workbench1}, + {"WORKBENCH2", super.cluster.Services.Workbench2}, {"WS", super.cluster.Services.Websocket}, } { var host, port string @@ -108,28 +116,18 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er } } - args := []string{ - "-g", "error_log stderr info;", - "-g", "pid " + filepath.Join(super.wwwtempdir, "nginx.pid") + ";", - "-c", conffile, - } - // Nginx ignores "user www-data;" when running as a non-root - // user... except that it causes it to ignore our other -g - // options. So we still have to decide for ourselves whether - // it's needed. - if u, err := user.Current(); err != nil { - return fmt.Errorf("user.Current(): %w", err) - } else if u.Uid == "0" { - args = append([]string{"-g", "user www-data;"}, args...) - } + configs := "error_log stderr info; " + configs += "pid " + filepath.Join(super.wwwtempdir, "nginx.pid") + "; " + configs += "user www-data; " super.waitShutdown.Add(1) go func() { defer super.waitShutdown.Done() - fail(super.RunProgram(ctx, ".", runOptions{}, nginx, args...)) + fail(super.RunProgram(ctx, ".", runOptions{}, nginx, "-g", configs, "-c", conffile)) }() // Choose one of the ports where Nginx should listen, and wait - // here until we can connect. If ExternalURL is https://foo (with no port) then we connect to "foo:https" + // here until we can connect. If ExternalURL is https://foo + // (with no port) then we connect to "foo:https" testurl := url.URL(super.cluster.Services.Controller.ExternalURL) if testurl.Port() == "" { testurl.Host = net.JoinHostPort(testurl.Host, testurl.Scheme) diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go index 526411c41f..7daceccb93 100644 --- a/lib/boot/supervisor.go +++ b/lib/boot/supervisor.go @@ -42,7 +42,9 @@ type Supervisor struct { ClusterType string // e.g., production ListenHost string // e.g., localhost ControllerAddr string // e.g., 127.0.0.1:8000 + Workbench2Source string // e.g., /home/username/src/arvados-workbench2 NoWorkbench1 bool + NoWorkbench2 bool OwnTemporaryDatabase bool Stderr io.Writer @@ -58,8 +60,8 @@ type Supervisor struct { waitShutdown sync.WaitGroup bindir string - tempdir string - wwwtempdir string + tempdir string // in production mode, this is accessible only to root + wwwtempdir string // in production mode, this is accessible only to www-data configfile string environ []string // for child processes } @@ -258,6 +260,11 @@ func (super *Supervisor) run(cfg *arvados.Config) error { runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}}, ) } + if !super.NoWorkbench2 { + tasks = append(tasks, + runWorkbench2{svc: super.cluster.Services.Workbench2}, + ) + } if super.ClusterType != "test" { tasks = append(tasks, runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.DispatchCloud}, @@ -482,6 +489,7 @@ type runOptions struct { output io.Writer // attach stdout env []string // add/replace environment variables user string // run as specified user + stdin io.Reader } // RunProgram runs prog with args, using dir as working directory. If ctx is @@ -525,6 +533,7 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, opts runOpt } cmd := exec.Command(super.lookPath(prog), args...) + cmd.Stdin = opts.stdin stdout, err := cmd.StdoutPipe() if err != nil { return err @@ -628,32 +637,40 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error { return err } usedPort := map[string]bool{} - nextPort := func(host string) string { + nextPort := func(host string) (string, error) { for { port, err := availablePort(host) if err != nil { - panic(err) + port, err = availablePort(super.ListenHost) + } + if err != nil { + return "", err } if usedPort[port] { continue } usedPort[port] = true - return port + return port, nil } } if cluster.Services.Controller.ExternalURL.Host == "" { h, p, err := net.SplitHostPort(super.ControllerAddr) if err != nil { - return err + return fmt.Errorf("SplitHostPort(ControllerAddr): %w", err) } if h == "" { h = super.ListenHost } if p == "0" { - p = nextPort(h) + p, err = nextPort(h) + if err != nil { + return err + } } cluster.Services.Controller.ExternalURL = arvados.URL{Scheme: "https", Host: net.JoinHostPort(h, p), Path: "/"} } + u := url.URL(cluster.Services.Controller.ExternalURL) + defaultExtHost := u.Hostname() for _, svc := range []*arvados.Service{ &cluster.Services.Controller, &cluster.Services.DispatchCloud, @@ -666,24 +683,32 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error { &cluster.Services.WebDAVDownload, &cluster.Services.Websocket, &cluster.Services.Workbench1, + &cluster.Services.Workbench2, } { if svc == &cluster.Services.DispatchCloud && super.ClusterType == "test" { continue } if svc.ExternalURL.Host == "" { + port, err := nextPort(defaultExtHost) + if err != nil { + return err + } + host := net.JoinHostPort(defaultExtHost, port) if svc == &cluster.Services.Controller || svc == &cluster.Services.GitHTTP || svc == &cluster.Services.Health || svc == &cluster.Services.Keepproxy || svc == &cluster.Services.WebDAV || svc == &cluster.Services.WebDAVDownload || - svc == &cluster.Services.Workbench1 { - svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"} + svc == &cluster.Services.Workbench1 || + svc == &cluster.Services.Workbench2 { + svc.ExternalURL = arvados.URL{Scheme: "https", Host: host, Path: "/"} } else if svc == &cluster.Services.Websocket { - svc.ExternalURL = arvados.URL{Scheme: "wss", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/websocket"} + svc.ExternalURL = arvados.URL{Scheme: "wss", Host: host, Path: "/websocket"} } } - if super.NoWorkbench1 && svc == &cluster.Services.Workbench1 { + if super.NoWorkbench1 && svc == &cluster.Services.Workbench1 || + super.NoWorkbench2 && svc == &cluster.Services.Workbench2 { // When workbench1 is disabled, it gets an // ExternalURL (so we have a valid listening // port to write in our Nginx config) but no @@ -692,8 +717,13 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error { continue } if len(svc.InternalURLs) == 0 { + port, err := nextPort(super.ListenHost) + if err != nil { + return err + } + host := net.JoinHostPort(super.ListenHost, port) svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{ - {Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}: {}, + {Scheme: "http", Host: host, Path: "/"}: {}, } } } @@ -721,7 +751,12 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error { } if super.ClusterType == "test" { // Add a second keepstore process. - cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}] = arvados.ServiceInstance{} + port, err := nextPort(super.ListenHost) + if err != nil { + return err + } + host := net.JoinHostPort(super.ListenHost, port) + cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: host, Path: "/"}] = arvados.ServiceInstance{} // Create a directory-backed volume for each keepstore // process. @@ -755,10 +790,14 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error { } } if super.OwnTemporaryDatabase { + port, err := nextPort("localhost") + if err != nil { + return err + } cluster.PostgreSQL.Connection = arvados.PostgreSQLConnection{ "client_encoding": "utf8", - "host": super.ListenHost, - "port": nextPort(super.ListenHost), + "host": "localhost", + "port": port, "dbname": "arvados_test", "user": "arvados", "password": "insecure_arvados_test", diff --git a/lib/boot/workbench2.go b/lib/boot/workbench2.go new file mode 100644 index 0000000000..5a319ebfe4 --- /dev/null +++ b/lib/boot/workbench2.go @@ -0,0 +1,73 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package boot + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "io/ioutil" + "net" + "os" + + "git.arvados.org/arvados.git/sdk/go/arvados" +) + +type runWorkbench2 struct { + svc arvados.Service +} + +func (runner runWorkbench2) String() string { + return "runWorkbench2" +} + +func (runner runWorkbench2) Run(ctx context.Context, fail func(error), super *Supervisor) error { + host, port, err := internalPort(runner.svc) + if err != nil { + return fmt.Errorf("bug: no internalPort for %q: %v (%#v)", runner, err, runner.svc) + } + super.waitShutdown.Add(1) + go func() { + defer super.waitShutdown.Done() + if super.ClusterType == "production" { + err = super.RunProgram(ctx, "/var/lib/arvados/workbench2", runOptions{ + user: "www-data", + }, "arvados-server", "workbench2", super.cluster.Services.Controller.ExternalURL.Host, net.JoinHostPort(host, port), ".") + } else if super.Workbench2Source == "" { + super.logger.Info("skipping Workbench2: Workbench2Source==\"\" and not in production mode") + return + } else { + stdinr, stdinw := io.Pipe() + defer stdinw.Close() + go func() { + <-ctx.Done() + stdinw.Close() + }() + if err = os.Mkdir(super.Workbench2Source+"/public/_health", 0777); err != nil && !errors.Is(err, fs.ErrExist) { + fail(err) + return + } + if err = ioutil.WriteFile(super.Workbench2Source+"/public/_health/ping", []byte(`{"health":"OK"}`), 0666); err != nil { + fail(err) + return + } + err = super.RunProgram(ctx, super.Workbench2Source, runOptions{ + env: []string{ + "CI=true", + "HTTPS=false", + "PORT=" + port, + "REACT_APP_ARVADOS_API_HOST=" + super.cluster.Services.Controller.ExternalURL.Host, + }, + // If we don't connect stdin, "yarn start" just exits. + stdin: stdinr, + }, "yarn", "start") + fail(errors.New("`yarn start` exited")) + } + fail(err) + }() + return nil +} diff --git a/lib/config/export.go b/lib/config/export.go index dae749c874..8e23f07328 100644 --- a/lib/config/export.go +++ b/lib/config/export.go @@ -220,6 +220,9 @@ var whitelist = map[string]bool{ "SystemLogs": false, "SystemRootToken": false, "TLS": false, + "TLS.Certificate": false, + "TLS.Insecure": true, + "TLS.Key": false, "Users": true, "Users.ActivatedUsersAreVisibleToOthers": false, "Users.AdminNotifierEmailFrom": false, diff --git a/lib/controller/integration_test.go b/lib/controller/integration_test.go index b71c4afb55..50cf89c0d4 100644 --- a/lib/controller/integration_test.go +++ b/lib/controller/integration_test.go @@ -135,6 +135,7 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) { filepath.Join(cwd, "..", ".."), id, cfg, "127.0.0."+id[3:], c.Log) tc.Super.NoWorkbench1 = true + tc.Super.NoWorkbench2 = true tc.Start() s.testClusters[id] = tc } diff --git a/lib/install/arvados.service b/lib/install/arvados.service new file mode 100644 index 0000000000..cb411c63a5 --- /dev/null +++ b/lib/install/arvados.service @@ -0,0 +1,27 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +[Unit] +Description=Arvados server +Documentation=https://doc.arvados.org/ +After=network.target +AssertPathExists=/etc/arvados/config.yml + +# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section +StartLimitIntervalSec=0 + +[Service] +Type=notify +EnvironmentFile=-/etc/arvados/environment +ExecStart=/usr/bin/arvados-server boot +# Set a reasonable default for the open file limit +LimitNOFILE=65536 +Restart=always +RestartSec=1 + +# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section +StartLimitInterval=0 + +[Install] +WantedBy=multi-user.target diff --git a/lib/install/deps.go b/lib/install/deps.go index 483ce9c933..cec24bcec1 100644 --- a/lib/install/deps.go +++ b/lib/install/deps.go @@ -8,6 +8,7 @@ import ( "bufio" "bytes" "context" + _ "embed" "errors" "flag" "fmt" @@ -28,8 +29,22 @@ import ( var Command cmd.Handler = &installCommand{} -const devtestDatabasePassword = "insecure_arvados_test" -const goversion = "1.17.1" +const goversion = "1.17.7" + +const ( + rubyversion = "2.7.5" + bundlerversion = "2.2.19" + singularityversion = "3.7.4" + pjsversion = "1.9.8" + geckoversion = "0.24.0" + gradleversion = "5.3.1" + nodejsversion = "v12.22.11" + devtestDatabasePassword = "insecure_arvados_test" + workbench2version = "5e020488f67b5bc919796e0dc8b0b9f3b3ff23b0" +) + +//go:embed arvados.service +var arvadosServiceFile []byte type installCommand struct { ClusterType string @@ -139,7 +154,6 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read "default-jdk-headless", "default-jre-headless", "gettext", - "iceweasel", "libattr1-dev", "libcrypt-ssleay-perl", "libfuse-dev", @@ -158,7 +172,6 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read "make", "net-tools", "pandoc", - "perl-modules", "pkg-config", "postgresql", "postgresql-contrib", @@ -177,16 +190,23 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read "wget", "xvfb", ) + if test { + if osv.Debian && osv.Major <= 10 { + pkgs = append(pkgs, "iceweasel") + } else { + pkgs = append(pkgs, "firefox") + } + } if dev || test { - pkgs = append(pkgs, - "squashfs-tools", // for singularity - ) + pkgs = append(pkgs, "squashfs-tools") // for singularity } switch { + case osv.Debian && osv.Major >= 11: + pkgs = append(pkgs, "libcurl4", "perl-modules-5.32") case osv.Debian && osv.Major >= 10: - pkgs = append(pkgs, "libcurl4") + pkgs = append(pkgs, "libcurl4", "perl-modules") default: - pkgs = append(pkgs, "libcurl3") + pkgs = append(pkgs, "libcurl3", "perl-modules") } cmd := exec.CommandContext(ctx, "apt-get") if inst.EatMyData { @@ -206,7 +226,6 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read os.Mkdir("/var/lib/arvados", 0755) os.Mkdir("/var/lib/arvados/tmp", 0700) if prod || pkg { - os.Mkdir("/var/lib/arvados/wwwtmp", 0700) u, er := user.Lookup("www-data") if er != nil { err = fmt.Errorf("user.Lookup(%q): %w", "www-data", er) @@ -214,12 +233,12 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read } uid, _ := strconv.Atoi(u.Uid) gid, _ := strconv.Atoi(u.Gid) + os.Mkdir("/var/lib/arvados/wwwtmp", 0700) err = os.Chown("/var/lib/arvados/wwwtmp", uid, gid) if err != nil { return 1 } } - rubyversion := "2.7.2" rubymajorversion := rubyversion[:strings.LastIndex(rubyversion, ".")] if haverubyversion, err := exec.Command("/var/lib/arvados/bin/ruby", "-v").CombinedOutput(); err == nil && bytes.HasPrefix(haverubyversion, []byte("ruby "+rubyversion)) { logger.Print("ruby " + rubyversion + " already installed") @@ -247,7 +266,7 @@ make install cd /tmp rm -rf /var/lib/arvados/go/ wget --progress=dot:giga -O- https://storage.googleapis.com/golang/go`+goversion+`.linux-amd64.tar.gz | tar -C /var/lib/arvados -xzf - -ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/ +ln -sfv /var/lib/arvados/go/bin/* /usr/local/bin/ `, stdout, stderr) if err != nil { return 1 @@ -256,49 +275,32 @@ ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/ } if !prod && !pkg { - pjsversion := "1.9.8" if havepjsversion, err := exec.Command("/usr/local/bin/phantomjs", "--version").CombinedOutput(); err == nil && string(havepjsversion) == "1.9.8\n" { logger.Print("phantomjs " + pjsversion + " already installed") } else { err = inst.runBash(` PJS=phantomjs-`+pjsversion+`-linux-x86_64 wget --progress=dot:giga -O- https://cache.arvados.org/$PJS.tar.bz2 | tar -C /var/lib/arvados -xjf - -ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/ +ln -sfv /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/ `, stdout, stderr) if err != nil { return 1 } } - geckoversion := "0.24.0" if havegeckoversion, err := exec.Command("/usr/local/bin/geckodriver", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegeckoversion), " "+geckoversion+" ") { logger.Print("geckodriver " + geckoversion + " already installed") } else { err = inst.runBash(` GD=v`+geckoversion+` wget --progress=dot:giga -O- https://github.com/mozilla/geckodriver/releases/download/$GD/geckodriver-$GD-linux64.tar.gz | tar -C /var/lib/arvados/bin -xzf - geckodriver -ln -sf /var/lib/arvados/bin/geckodriver /usr/local/bin/ +ln -sfv /var/lib/arvados/bin/geckodriver /usr/local/bin/ `, stdout, stderr) if err != nil { return 1 } } - nodejsversion := "v12.22.2" - if havenodejsversion, err := exec.Command("/usr/local/bin/node", "--version").CombinedOutput(); err == nil && string(havenodejsversion) == nodejsversion+"\n" { - logger.Print("nodejs " + nodejsversion + " already installed") - } else { - err = inst.runBash(` -NJS=`+nodejsversion+` -wget --progress=dot:giga -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf - -ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/ -`, stdout, stderr) - if err != nil { - return 1 - } - } - - gradleversion := "5.3.1" if havegradleversion, err := exec.Command("/usr/local/bin/gradle", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegradleversion), "Gradle "+gradleversion+"\n") { logger.Print("gradle " + gradleversion + " already installed") } else { @@ -308,7 +310,7 @@ zip=/var/lib/arvados/tmp/gradle-${G}-bin.zip trap "rm ${zip}" ERR wget --progress=dot:giga -O${zip} https://services.gradle.org/distributions/gradle-${G}-bin.zip unzip -o -d /var/lib/arvados ${zip} -ln -sf /var/lib/arvados/gradle-${G}/bin/gradle /usr/local/bin/ +ln -sfv /var/lib/arvados/gradle-${G}/bin/gradle /usr/local/bin/ rm ${zip} `, stdout, stderr) if err != nil { @@ -316,7 +318,6 @@ rm ${zip} } } - singularityversion := "3.7.4" if havesingularityversion, err := exec.Command("/var/lib/arvados/bin/singularity", "--version").CombinedOutput(); err == nil && strings.Contains(string(havesingularityversion), singularityversion) { logger.Print("singularity " + singularityversion + " already installed") } else if dev || test { @@ -458,7 +459,115 @@ make -C ./builddir install } } + if !prod { + if havenodejsversion, err := exec.Command("/usr/local/bin/node", "--version").CombinedOutput(); err == nil && string(havenodejsversion) == nodejsversion+"\n" { + logger.Print("nodejs " + nodejsversion + " already installed") + } else { + err = inst.runBash(` +NJS=`+nodejsversion+` +wget --progress=dot:giga -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf - +ln -sfv /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/ +`, stdout, stderr) + if err != nil { + return 1 + } + } + + if haveyarnversion, err := exec.Command("/usr/local/bin/yarn", "--version").CombinedOutput(); err == nil && len(haveyarnversion) > 0 { + logger.Print("yarn " + strings.TrimSpace(string(haveyarnversion)) + " already installed") + } else { + err = inst.runBash(` +npm install -g yarn +ln -sfv /var/lib/arvados/node-`+nodejsversion+`-linux-x64/bin/{yarn,yarnpkg} /usr/local/bin/ +`, stdout, stderr) + if err != nil { + return 1 + } + } + + if havewb2version, err := exec.Command("git", "--git-dir=/var/lib/arvados/arvados-workbench2/.git", "log", "-n1", "--format=%H").CombinedOutput(); err == nil && string(havewb2version) == workbench2version+"\n" { + logger.Print("workbench2 repo is already at " + workbench2version) + } else { + err = inst.runBash(` +V=`+workbench2version+` +cd /var/lib/arvados +if [[ ! -e arvados-workbench2 ]]; then + git clone https://git.arvados.org/arvados-workbench2.git + cd arvados-workbench2 + git checkout $V +else + cd arvados-workbench2 + if ! git checkout $V; then + git fetch + git checkout yarn.lock + git checkout $V + fi +fi +rm -rf build +`, stdout, stderr) + if err != nil { + return 1 + } + } + + if err = inst.runBash(` +cd /var/lib/arvados/arvados-workbench2 +yarn install +`, stdout, stderr); err != nil { + return 1 + } + } + if prod || pkg { + // Install Go programs to /var/lib/arvados/bin/ + for _, srcdir := range []string{ + "cmd/arvados-client", + "cmd/arvados-server", + "services/arv-git-httpd", + "services/crunch-dispatch-local", + "services/crunch-dispatch-slurm", + "services/health", + "services/keep-balance", + "services/keep-web", + "services/keepproxy", + "services/keepstore", + "services/ws", + } { + fmt.Fprintf(stderr, "building %s...\n", srcdir) + cmd := exec.Command("go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+inst.PackageVersion+" -X main.version="+inst.PackageVersion+" -s -w") + cmd.Env = append(cmd.Env, os.Environ()...) + cmd.Env = append(cmd.Env, "GOBIN=/var/lib/arvados/bin") + cmd.Dir = filepath.Join(inst.SourcePath, srcdir) + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return 1 + } + } + + // Symlink user-facing Go programs /usr/bin/x -> + // /var/lib/arvados/bin/x + for _, prog := range []string{"arvados-client", "arvados-server"} { + err = os.Remove("/usr/bin/" + prog) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return 1 + } + err = os.Symlink("/var/lib/arvados/bin/"+prog, "/usr/bin/"+prog) + if err != nil { + return 1 + } + } + + // Copy assets from source tree to /var/lib/arvados/share + cmd := exec.Command("install", "-v", "-t", "/var/lib/arvados/share", filepath.Join(inst.SourcePath, "sdk/python/tests/nginx.conf")) + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return 1 + } + // Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/ for dstdir, srcdir := range map[string]string{ "railsapi": "services/api", @@ -469,7 +578,9 @@ make -C ./builddir install "-a", "--no-owner", "--no-group", "--delete-after", "--delete-excluded", "--exclude", "/coverage", "--exclude", "/log", + "--exclude", "/node_modules", "--exclude", "/tmp", + "--exclude", "/public/assets", "--exclude", "/vendor", "--exclude", "/config/environments", "./", "/var/lib/arvados/"+dstdir+"/") @@ -481,14 +592,25 @@ make -C ./builddir install return 1 } for _, cmdline := range [][]string{ - {"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger"}, + {"mkdir", "-p", "log", "public/assets", "tmp", "vendor", ".bundle", "/var/www/.bundle", "/var/www/.gem", "/var/www/.npm", "/var/www/.passenger"}, {"touch", "log/production.log"}, - {"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"}, - {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:2.2.19"}, - {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"}, + {"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.bundle", "/var/www/.gem", "/var/www/.npm", "/var/www/.passenger", "log", "tmp", "vendor", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:" + bundlerversion}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem", "--without", "development test diagnostics performance"}, + + {"chown", "www-data:www-data", ".", "public/assets"}, + // {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "system", "true"}, + {"sudo", "-u", "www-data", "ARVADOS_CONFIG=none", "RAILS_GROUPS=assets", "RAILS_ENV=production", "/var/lib/arvados/bin/bundle", "exec", "rake", "npm:install"}, + {"sudo", "-u", "www-data", "ARVADOS_CONFIG=none", "RAILS_GROUPS=assets", "RAILS_ENV=production", "/var/lib/arvados/bin/bundle", "exec", "rake", "assets:precompile"}, + {"chown", "root:root", "."}, + {"chown", "-R", "root:root", "public/assets", "vendor"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"}, {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"}, } { + if cmdline[len(cmdline)-2] == "rake" && dstdir != "workbench1" { + continue + } cmd = exec.Command(cmdline[0], cmdline[1:]...) cmd.Dir = "/var/lib/arvados/" + dstdir cmd.Stdout = stdout @@ -513,38 +635,28 @@ make -C ./builddir install } } - // Install Go programs to /var/lib/arvados/bin/ - for _, srcdir := range []string{ - "cmd/arvados-client", - "cmd/arvados-server", - "services/arv-git-httpd", - "services/crunch-dispatch-local", - "services/crunch-dispatch-slurm", - "services/health", - "services/keep-balance", - "services/keep-web", - "services/keepproxy", - "services/keepstore", - "services/ws", - } { - fmt.Fprintf(stderr, "building %s...\n", srcdir) - cmd := exec.Command("go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+inst.PackageVersion+" -X main.version="+inst.PackageVersion) - cmd.Env = append(cmd.Env, os.Environ()...) - cmd.Env = append(cmd.Env, "GOBIN=/var/lib/arvados/bin") - cmd.Dir = filepath.Join(inst.SourcePath, srcdir) - cmd.Stdout = stdout - cmd.Stderr = stderr - err = cmd.Run() - if err != nil { - return 1 - } + // Install workbench2 app to /var/lib/arvados/workbench2/ + if err = inst.runBash(` +cd /var/lib/arvados/arvados-workbench2 +VERSION="`+inst.PackageVersion+`" BUILD_NUMBER=1 GIT_COMMIT="`+workbench2version[:9]+`" yarn build +rsync -a --delete-after build/ /var/lib/arvados/workbench2/ +`, stdout, stderr); err != nil { + return 1 } - // Copy assets from source tree to /var/lib/arvados/share - cmd := exec.Command("install", "-v", "-t", "/var/lib/arvados/share", filepath.Join(inst.SourcePath, "sdk/python/tests/nginx.conf")) - cmd.Stdout = stdout - cmd.Stderr = stderr - err = cmd.Run() + err = os.WriteFile("/lib/systemd/system/arvados.service", arvadosServiceFile, 0777) + if err != nil { + return 1 + } + // This is equivalent to "systemd enable", but does + // not depend on the systemctl program being + // available. + symlink := "/etc/systemd/system/multi-user.target.wants/arvados.service" + err = os.Remove(symlink) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return 1 + } + err = os.Symlink("/lib/systemd/system/arvados.service", symlink) if err != nil { return 1 } diff --git a/lib/install/init.go b/lib/install/init.go index 1d063506b8..07c0dc1ed5 100644 --- a/lib/install/init.go +++ b/lib/install/init.go @@ -33,6 +33,8 @@ type initCommand struct { ClusterID string Domain string PostgreSQLPassword string + Login string + Insecure bool } func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { @@ -59,6 +61,8 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0") flags.StringVar(&initcmd.ClusterID, "cluster-id", "", "cluster `id`, like x1234 for a dev cluster") flags.StringVar(&initcmd.Domain, "domain", hostname, "cluster public DNS `name`, like x1234.arvadosapi.com") + flags.StringVar(&initcmd.Login, "login", "", "login `backend`: test, pam, or ''") + flags.BoolVar(&initcmd.Insecure, "insecure", false, "accept invalid TLS certificates and configure TrustAllContent (do not use in production!)") if ok, code := cmd.ParseFlags(flags, prog, args, "", stderr); !ok { return code } else if *versionFlag { @@ -102,55 +106,58 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read Services: Controller: InternalURLs: - "http://0.0.0.0:8003/": {} - ExternalURL: {{printf "%q" ( print "https://" .Domain "/" ) }} + "http://0.0.0.0:9000/": {} + ExternalURL: {{printf "%q" ( print "https://" .Domain ":4440/" ) }} RailsAPI: InternalURLs: - "http://0.0.0.0:8004/": {} + "http://0.0.0.0:9001/": {} Websocket: InternalURLs: - "http://0.0.0.0:8005/": {} - ExternalURL: {{printf "%q" ( print "wss://ws." .Domain "/" ) }} + "http://0.0.0.0:9004/": {} + ExternalURL: {{printf "%q" ( print "wss://" .Domain ":4444/websocket" ) }} Keepbalance: InternalURLs: - "http://0.0.0.0:9005/": {} + "http://0.0.0.0:9019/": {} GitHTTP: InternalURLs: - "http://0.0.0.0:9001/": {} - ExternalURL: {{printf "%q" ( print "https://git." .Domain "/" ) }} + "http://0.0.0.0:9005/": {} + ExternalURL: {{printf "%q" ( print "https://" .Domain ":4445/" ) }} DispatchCloud: InternalURLs: "http://0.0.0.0:9006/": {} Keepproxy: InternalURLs: - "http://0.0.0.0:25108/": {} - ExternalURL: {{printf "%q" ( print "https://keep." .Domain "/" ) }} + "http://0.0.0.0:9007/": {} + ExternalURL: {{printf "%q" ( print "https://" .Domain ":4447/" ) }} WebDAV: InternalURLs: - "http://0.0.0.0:9002/": {} - ExternalURL: {{printf "%q" ( print "https://*.collections." .Domain "/" ) }} + "http://0.0.0.0:9008/": {} + ExternalURL: {{printf "%q" ( print "https://" .Domain ":4448/" ) }} WebDAVDownload: InternalURLs: - "http://0.0.0.0:8004/": {} - ExternalURL: {{printf "%q" ( print "https://download." .Domain "/" ) }} + "http://0.0.0.0:9009/": {} + ExternalURL: {{printf "%q" ( print "https://" .Domain ":4449/" ) }} Keepstore: InternalURLs: - "http://0.0.0.0:25107/": {} + "http://0.0.0.0:9010/": {} Composer: - ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/composer" ) }} + ExternalURL: {{printf "%q" ( print "https://" .Domain ":4459/composer" ) }} Workbench1: InternalURLs: - "http://0.0.0.0:8001/": {} - ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/" ) }} - #Workbench2: - # InternalURLs: - # "http://0.0.0.0:8002/": {} - # ExternalURL: {{printf "%q" ( print "https://workbench2." .Domain "/" ) }} + "http://0.0.0.0:9002/": {} + ExternalURL: {{printf "%q" ( print "https://" .Domain ":4442/" ) }} + Workbench2: + InternalURLs: + "http://0.0.0.0:9003/": {} + ExternalURL: {{printf "%q" ( print "https://" .Domain ":4443/" ) }} Health: InternalURLs: - "http://0.0.0.0:9007/": {} + "http://0.0.0.0:9011/": {} Collections: BlobSigningKey: {{printf "%q" ( .RandomHex 50 )}} + {{if .Insecure}} + TrustAllContent: true + {{end}} Containers: DispatchPrivateKey: {{printf "%q" .GenerateSSHPrivateKey}} ManagementToken: {{printf "%q" ( .RandomHex 50 )}} @@ -161,8 +168,10 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read user: arvados password: {{printf "%q" .PostgreSQLPassword}} SystemRootToken: {{printf "%q" ( .RandomHex 50 )}} + {{if .Insecure}} TLS: Insecure: true + {{end}} Volumes: {{.ClusterID}}-nyw5e-000000000000000: Driver: Directory @@ -171,6 +180,26 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read Replication: 2 Workbench: SecretKeyBase: {{printf "%q" ( .RandomHex 50 )}} + Login: + {{if eq .Login "pam"}} + PAM: + Enable: true + {{else if eq .Login "test"}} + Test: + Enable: true + Users: + admin: + Email: admin@example.com + Password: admin + {{else}} + {} + {{end}} + Users: + {{if eq .Login "test"}} + AutoAdminUserWithEmail: admin@example.com + {{else}} + {} + {{end}} `) if err != nil { return 1 diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go index a666ef8ec0..23d7e8d431 100644 --- a/sdk/go/health/aggregator.go +++ b/sdk/go/health/aggregator.go @@ -6,8 +6,8 @@ package health import ( "context" + "crypto/tls" "encoding/json" - "errors" "fmt" "net/http" "net/url" @@ -35,7 +35,13 @@ type Aggregator struct { } func (agg *Aggregator) setup() { - agg.httpClient = http.DefaultClient + agg.httpClient = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: agg.Cluster.TLS.Insecure, + }, + }, + } if agg.timeout == 0 { // this is always the case, except in the test suite agg.timeout = defaultTimeout @@ -127,7 +133,14 @@ func (agg *Aggregator) ClusterHealth() ClusterHealthResponse { } mtx.Unlock() + checkURLs := map[arvados.URL]bool{} for addr := range svc.InternalURLs { + checkURLs[addr] = true + } + if len(checkURLs) == 0 && svc.ExternalURL.Host != "" { + checkURLs[svc.ExternalURL] = true + } + for addr := range checkURLs { wg.Add(1) go func(svcName arvados.ServiceName, addr arvados.URL) { defer wg.Done() @@ -176,19 +189,14 @@ func (agg *Aggregator) pingURL(svcURL arvados.URL) (*url.URL, error) { func (agg *Aggregator) ping(target *url.URL) (result CheckResult) { t0 := time.Now() - - var err error defer func() { result.ResponseTime = json.Number(fmt.Sprintf("%.6f", time.Since(t0).Seconds())) - if err != nil { - result.Health, result.Error = "ERROR", err.Error() - } else { - result.Health = "OK" - } }() + result.Health = "ERROR" req, err := http.NewRequest("GET", target.String(), nil) if err != nil { + result.Error = err.Error() return } req.Header.Set("Authorization", "Bearer "+agg.Cluster.ManagementToken) @@ -201,22 +209,26 @@ func (agg *Aggregator) ping(target *url.URL) (result CheckResult) { req = req.WithContext(ctx) resp, err := agg.httpClient.Do(req) if err != nil { + result.Error = err.Error() return } result.HTTPStatusCode = resp.StatusCode result.HTTPStatusText = resp.Status err = json.NewDecoder(resp.Body).Decode(&result.Response) if err != nil { - err = fmt.Errorf("cannot decode response: %s", err) + result.Error = fmt.Sprintf("cannot decode response: %s", err) } else if resp.StatusCode != http.StatusOK { - err = fmt.Errorf("HTTP %d %s", resp.StatusCode, resp.Status) + result.Error = fmt.Sprintf("HTTP %d %s", resp.StatusCode, resp.Status) } else if h, _ := result.Response["health"].(string); h != "OK" { if e, ok := result.Response["error"].(string); ok && e != "" { - err = errors.New(e) + result.Error = e + return } else { - err = fmt.Errorf("health=%q in ping response", h) + result.Error = fmt.Sprintf("health=%q in ping response", h) + return } } + result.Health = "OK" return } diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf index 35b780071a..4d1e06613a 100644 --- a/sdk/python/tests/nginx.conf +++ b/sdk/python/tests/nginx.conf @@ -3,7 +3,6 @@ # SPDX-License-Identifier: Apache-2.0 daemon off; -error_log "{{ERRORLOG}}" info; # Yes, must be specified here _and_ cmdline events { } http { @@ -11,11 +10,11 @@ http { '[$time_local] "$http_x_request_id" $server_name $status $body_bytes_sent $request_time $request_method "$scheme://$http_host$request_uri" $remote_addr:$remote_port ' '"$http_referer" "$http_user_agent"'; access_log "{{ACCESSLOG}}" customlog; - client_body_temp_path "{{TMPDIR}}/nginx"; - proxy_temp_path "{{TMPDIR}}/nginx"; - fastcgi_temp_path "{{TMPDIR}}/nginx"; - uwsgi_temp_path "{{TMPDIR}}/nginx"; - scgi_temp_path "{{TMPDIR}}/nginx"; + client_body_temp_path "{{TMPDIR}}"; + proxy_temp_path "{{TMPDIR}}"; + fastcgi_temp_path "{{TMPDIR}}"; + uwsgi_temp_path "{{TMPDIR}}"; + scgi_temp_path "{{TMPDIR}}"; upstream controller { server {{LISTENHOST}}:{{CONTROLLERPORT}}; } @@ -64,6 +63,7 @@ http { proxy_set_header X-Forwarded-Proto https; proxy_redirect off; + client_max_body_size 67108864; proxy_http_version 1.1; proxy_request_buffering off; } @@ -147,7 +147,7 @@ http { } server { listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl; - server_name workbench1 workbench.*; + server_name workbench1 workbench1.* workbench.*; ssl_certificate "{{SSLCERT}}"; ssl_certificate_key "{{SSLKEY}}"; location / { @@ -158,4 +158,20 @@ http { proxy_redirect off; } } + upstream workbench2 { + server {{LISTENHOST}}:{{WORKBENCH2PORT}}; + } + server { + listen {{LISTENHOST}}:{{WORKBENCH2SSLPORT}} ssl; + server_name workbench2 workbench2.*; + ssl_certificate "{{SSLCERT}}"; + ssl_certificate_key "{{SSLKEY}}"; + location / { + proxy_pass http://workbench2; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_redirect off; + } + } } diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index f917832509..6514c2af45 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -620,6 +620,7 @@ def run_nginx(): nginxconf = {} nginxconf['LISTENHOST'] = 'localhost' nginxconf['CONTROLLERPORT'] = internal_port_from_config("Controller") + nginxconf['ARVADOS_API_HOST'] = "0.0.0.0:" + str(external_port_from_config("Controller")) nginxconf['CONTROLLERSSLPORT'] = external_port_from_config("Controller") nginxconf['KEEPWEBPORT'] = internal_port_from_config("WebDAV") nginxconf['KEEPWEBDLSSLPORT'] = external_port_from_config("WebDAVDownload") @@ -634,11 +635,13 @@ def run_nginx(): nginxconf['WSSSLPORT'] = external_port_from_config("Websocket") nginxconf['WORKBENCH1PORT'] = internal_port_from_config("Workbench1") nginxconf['WORKBENCH1SSLPORT'] = external_port_from_config("Workbench1") + nginxconf['WORKBENCH2PORT'] = internal_port_from_config("Workbench2") + nginxconf['WORKBENCH2SSLPORT'] = external_port_from_config("Workbench2") nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem') nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key') nginxconf['ACCESSLOG'] = _logfilename('nginx_access') nginxconf['ERRORLOG'] = _logfilename('nginx_error') - nginxconf['TMPDIR'] = TEST_TMPDIR + nginxconf['TMPDIR'] = TEST_TMPDIR + '/nginx' conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf') conffile = os.path.join(TEST_TMPDIR, 'nginx.conf') @@ -653,8 +656,7 @@ def run_nginx(): nginx = subprocess.Popen( ['nginx', - '-g', 'error_log stderr info;', - '-g', 'pid '+_pidfile('nginx')+';', + '-g', 'error_log stderr info; pid '+_pidfile('nginx')+';', '-c', conffile], env=env, stdin=open('/dev/null'), stdout=sys.stderr) _wait_until_port_listens(nginxconf['CONTROLLERSSLPORT']) @@ -667,6 +669,8 @@ def setup_config(): websocket_external_port = find_available_port() workbench1_port = find_available_port() workbench1_external_port = find_available_port() + workbench2_port = find_available_port() + workbench2_external_port = find_available_port() git_httpd_port = find_available_port() git_httpd_external_port = find_available_port() health_httpd_port = find_available_port() @@ -720,6 +724,12 @@ def setup_config(): "http://%s:%s"%(localhost, workbench1_port): {}, }, }, + "Workbench2": { + "ExternalURL": "https://%s:%s/" % (localhost, workbench2_external_port), + "InternalURLs": { + "http://%s:%s"%(localhost, workbench2_port): {}, + }, + }, "GitHTTP": { "ExternalURL": "https://%s:%s" % (localhost, git_httpd_external_port), "InternalURLs": { diff --git a/tools/sync-groups/federation_test.go b/tools/sync-groups/federation_test.go index 1bbdaa3fa3..d5fed3e29f 100644 --- a/tools/sync-groups/federation_test.go +++ b/tools/sync-groups/federation_test.go @@ -115,6 +115,7 @@ func (s *FederationSuite) SetUpSuite(c *check.C) { filepath.Join(cwd, "..", ".."), id, cfg, "127.0.0."+id[3:], c.Log) tc.Super.NoWorkbench1 = true + tc.Super.NoWorkbench2 = true tc.Start() s.testClusters[id] = tc }