Merge branch '16053-install-deps'

refs #16053

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@tomclegg.ca>

diff --combined cmd/arvados-server/cmd.go
index 80d43ad848fa0925327a2abe6fa785c968b536ae,328599826f79cdf95f49965219114b9f156ec081..fcea2223da70d5a174ee74b8281ebd3d20e0b503
--- 1/cmd/arvados-server/cmd.go
--- 2/cmd/arvados-server/cmd.go
+++ b/cmd/arvados-server/cmd.go
@@@ -14,7 -14,7 +14,8 @@@ import 
        "git.arvados.org/arvados.git/lib/controller"
        "git.arvados.org/arvados.git/lib/crunchrun"
        "git.arvados.org/arvados.git/lib/dispatchcloud"
+       "git.arvados.org/arvados.git/lib/install"
 +      "git.arvados.org/arvados.git/services/ws"
  )
  
  var (
@@@ -26,12 -26,12 +27,13 @@@
                "boot":            boot.Command,
                "cloudtest":       cloudtest.Command,
                "config-check":    config.CheckCommand,
--              "config-dump":     config.DumpCommand,
                "config-defaults": config.DumpDefaultsCommand,
++              "config-dump":     config.DumpCommand,
                "controller":      controller.Command,
                "crunch-run":      crunchrun.Command,
                "dispatch-cloud":  dispatchcloud.Command,
+               "install":         install.Command,
 +              "ws":              ws.Command,
        })
  )
  

diff --combined lib/boot/supervisor.go
index 5de92bc533aee18f5f62865500fb605e22e00d67,9f50013e39841517fb486f35d226ea344b359946..7f5d6a9baae2dd4eaa2b2e66fea9585f7be3bdc1
--- 1/lib/boot/supervisor.go
--- 2/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@@ -126,7 -126,7 +126,7 @@@ func (super *Supervisor) run(cfg *arvad
        super.setEnv("ARVADOS_CONFIG", super.configfile)
        super.setEnv("RAILS_ENV", super.ClusterType)
        super.setEnv("TMPDIR", super.tempdir)
-       super.prependEnv("PATH", filepath.Join(super.tempdir, "bin")+":")
+       super.prependEnv("PATH", super.tempdir+"/bin:/var/lib/arvados/bin:")
  
        super.cluster, err = cfg.GetCluster("")
        if err != nil {
@@@ -182,7 -182,7 +182,7 @@@
                runGoProgram{src: "services/keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
                runGoProgram{src: "services/keepstore", svc: super.cluster.Services.Keepstore},
                runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
 -              runGoProgram{src: "services/ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}},
 +              runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}},
                installPassenger{src: "services/api"},
                runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
                installPassenger{src: "apps/workbench", depends: []supervisedTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup
@@@ -360,7 -360,11 +360,11 @@@ func (super *Supervisor) setupRubyEnv(
                        "GEM_HOME=",
                        "GEM_PATH=",
                })
-               cmd := exec.Command("gem", "env", "gempath")
+               gem := "gem"
+               if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil {
+                       gem = "/var/lib/arvados/bin/gem"
+               }
+               cmd := exec.Command(gem, "env", "gempath")
                cmd.Env = super.environ
                buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:...
                if err != nil || len(buf) == 0 {
@@@ -406,7 -410,11 +410,11 @@@ func (super *Supervisor) RunProgram(ct
        cmdline := fmt.Sprintf("%s", append([]string{prog}, args...))
        super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
  
-       logprefix := strings.TrimPrefix(prog, super.tempdir+"/bin/")
+       logprefix := prog
+       if logprefix == "setuidgid" && len(args) >= 3 {
+               logprefix = args[2]
+       }
+       logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
        if logprefix == "bundle" && len(args) > 2 && args[0] == "exec" {
                logprefix = args[1]
        } else if logprefix == "arvados-server" && len(args) > 1 {

diff --combined sdk/python/tests/run_test_server.py
index 93be8dc21fd61d5313973b429b31179784c6a0ce,4f4d13858d3bcd72feb0c1f95a1b19fb536763b8..22d4f62ea0fd1bf5a8d6718e2d410b79d5377d72
--- 1/sdk/python/tests/run_test_server.py
--- 2/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@@ -430,8 -430,7 +430,8 @@@ def run_ws()
      stop_ws()
      port = internal_port_from_config("Websocket")
      logf = open(_logfilename('ws'), 'a')
 -    ws = subprocess.Popen(["ws"],
 +    ws = subprocess.Popen(
 +        ["arvados-server", "ws"],
          stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
      with open(_pidfile('ws'), 'w') as f:
          f.write(str(ws.pid))
@@@ -662,11 -661,22 +662,22 @@@ def setup_config()
      keep_web_dl_port = find_available_port()
      keep_web_dl_external_port = find_available_port()
  
-     dbconf = os.path.join(os.environ["CONFIGSRC"], "config.yml")
- 
-     print("Getting config from %s" % dbconf, file=sys.stderr)
- 
-     pgconnection = yaml.safe_load(open(dbconf))["Clusters"]["zzzzz"]["PostgreSQL"]["Connection"]
+     configsrc = os.environ.get("CONFIGSRC", None)
+     if configsrc:
+         clusterconf = os.path.join(configsrc, "config.yml")
+         print("Getting config from %s" % clusterconf, file=sys.stderr)
+         pgconnection = yaml.safe_load(open(clusterconf))["Clusters"]["zzzzz"]["PostgreSQL"]["Connection"]
+     else:
+         # assume "arvados-server install -type test" has set up the
+         # conventional db credentials
+         pgconnection = {
+           "client_encoding": "utf8",
+           "host": "localhost",
+           "dbname": "arvados_test",
+           "user": "arvados",
+           "password": "insecure_arvados_test",
+             "template": "template0", # used by RailsAPI when [re]creating the database
+         }
  
      localhost = "127.0.0.1"
      services = {