16053: Merge branch 'master'

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@tomclegg.ca>

diff --combined build/run-tests.sh
index 113404338a91008c46647685c1a471393f78e239,7a3302de9bc98e968845ae6a42b83d3a51c0d663..721269b88eeba295d570838687297a0b634ba806
--- 1/build/run-tests.sh
--- 2/build/run-tests.sh
+++ b/build/run-tests.sh
@@@ -35,7 -35,7 +35,7 @@@ Options
  --short        Skip (or scale down) some slow tests.
  --interactive  Set up, then prompt for test/install steps to perform.
  WORKSPACE=path Arvados source tree to test.
 -CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests. (required)
 +CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests.
  services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb"
                 Restrict apiserver tests to the given file
  sdk/python_test="--test-suite tests.test_keep_locator"
@@@ -127,7 -127,7 +127,7 @@@ sdk/go/blockdiges
  sdk/go/asyncbuf
  sdk/go/stats
  sdk/go/crunchrunner
- sdk/cwl
+ sdk/cwl:py3
  sdk/R
  sdk/java-v2
  tools/sync-groups
@@@ -197,8 -197,10 +197,8 @@@ sanity_checks() 
      [[ -n "${skip[sanity]}" ]] && return 0
      ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
          || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
 -    [[ -n "$CONFIGSRC" ]] \
 -      || fatal "CONFIGSRC environment not set (see: $0 --help)"
 -    [[ -s "$CONFIGSRC/config.yml" ]] \
 -      || fatal "'$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)"
 +    [[ -z "$CONFIGSRC" ]] || [[ -s "$CONFIGSRC/config.yml" ]] \
 +      || fatal "CONFIGSRC is $CONFIGSRC but '$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)"
      echo Checking dependencies:
      echo "locale: ${LANG}"
      [[ "$(locale charmap)" = "UTF-8" ]] \
@@@ -551,14 -553,8 +551,14 @@@ setup_ruby_environment() 
          bundle="$(gem env gempath | cut -f1 -d:)/bin/bundle"
          (
              export HOME=$GEMHOME
 -            ("$bundle" version | grep -q 2.0.2) \
 -                || gem install --user bundler -v 2.0.2
 +            bundlers="$(gem list --details bundler)"
 +            versions=(1.11.0 1.17.3 2.0.2)
 +            for v in ${versions[@]}; do
 +                if ! echo "$bundlers" | fgrep -q "($v)"; then
 +                    gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done)
 +                    break
 +                fi
 +            done
              "$bundle" version | tee /dev/stderr | grep -q 'version 2'
          ) || fatal 'install bundler'
      fi
@@@ -594,11 -590,6 +594,11 @@@ setup_virtualenv() 
  }
  
  initialize() {
 +    # If dependencies like ruby, go, etc. are installed in
 +    # /var/lib/arvados -- presumably by "arvados-server install" --
 +    # then we want to use those versions, instead of whatever happens
 +    # to be installed in /usr.
 +    PATH="/var/lib/arvados/bin:${PATH}"
      sanity_checks
  
      echo "WORKSPACE=$WORKSPACE"
@@@ -1004,7 -995,6 +1004,6 @@@ pythonstuff=
      sdk/pam
      sdk/python
      sdk/python:py3
-     sdk/cwl
      sdk/cwl:py3
      services/dockercleaner:py3
      services/fuse
@@@ -1066,7 -1056,7 +1065,7 @@@ test_sdk/cli() 
  }
  
  test_sdk/java-v2() {
 -    cd "$WORKSPACE/sdk/java-v2" && gradle test
 +    cd "$WORKSPACE/sdk/java-v2" && gradle test ${testargs[sdk/java-v2]}
  }
  
  test_services/login-sync() {
@@@ -1282,7 -1272,8 +1281,8 @@@ els
                          ${verb}_${target}
                          ;;
                      *)
-                         testargs["$target"]="${opts}"
+                       argstarget=${target%:py3}
+                         testargs["$argstarget"]="${opts}"
                          tt="${testfuncargs[${target}]}"
                          tt="${tt:-$target}"
                          do_$verb $tt

diff --combined lib/boot/supervisor.go
index 8ef7e6ac1f921aa3559495b28193b070303321d2,bcf87812ab813c39a0bbcc6c801b9069c9dd4c7b..9f50013e39841517fb486f35d226ea344b359946
--- 1/lib/boot/supervisor.go
--- 2/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@@ -126,7 -126,7 +126,7 @@@ func (super *Supervisor) run(cfg *arvad
        super.setEnv("ARVADOS_CONFIG", super.configfile)
        super.setEnv("RAILS_ENV", super.ClusterType)
        super.setEnv("TMPDIR", super.tempdir)
 -      super.prependEnv("PATH", filepath.Join(super.tempdir, "bin")+":")
 +      super.prependEnv("PATH", super.tempdir+"/bin:/var/lib/arvados/bin:")
  
        super.cluster, err = cfg.GetCluster("")
        if err != nil {
@@@ -360,11 -360,7 +360,11 @@@ func (super *Supervisor) setupRubyEnv(
                        "GEM_HOME=",
                        "GEM_PATH=",
                })
 -              cmd := exec.Command("gem", "env", "gempath")
 +              gem := "gem"
 +              if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil {
 +                      gem = "/var/lib/arvados/bin/gem"
 +              }
 +              cmd := exec.Command(gem, "env", "gempath")
                cmd.Env = super.environ
                buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:...
                if err != nil || len(buf) == 0 {
@@@ -410,11 -406,7 +410,11 @@@ func (super *Supervisor) RunProgram(ct
        cmdline := fmt.Sprintf("%s", append([]string{prog}, args...))
        super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
  
 -      logprefix := strings.TrimPrefix(prog, super.tempdir+"/bin/")
 +      logprefix := prog
 +      if logprefix == "setuidgid" && len(args) >= 3 {
 +              logprefix = args[2]
 +      }
 +      logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
        if logprefix == "bundle" && len(args) > 2 && args[0] == "exec" {
                logprefix = args[1]
        } else if logprefix == "arvados-server" && len(args) > 1 {
@@@ -546,14 -538,17 +546,17 @@@ func (super *Supervisor) autofillConfig
                if svc == &cluster.Services.DispatchCloud && super.ClusterType == "test" {
                        continue
                }
-               if svc.ExternalURL.Host == "" && (svc == &cluster.Services.Controller ||
-                       svc == &cluster.Services.GitHTTP ||
-                       svc == &cluster.Services.Keepproxy ||
-                       svc == &cluster.Services.WebDAV ||
-                       svc == &cluster.Services.WebDAVDownload ||
-                       svc == &cluster.Services.Websocket ||
-                       svc == &cluster.Services.Workbench1) {
-                       svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}
+               if svc.ExternalURL.Host == "" {
+                       if svc == &cluster.Services.Controller ||
+                               svc == &cluster.Services.GitHTTP ||
+                               svc == &cluster.Services.Keepproxy ||
+                               svc == &cluster.Services.WebDAV ||
+                               svc == &cluster.Services.WebDAVDownload ||
+                               svc == &cluster.Services.Workbench1 {
+                               svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}
+                       } else if svc == &cluster.Services.Websocket {
+                               svc.ExternalURL = arvados.URL{Scheme: "wss", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}
+                       }
                }
                if len(svc.InternalURLs) == 0 {
                        svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{