From: Tom Clegg <tom@curii.com>
Date: Wed, 18 May 2022 18:01:21 +0000 (-0400)
Subject: 15370: Merge branch 'main' into 15370-loopback-dispatchcloud
X-Git-Tag: 2.5.0~142^2~9
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/bcad695db9a1c3aac5807faa153086e653107f51?hp=731c5e81f5aedc82d03786670610bde68bba27c7

15370: Merge branch 'main' into 15370-loopback-dispatchcloud

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>
---

diff --git a/.licenseignore b/.licenseignore
index 387aeda944..203c378bdc 100644
--- a/.licenseignore
+++ b/.licenseignore
@@ -50,6 +50,7 @@ docker/jobs/1078ECD7.key
 */script/rails
 sdk/cwl/tests/input/blorp.txt
 sdk/cwl/tests/tool/blub.txt
+sdk/cwl/tests/19109-upload-secondary/*
 sdk/cwl/tests/federation/data/*
 sdk/cwl/tests/fake-keep-mount/fake_collection_dir/.arvados#collection
 sdk/go/manifest/testdata/*_manifest
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 26705c0664..3e1ed6a94d 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -248,13 +248,13 @@ package_go_binary cmd/arvados-server arvados-git-httpd "$FORMAT" "$ARCH" \
     "Provide authenticated http access to Arvados-hosted git repositories"
 package_go_binary services/crunch-dispatch-local crunch-dispatch-local "$FORMAT" "$ARCH" \
     "Dispatch Crunch containers on the local system"
-package_go_binary services/crunch-dispatch-slurm crunch-dispatch-slurm "$FORMAT" "$ARCH" \
+package_go_binary cmd/arvados-server crunch-dispatch-slurm "$FORMAT" "$ARCH" \
     "Dispatch Crunch containers to a SLURM cluster"
 package_go_binary cmd/arvados-server crunch-run "$FORMAT" "$ARCH" \
     "Supervise a single Crunch container"
 package_go_binary services/crunchstat crunchstat "$FORMAT" "$ARCH" \
     "Gather cpu/memory/network statistics of running Crunch jobs"
-package_go_binary services/health arvados-health "$FORMAT" "$ARCH" \
+package_go_binary cmd/arvados-server arvados-health "$FORMAT" "$ARCH" \
     "Check health of all Arvados cluster services"
 package_go_binary cmd/arvados-server keep-balance "$FORMAT" "$ARCH" \
     "Rebalance and garbage-collect data blocks stored in Arvados Keep"
diff --git a/services/health/arvados-health.service b/cmd/arvados-server/arvados-health.service
similarity index 92%
rename from services/health/arvados-health.service
rename to cmd/arvados-server/arvados-health.service
index 4b8745d154..cf246b0ee2 100644
--- a/services/health/arvados-health.service
+++ b/cmd/arvados-server/arvados-health.service
@@ -12,7 +12,8 @@ AssertPathExists=/etc/arvados/config.yml
 StartLimitIntervalSec=0
 
 [Service]
-Type=simple
+Type=notify
+EnvironmentFile=-/etc/arvados/environment
 ExecStart=/usr/bin/arvados-health
 # Set a reasonable default for the open file limit
 LimitNOFILE=65536
diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go
index ae1e3fbeee..3a1fcd4c64 100644
--- a/cmd/arvados-server/cmd.go
+++ b/cmd/arvados-server/cmd.go
@@ -5,6 +5,7 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -21,6 +22,8 @@ import (
 	"git.arvados.org/arvados.git/lib/install"
 	"git.arvados.org/arvados.git/lib/lsf"
 	"git.arvados.org/arvados.git/lib/recovercollection"
+	"git.arvados.org/arvados.git/lib/service"
+	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"git.arvados.org/arvados.git/sdk/go/health"
 	"git.arvados.org/arvados.git/services/githttpd"
 	keepbalance "git.arvados.org/arvados.git/services/keep-balance"
@@ -28,6 +31,7 @@ import (
 	"git.arvados.org/arvados.git/services/keepproxy"
 	"git.arvados.org/arvados.git/services/keepstore"
 	"git.arvados.org/arvados.git/services/ws"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var (
@@ -47,6 +51,7 @@ var (
 		"dispatch-cloud":     dispatchcloud.Command,
 		"dispatch-lsf":       lsf.DispatchCommand,
 		"git-httpd":          githttpd.Command,
+		"health":             healthCommand,
 		"install":            install.Command,
 		"init":               install.InitCommand,
 		"keep-balance":       keepbalance.Command,
@@ -90,3 +95,17 @@ func (wb2command) RunCommand(prog string, args []string, stdin io.Reader, stdout
 	}
 	return 0
 }
+
+var healthCommand cmd.Handler = service.Command(arvados.ServiceNameHealth, func(ctx context.Context, cluster *arvados.Cluster, _ string, reg *prometheus.Registry) service.Handler {
+	mClockSkew := prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "arvados",
+		Subsystem: "health",
+		Name:      "clock_skew_seconds",
+		Help:      "Clock skew observed in most recent health check",
+	})
+	reg.MustRegister(mClockSkew)
+	return &health.Aggregator{
+		Cluster:         cluster,
+		MetricClockSkew: mClockSkew,
+	}
+})
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service b/cmd/arvados-server/crunch-dispatch-slurm.service
similarity index 88%
rename from services/crunch-dispatch-slurm/crunch-dispatch-slurm.service
rename to cmd/arvados-server/crunch-dispatch-slurm.service
index 86830f3a7f..51b4e58c35 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service
+++ b/cmd/arvados-server/crunch-dispatch-slurm.service
@@ -6,18 +6,19 @@
 Description=Arvados Crunch Dispatcher for SLURM
 Documentation=https://doc.arvados.org/
 After=network.target
+AssertPathExists=/etc/arvados/config.yml
 
 # systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
 Type=notify
+EnvironmentFile=-/etc/arvados/environment
 ExecStart=/usr/bin/crunch-dispatch-slurm
 # Set a reasonable default for the open file limit
 LimitNOFILE=65536
 Restart=always
 RestartSec=1
-LimitNOFILE=1000000
 
 # systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
 StartLimitInterval=0
diff --git a/doc/admin/config-urls.html.textile.liquid b/doc/admin/config-urls.html.textile.liquid
index 1358fd81e1..e518ea1bf7 100644
--- a/doc/admin/config-urls.html.textile.liquid
+++ b/doc/admin/config-urls.html.textile.liquid
@@ -255,10 +255,14 @@ server {
   client_max_body_size 128m;
 
   location / {
-    proxy_pass            http://controller;
-    proxy_redirect        off;
-    proxy_connect_timeout 90s;
-    proxy_read_timeout    300s;
+    proxy_pass               http://controller;
+    proxy_redirect           off;
+    proxy_connect_timeout    90s;
+    proxy_read_timeout       300s;
+    proxy_max_temp_file_size 0;
+    proxy_request_buffering  off;
+    proxy_buffering          off;
+    proxy_http_version       1.1;
 
     proxy_set_header      Host              $http_host;
     proxy_set_header      Upgrade           $http_upgrade;
diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid
index 227a8cf07b..3f6009a803 100644
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@ -32,10 +32,31 @@ h2(#main). development main (as of 2022-04-08)
 
 "previous: Upgrading to 2.4.0":#v2_4_0
 
+h3. Slurm dispatcher requires configuration update
+
+If you use the Slurm dispatcher (@crunch-dispatch-slurm@) you must add a @Services.DispatchSLURM.InternalURLs@ section to your configuration file, as shown on the "updated install page":{{site.baseurl}}/install/crunch2-slurm/install-dispatch.html.
+
+h3. New proxy parameters for arvados-controller
+
+We now recommend disabling nginx proxy caching for arvados-controller, to avoid truncation of large responses.
+
+In your Nginx configuration file (@/etc/nginx/conf.d/arvados-api-and-controller.conf@), add the following lines to the @location /@ block with @http://controller@ (see "Update nginx configuration":{{site.baseurl}}/install/install-api-server.html#update-nginx for an example) and reload/restart Nginx (@sudo nginx -s reload@).
+
+<pre>
+    proxy_max_temp_file_size 0;
+    proxy_request_buffering  off;
+    proxy_buffering          off;
+    proxy_http_version       1.1;
+</pre>
+
 h3. Now recommending Singularity 3.9.9
 
 The compute image "build script":{{site.baseurl}}/install/crunch2-cloud/install-compute-node.html now installs Singularity 3.9.9 instead of 3.7.4. The newer version includes a bugfix that should resolve "intermittent loopback device errors":https://dev.arvados.org/issues/18489 when running containers.
 
+h3. Changes to @arvados-cwl-runner --create-workflow@ and @--update-workflow@
+
+When using @arvados-cwl-runner --create-workflow@ or @--update-workflow@, by default it will now make a copy of all collection and Docker image dependencies in the target project.  Running workflows retains the old behavior (use the dependencies wherever they are found).  The can be controlled explicit with @--copy-deps@ and @--no-copy-deps@.
+
 h2(#v2_4_0). v2.4.0 (2022-04-08)
 
 "previous: Upgrading to 2.3.1":#v2_3_1
diff --git a/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid b/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid
index 52553a35e7..9b664ec9ef 100644
--- a/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid
+++ b/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid
@@ -26,9 +26,18 @@ This assumes you already have a Slurm cluster, and have set up all of your compu
 
 The Arvados Slurm dispatcher can run on any node that can submit requests to both the Arvados API server and the Slurm controller (via @sbatch@).  It is not resource-intensive, so you can run it on the API server node.
 
-h2(#update-config). Update config.yml (optional)
+h2(#update-config). Update config.yml
 
-Crunch-dispatch-slurm reads the common configuration file at @config.yml@.
+Crunch-dispatch-slurm reads the common configuration file at @/etc/arvados/config.yml@.
+
+Add a DispatchSLURM entry to the Services section, using the hostname where @crunch-dispatch-slurm@ will run, and an available port:
+
+<notextile>
+<pre>    Services:
+      DispatchSLURM:
+        InternalURLs:
+          "http://<code class="userinput">hostname.zzzzz.arvadosapi.com:9007</code>": {}</pre>
+</notextile>
 
 The following configuration parameters are optional.
 
diff --git a/doc/install/install-api-server.html.textile.liquid b/doc/install/install-api-server.html.textile.liquid
index 6c3eabba4f..4c9f168e82 100644
--- a/doc/install/install-api-server.html.textile.liquid
+++ b/doc/install/install-api-server.html.textile.liquid
@@ -148,10 +148,14 @@ server {
   client_max_body_size 128m;
 
   location / {
-    proxy_pass            http://controller;
-    proxy_redirect        off;
-    proxy_connect_timeout 90s;
-    proxy_read_timeout    300s;
+    proxy_pass               http://controller;
+    proxy_redirect           off;
+    proxy_connect_timeout    90s;
+    proxy_read_timeout       300s;
+    proxy_max_temp_file_size 0;
+    proxy_request_buffering  off;
+    proxy_buffering          off;
+    proxy_http_version       1.1;
 
     proxy_set_header      Host              $http_host;
     proxy_set_header      Upgrade           $http_upgrade;
diff --git a/doc/user/cwl/cwl-runner.html.textile.liquid b/doc/user/cwl/cwl-runner.html.textile.liquid
index 07663849ad..d3aed6ce58 100644
--- a/doc/user/cwl/cwl-runner.html.textile.liquid
+++ b/doc/user/cwl/cwl-runner.html.textile.liquid
@@ -121,16 +121,20 @@ If there is already a Docker image in Arvados with the same name, it will use th
 
 The @--match-submitter-images@ option will check the id of the image in the local Docker instance and compare it to the id of the image already in Arvados with the same name and tag.  If they are different, it will choose the image matching the local image id, which will be uploaded it if necessary.  This helpful for development, if you locally rebuild the image with the 'latest' tag, the @--match-submitter-images@ will ensure that the newer version is used.
 
+h3(#dependencies). Dependencies
+
+Dependencies include collections and Docker images referenced by the workflow.  Dependencies are automatically uploaded by @arvados-cwl-runner@ if they are not present or need to be updated.  When running a workflow, dependencies that already exist somewhere on the Arvados instance (from a previous upload) will not be uploaded or copied, regardless of the project they are located in.  Sometimes this creates problems when sharing a workflow run with others.  In this case, use @--copy-deps@ to indicate that you want all dependencies to be copied into the destination project (specified by @--project-uuid@).
+
 h3. Command line options
 
 See "arvados-cwl-runner options":{{site.baseurl}}/user/cwl/cwl-run-options.html
 
 h2(#registering). Registering a workflow to use in Workbench
 
-Use @--create-workflow@ to register a CWL workflow with Arvados.  This enables you to share workflows with other Arvados users, and run them by clicking the <span class="btn btn-sm btn-primary"><i class="fa fa-fw fa-gear"></i> Run a process...</span> button on the Workbench Dashboard and on the command line by UUID.
+Use @--create-workflow@ to register a CWL workflow with Arvados.  Use @--project-uuid@ to upload the workflow to a specific project, along with its dependencies.  You can share the workflow with other Arvados users by sharing that project.  You can run the workflow by clicking the <span class="btn btn-sm btn-primary"><i class="fa fa-fw fa-gear"></i> Run a process...</span> button on the Workbench Dashboard, and on the command line by UUID.
 
 <notextile>
-<pre><code>~/arvados/doc/user/cwl/bwa-mem$ <span class="userinput">arvados-cwl-runner --create-workflow bwa-mem.cwl</span>
+<pre><code>~/arvados/doc/user/cwl/bwa-mem$ <span class="userinput">arvados-cwl-runner --project-uuid zzzzz-j7d0g-p32bi47ogkjke11 --create-workflow bwa-mem.cwl</span>
 arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
 2016-07-01 12:21:01 arvados.arv-run[15796] INFO: Upload local files: "bwa-mem.cwl"
 2016-07-01 12:21:01 arvados.arv-run[15796] INFO: Uploaded to zzzzz-4zz18-7e0hedrmkuyoei3
@@ -142,7 +146,7 @@ zzzzz-p5p6p-rjleou1dwr167v5
 You can provide a partial input file to set default values for the workflow input parameters.  You can also use the @--name@ option to set the name of the workflow:
 
 <notextile>
-<pre><code>~/arvados/doc/user/cwl/bwa-mem$ <span class="userinput">arvados-cwl-runner --name "My workflow with defaults" --create-workflow bwa-mem.cwl bwa-mem-template.yml</span>
+<pre><code>~/arvados/doc/user/cwl/bwa-mem$ <span class="userinput">arvados-cwl-runner --name "My workflow with defaults" --project-uuid zzzzz-j7d0g-p32bi47ogkjke11 --create-workflow bwa-mem.cwl bwa-mem-template.yml</span>
 arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
 2016-07-01 14:09:50 arvados.arv-run[3730] INFO: Upload local files: "bwa-mem.cwl"
 2016-07-01 14:09:50 arvados.arv-run[3730] INFO: Uploaded to zzzzz-4zz18-0f91qkovk4ml18o
@@ -151,6 +155,20 @@ zzzzz-p5p6p-zuniv58hn8d0qd8
 </code></pre>
 </notextile>
 
+Use @--update-workflow <uuid>@ to update an existing workflow.
+
+<notextile>
+<pre><code>~/arvados/doc/user/cwl/bwa-mem$ <span class="userinput">arvados-cwl-runner --update-workflow zzzzz-p5p6p-zuniv58hn8d0qd8 bwa-mem.cwl</span>
+arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
+2016-07-01 12:21:01 arvados.arv-run[15796] INFO: Upload local files: "bwa-mem.cwl"
+2016-07-01 12:21:01 arvados.arv-run[15796] INFO: Uploaded to zzzzz-4zz18-7e0hedrmkuyoei3
+2016-07-01 12:21:01 arvados.cwl-runner[15796] INFO: Created template zzzzz-p5p6p-zuniv58hn8d0qd8
+zzzzz-p5p6p-zuniv58hn8d0qd8
+</code></pre>
+</notextile>
+
+When using @--create-workflow@ or @--update-workflow@, the @--copy-deps@ and @--match-submitter-images@ options are enabled by default.
+
 h3. Running registered workflows at the command line
 
 You can run a registered workflow at the command line by its UUID:
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index d69cc9f18c..7e641c62dd 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -365,7 +365,7 @@ func (super *Supervisor) runCluster() error {
 		runNginx{},
 		runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{seedDatabase{}}},
 		runServiceCommand{name: "git-httpd", svc: super.cluster.Services.GitHTTP},
-		runGoProgram{src: "services/health", svc: super.cluster.Services.Health},
+		runServiceCommand{name: "health", svc: super.cluster.Services.Health},
 		runServiceCommand{name: "keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
 		runServiceCommand{name: "keepstore", svc: super.cluster.Services.Keepstore},
 		runServiceCommand{name: "keep-web", svc: super.cluster.Services.WebDAV},
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index b09c1ecb9f..d965633055 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -55,6 +55,9 @@ Clusters:
       DispatchLSF:
         InternalURLs: {SAMPLE: {}}
         ExternalURL: ""
+      DispatchSLURM:
+        InternalURLs: {SAMPLE: {}}
+        ExternalURL: ""
       Keepproxy:
         InternalURLs: {SAMPLE: {}}
         ExternalURL: ""
diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index 474fbf4ade..0253ac3fa8 100644
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -422,7 +422,7 @@ func (runner *ContainerRunner) SetupMounts() (map[string]bindmount, error) {
 		"--storage-classes", strings.Join(runner.Container.OutputStorageClasses, ","),
 		fmt.Sprintf("--crunchstat-interval=%v", runner.statInterval.Seconds())}
 
-	if runner.executor.Runtime() == "docker" {
+	if _, isdocker := runner.executor.(*dockerExecutor); isdocker {
 		arvMountCmd = append(arvMountCmd, "--allow-other")
 	}
 
@@ -1479,7 +1479,10 @@ func (runner *ContainerRunner) NewArvLogWriter(name string) (io.WriteCloser, err
 func (runner *ContainerRunner) Run() (err error) {
 	runner.CrunchLog.Printf("crunch-run %s started", cmd.Version.String())
 	runner.CrunchLog.Printf("%s", currentUserAndGroups())
-	runner.CrunchLog.Printf("Executing container '%s' using %s runtime", runner.Container.UUID, runner.executor.Runtime())
+	v, _ := exec.Command("arv-mount", "--version").CombinedOutput()
+	runner.CrunchLog.Printf("Using FUSE mount: %s", v)
+	runner.CrunchLog.Printf("Using container runtime: %s", runner.executor.Runtime())
+	runner.CrunchLog.Printf("Executing container: %s", runner.Container.UUID)
 
 	hostname, hosterr := os.Hostname()
 	if hosterr != nil {
diff --git a/lib/crunchrun/crunchrun_test.go b/lib/crunchrun/crunchrun_test.go
index 1d2c7b09fd..347703a95b 100644
--- a/lib/crunchrun/crunchrun_test.go
+++ b/lib/crunchrun/crunchrun_test.go
@@ -22,6 +22,7 @@ import (
 	"testing"
 	"time"
 
+	"git.arvados.org/arvados.git/lib/cmd"
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"git.arvados.org/arvados.git/sdk/go/arvadosclient"
 	"git.arvados.org/arvados.git/sdk/go/arvadostest"
@@ -128,6 +129,7 @@ func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvad
 	return e.loadErr
 }
 func (e *stubExecutor) Runtime() string                 { return "stub" }
+func (e *stubExecutor) Version() string                 { return "stub " + cmd.Version.String() }
 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
@@ -885,7 +887,8 @@ func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 	c.Assert(s.api.Logs["crunch-run"], NotNil)
 	c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 	c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`)
-	c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' using stub runtime.*`)
+	c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container: zzzzz-zzzzz-zzzzzzzzzzzzzzz.*`)
+	c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Using container runtime: stub.*`)
 }
 
 func (s *TestSuite) TestContainerRecordLog(c *C) {
diff --git a/lib/crunchrun/docker.go b/lib/crunchrun/docker.go
index e62f2a39ba..f3808cb357 100644
--- a/lib/crunchrun/docker.go
+++ b/lib/crunchrun/docker.go
@@ -46,7 +46,20 @@ func newDockerExecutor(containerUUID string, logf func(string, ...interface{}),
 	}, err
 }
 
-func (e *dockerExecutor) Runtime() string { return "docker" }
+func (e *dockerExecutor) Runtime() string {
+	v, _ := e.dockerclient.ServerVersion(context.Background())
+	info := ""
+	for _, cv := range v.Components {
+		if info != "" {
+			info += ", "
+		}
+		info += cv.Name + " " + cv.Version
+	}
+	if info == "" {
+		info = "(unknown version)"
+	}
+	return "docker " + info
+}
 
 func (e *dockerExecutor) LoadImage(imageID string, imageTarballPath string, container arvados.Container, arvMountPoint string,
 	containerClient *arvados.Client) error {
diff --git a/lib/crunchrun/executor.go b/lib/crunchrun/executor.go
index dc1bc20b7c..0a65f4d634 100644
--- a/lib/crunchrun/executor.go
+++ b/lib/crunchrun/executor.go
@@ -60,6 +60,6 @@ type containerExecutor interface {
 	// Release resources (temp dirs, stopped containers)
 	Close()
 
-	// Name of runtime engine ("docker", "singularity")
+	// Name and version of runtime engine ("docker 20.10.16", "singularity-ce version 3.9.9")
 	Runtime() string
 }
diff --git a/lib/crunchrun/executor_test.go b/lib/crunchrun/executor_test.go
index 1833fc8ac4..5b146a6321 100644
--- a/lib/crunchrun/executor_test.go
+++ b/lib/crunchrun/executor_test.go
@@ -52,6 +52,7 @@ func (s *executorSuite) TearDownTest(c *C) {
 }
 
 func (s *executorSuite) TestExecTrivialContainer(c *C) {
+	c.Logf("Using container runtime: %s", s.executor.Runtime())
 	s.spec.Command = []string{"echo", "ok"}
 	s.checkRun(c, 0)
 	c.Check(s.stdout.String(), Equals, "ok\n")
diff --git a/lib/crunchrun/integration_test.go b/lib/crunchrun/integration_test.go
index ff2165338a..ce92a9b807 100644
--- a/lib/crunchrun/integration_test.go
+++ b/lib/crunchrun/integration_test.go
@@ -162,11 +162,13 @@ func (s *integrationSuite) setup(c *C) {
 func (s *integrationSuite) TestRunTrivialContainerWithDocker(c *C) {
 	s.engine = "docker"
 	s.testRunTrivialContainer(c)
+	c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*Using container runtime: docker Engine \d+\.\d+.*`)
 }
 
 func (s *integrationSuite) TestRunTrivialContainerWithSingularity(c *C) {
 	s.engine = "singularity"
 	s.testRunTrivialContainer(c)
+	c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*Using container runtime: singularity.* version 3\.\d+.*`)
 }
 
 func (s *integrationSuite) TestRunTrivialContainerWithLocalKeepstore(c *C) {
diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go
index 64a3773250..1af0d420e4 100644
--- a/lib/crunchrun/singularity.go
+++ b/lib/crunchrun/singularity.go
@@ -10,6 +10,7 @@ import (
 	"os"
 	"os/exec"
 	"sort"
+	"strings"
 	"syscall"
 	"time"
 
@@ -36,7 +37,13 @@ func newSingularityExecutor(logf func(string, ...interface{})) (*singularityExec
 	}, nil
 }
 
-func (e *singularityExecutor) Runtime() string { return "singularity" }
+func (e *singularityExecutor) Runtime() string {
+	buf, err := exec.Command("singularity", "--version").CombinedOutput()
+	if err != nil {
+		return "singularity (unknown version)"
+	}
+	return strings.TrimSuffix(string(buf), "\n")
+}
 
 func (e *singularityExecutor) getOrCreateProject(ownerUuid string, name string, containerClient *arvados.Client) (*arvados.Group, error) {
 	var gp arvados.GroupList
@@ -292,6 +299,14 @@ func (e *singularityExecutor) execCmd(path string) *exec.Cmd {
 		// us to select specific devices we need to propagate that.
 		env = append(env, "SINGULARITYENV_CUDA_VISIBLE_DEVICES="+cudaVisibleDevices)
 	}
+	// Singularity's default behavior is to evaluate each
+	// SINGULARITYENV_* env var with a shell as a double-quoted
+	// string and pass the result to the contained
+	// process. Singularity 3.10+ has an option to pass env vars
+	// through literally without evaluating, which is what we
+	// want. See https://github.com/sylabs/singularity/pull/704
+	// and https://dev.arvados.org/issues/19081
+	env = append(env, "SINGULARITY_NO_EVAL=1")
 
 	args = append(args, e.imageFilename)
 	args = append(args, e.spec.Command...)
diff --git a/lib/crunchrun/singularity_test.go b/lib/crunchrun/singularity_test.go
index cdeafee882..bad2abef33 100644
--- a/lib/crunchrun/singularity_test.go
+++ b/lib/crunchrun/singularity_test.go
@@ -48,5 +48,5 @@ func (s *singularityStubSuite) TestSingularityExecArgs(c *C) {
 	e.imageFilename = "/fake/image.sif"
 	cmd := e.execCmd("./singularity")
 	c.Check(cmd.Args, DeepEquals, []string{"./singularity", "exec", "--containall", "--cleanenv", "--pwd", "/WorkingDir", "--net", "--network=none", "--nv", "--bind", "/hostpath:/mnt:ro", "/fake/image.sif"})
-	c.Check(cmd.Env, DeepEquals, []string{"SINGULARITYENV_FOO=bar"})
+	c.Check(cmd.Env, DeepEquals, []string{"SINGULARITYENV_FOO=bar", "SINGULARITY_NO_EVAL=1"})
 }
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index c73b358ecc..21b629f37a 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -217,6 +217,10 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     exgroup.add_argument("--enable-preemptible", dest="enable_preemptible", default=None, action="store_true", help="Use preemptible instances. Control individual steps with arv:UsePreemptible hint.")
     exgroup.add_argument("--disable-preemptible", dest="enable_preemptible", default=None, action="store_false", help="Don't use preemptible instances.")
 
+    exgroup = parser.add_mutually_exclusive_group()
+    exgroup.add_argument("--copy-deps", dest="copy_deps", default=None, action="store_true", help="Copy dependencies into the destination project.")
+    exgroup.add_argument("--no-copy-deps", dest="copy_deps", default=None, action="store_false", help="Leave dependencies where they are.")
+
     parser.add_argument(
         "--skip-schemas",
         action="store_true",
@@ -363,5 +367,5 @@ def main(args=sys.argv[1:],
                              logger_handler=arvados.log_handler,
                              custom_schema_callback=add_arv_hints,
                              loadingContext=executor.loadingContext,
-                             runtimeContext=executor.runtimeContext,
+                             runtimeContext=executor.toplevel_runtimeContext,
                              input_required=not (arvargs.create_workflow or arvargs.update_workflow))
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index f75bde81e6..5082cc2f4b 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -247,7 +247,8 @@ class ArvadosContainer(JobBase):
                                                                     runtimeContext.project_uuid,
                                                                     runtimeContext.force_docker_pull,
                                                                     runtimeContext.tmp_outdir_prefix,
-                                                                    runtimeContext.match_local_docker)
+                                                                    runtimeContext.match_local_docker,
+                                                                    runtimeContext.copy_deps)
 
         network_req, _ = self.get_requirement("NetworkAccess")
         if network_req:
@@ -465,7 +466,7 @@ class RunnerContainer(Runner):
             "cwd": "/var/spool/cwl",
             "priority": self.priority,
             "state": "Committed",
-            "container_image": arvados_jobs_image(self.arvrunner, self.jobs_image),
+            "container_image": arvados_jobs_image(self.arvrunner, self.jobs_image, runtimeContext),
             "mounts": {
                 "/var/lib/cwl/cwl.input.json": {
                     "kind": "json",
@@ -500,7 +501,7 @@ class RunnerContainer(Runner):
                 "portable_data_hash": "%s" % workflowcollection
             }
         else:
-            packed = packed_workflow(self.arvrunner, self.embedded_tool, self.merged_map)
+            packed = packed_workflow(self.arvrunner, self.embedded_tool, self.merged_map, runtimeContext)
             workflowpath = "/var/lib/cwl/workflow.json#main"
             container_req["mounts"]["/var/lib/cwl/workflow.json"] = {
                 "kind": "json",
@@ -550,17 +551,17 @@ class RunnerContainer(Runner):
         if runtimeContext.intermediate_storage_classes != "default" and runtimeContext.intermediate_storage_classes:
             command.append("--intermediate-storage-classes=" + runtimeContext.intermediate_storage_classes)
 
-        if self.on_error:
+        if runtimeContext.on_error:
             command.append("--on-error=" + self.on_error)
 
-        if self.intermediate_output_ttl:
-            command.append("--intermediate-output-ttl=%d" % self.intermediate_output_ttl)
+        if runtimeContext.intermediate_output_ttl:
+            command.append("--intermediate-output-ttl=%d" % runtimeContext.intermediate_output_ttl)
 
-        if self.arvrunner.trash_intermediate:
+        if runtimeContext.trash_intermediate:
             command.append("--trash-intermediate")
 
-        if self.arvrunner.project_uuid:
-            command.append("--project-uuid="+self.arvrunner.project_uuid)
+        if runtimeContext.project_uuid:
+            command.append("--project-uuid="+runtimeContext.project_uuid)
 
         if self.enable_dev:
             command.append("--enable-dev")
@@ -581,8 +582,8 @@ class RunnerContainer(Runner):
     def run(self, runtimeContext):
         runtimeContext.keepprefix = "keep:"
         job_spec = self.arvados_job_spec(runtimeContext)
-        if self.arvrunner.project_uuid:
-            job_spec["owner_uuid"] = self.arvrunner.project_uuid
+        if runtimeContext.project_uuid:
+            job_spec["owner_uuid"] = runtimeContext.project_uuid
 
         extra_submit_params = {}
         if runtimeContext.submit_runner_cluster:
diff --git a/sdk/cwl/arvados_cwl/arvdocker.py b/sdk/cwl/arvados_cwl/arvdocker.py
index 04e2a4cffc..cf0b3b9daf 100644
--- a/sdk/cwl/arvados_cwl/arvdocker.py
+++ b/sdk/cwl/arvados_cwl/arvdocker.py
@@ -57,7 +57,7 @@ def determine_image_id(dockerImageId):
 
 
 def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid,
-                         force_pull, tmp_outdir_prefix, match_local_docker):
+                         force_pull, tmp_outdir_prefix, match_local_docker, copy_deps):
     """Check if a Docker image is available in Keep, if not, upload it using arv-keepdocker."""
 
     if "http://arvados.org/cwl#dockerCollectionPDH" in dockerRequirement:
@@ -80,11 +80,21 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
         image_name = sp[0]
         image_tag = sp[1] if len(sp) > 1 else "latest"
 
-        images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3,
+        out_of_project_images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3,
                                                                 image_name=image_name,
-                                                                image_tag=image_tag)
+                                                                image_tag=image_tag,
+                                                                project_uuid=None)
 
-        if images and match_local_docker:
+        if copy_deps:
+            # Only images that are available in the destination project
+            images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3,
+                                                                    image_name=image_name,
+                                                                    image_tag=image_tag,
+                                                                    project_uuid=project_uuid)
+        else:
+            images = out_of_project_images
+
+        if match_local_docker:
             local_image_id = determine_image_id(dockerRequirement["dockerImageId"])
             if local_image_id:
                 # find it in the list
@@ -98,15 +108,25 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
                     # force re-upload.
                     images = []
 
+                for i in out_of_project_images:
+                    if i[1]["dockerhash"] == local_image_id:
+                        found = True
+                        out_of_project_images = [i]
+                        break
+                if not found:
+                    # force re-upload.
+                    out_of_project_images = []
+
         if not images:
-            # Fetch Docker image if necessary.
-            try:
-                result = cwltool.docker.DockerCommandLineJob.get_image(dockerRequirement, pull_image,
-                                                              force_pull, tmp_outdir_prefix)
-                if not result:
-                    raise WorkflowException("Docker image '%s' not available" % dockerRequirement["dockerImageId"])
-            except OSError as e:
-                raise WorkflowException("While trying to get Docker image '%s', failed to execute 'docker': %s" % (dockerRequirement["dockerImageId"], e))
+            if not out_of_project_images:
+                # Fetch Docker image if necessary.
+                try:
+                    result = cwltool.docker.DockerCommandLineJob.get_image(dockerRequirement, pull_image,
+                                                                  force_pull, tmp_outdir_prefix)
+                    if not result:
+                        raise WorkflowException("Docker image '%s' not available" % dockerRequirement["dockerImageId"])
+                except OSError as e:
+                    raise WorkflowException("While trying to get Docker image '%s', failed to execute 'docker': %s" % (dockerRequirement["dockerImageId"], e))
 
             # Upload image to Arvados
             args = []
@@ -125,7 +145,8 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
 
             images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3,
                                                                     image_name=image_name,
-                                                                    image_tag=image_tag)
+                                                                    image_tag=image_tag,
+                                                                    project_uuid=project_uuid)
 
         if not images:
             raise WorkflowException("Could not find Docker image %s:%s" % (image_name, image_tag))
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 4fe82a6fe1..51e7cd8b9e 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -37,11 +37,12 @@ metrics = logging.getLogger('arvados.cwl-runner.metrics')
 max_res_pars = ("coresMin", "coresMax", "ramMin", "ramMax", "tmpdirMin", "tmpdirMax")
 sum_res_pars = ("outdirMin", "outdirMax")
 
-def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None,
+def upload_workflow(arvRunner, tool, job_order, project_uuid,
+                    runtimeContext, uuid=None,
                     submit_runner_ram=0, name=None, merged_map=None,
                     submit_runner_image=None):
 
-    packed = packed_workflow(arvRunner, tool, merged_map)
+    packed = packed_workflow(arvRunner, tool, merged_map, runtimeContext)
 
     adjustDirObjs(job_order, trim_listing)
     adjustFileObjs(job_order, trim_anonymous_location)
@@ -57,7 +58,8 @@ def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None,
         name = tool.tool.get("label", os.path.basename(tool.tool["id"]))
 
     upload_dependencies(arvRunner, name, tool.doc_loader,
-                        packed, tool.tool["id"], False)
+                        packed, tool.tool["id"], False,
+                        runtimeContext)
 
     wf_runner_resources = None
 
@@ -72,7 +74,9 @@ def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None,
         wf_runner_resources = {"class": "http://arvados.org/cwl#WorkflowRunnerResources"}
         hints.append(wf_runner_resources)
 
-    wf_runner_resources["acrContainerImage"] = arvados_jobs_image(arvRunner, submit_runner_image or "arvados/jobs:"+__version__)
+    wf_runner_resources["acrContainerImage"] = arvados_jobs_image(arvRunner,
+                                                                  submit_runner_image or "arvados/jobs:"+__version__,
+                                                                  runtimeContext)
 
     if submit_runner_ram:
         wf_runner_resources["ramMin"] = submit_runner_ram
@@ -194,7 +198,8 @@ class ArvadosWorkflow(Workflow):
                                 self.doc_loader,
                                 joborder,
                                 joborder.get("id", "#"),
-                                False)
+                                False,
+                                runtimeContext)
 
             if self.wf_pdh is None:
                 packed = pack(self.loadingContext, self.tool["id"], loader=self.doc_loader)
@@ -237,7 +242,8 @@ class ArvadosWorkflow(Workflow):
                                     self.doc_loader,
                                     packed,
                                     self.tool["id"],
-                                    False)
+                                    False,
+                                    runtimeContext)
 
                 # Discover files/directories referenced by the
                 # workflow (mainly "default" values)
@@ -301,7 +307,7 @@ class ArvadosWorkflow(Workflow):
             if self.wf_pdh is None:
                 adjustFileObjs(packed, keepmount)
                 adjustDirObjs(packed, keepmount)
-                self.wf_pdh = upload_workflow_collection(self.arvrunner, shortname(self.tool["id"]), packed)
+                self.wf_pdh = upload_workflow_collection(self.arvrunner, shortname(self.tool["id"]), packed, runtimeContext)
 
         self.loadingContext = self.loadingContext.copy()
         self.loadingContext.metadata = self.loadingContext.metadata.copy()
diff --git a/sdk/cwl/arvados_cwl/context.py b/sdk/cwl/arvados_cwl/context.py
index 316250106b..64f85e2076 100644
--- a/sdk/cwl/arvados_cwl/context.py
+++ b/sdk/cwl/arvados_cwl/context.py
@@ -38,6 +38,7 @@ class ArvRuntimeContext(RuntimeContext):
         self.collection_cache_size = 256
         self.match_local_docker = False
         self.enable_preemptible = None
+        self.copy_deps = None
 
         super(ArvRuntimeContext, self).__init__(kwargs)
 
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 680ca0b7b2..1759e4ac28 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -197,11 +197,11 @@ The 'jobs' API is no longer supported.
             handler = RuntimeStatusLoggingHandler(self.runtime_status_update)
             root_logger.addHandler(handler)
 
-        self.runtimeContext = ArvRuntimeContext(vars(arvargs))
-        self.runtimeContext.make_fs_access = partial(CollectionFsAccess,
+        self.toplevel_runtimeContext = ArvRuntimeContext(vars(arvargs))
+        self.toplevel_runtimeContext.make_fs_access = partial(CollectionFsAccess,
                                                      collection_cache=self.collection_cache)
 
-        validate_cluster_target(self, self.runtimeContext)
+        validate_cluster_target(self, self.toplevel_runtimeContext)
 
 
     def arv_make_tool(self, toolpath_object, loadingContext):
@@ -517,7 +517,6 @@ The 'jobs' API is no longer supported.
 
         updated_tool.visit(self.check_features)
 
-        self.project_uuid = runtimeContext.project_uuid
         self.pipeline = None
         self.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
         self.secret_store = runtimeContext.secret_store
@@ -535,6 +534,8 @@ The 'jobs' API is no longer supported.
         if runtimeContext.submit_request_uuid and self.work_api != "containers":
             raise Exception("--submit-request-uuid requires containers API, but using '{}' api".format(self.work_api))
 
+        runtimeContext = runtimeContext.copy()
+
         default_storage_classes = ",".join([k for k,v in self.api.config().get("StorageClasses", {"default": {"Default": True}}).items() if v.get("Default") is True])
         if runtimeContext.storage_classes == "default":
             runtimeContext.storage_classes = default_storage_classes
@@ -544,9 +545,25 @@ The 'jobs' API is no longer supported.
         if not runtimeContext.name:
             runtimeContext.name = self.name = updated_tool.tool.get("label") or updated_tool.metadata.get("label") or os.path.basename(updated_tool.tool["id"])
 
+        if runtimeContext.copy_deps is None and (runtimeContext.create_workflow or runtimeContext.update_workflow):
+            # When creating or updating workflow record, by default
+            # always copy dependencies and ensure Docker images are up
+            # to date.
+            runtimeContext.copy_deps = True
+            runtimeContext.match_local_docker = True
+
+        if runtimeContext.update_workflow and self.project_uuid is None:
+            # If we are updating a workflow, make sure anything that
+            # gets uploaded goes into the same parent project, unless
+            # an alternate --project-uuid was provided.
+            existing_wf = self.api.workflows().get(uuid=runtimeContext.update_workflow).execute()
+            runtimeContext.project_uuid = existing_wf["owner_uuid"]
+
+        self.project_uuid = runtimeContext.project_uuid
+
         # Upload local file references in the job order.
         job_order = upload_job_order(self, "%s input" % runtimeContext.name,
-                                     updated_tool, job_order)
+                                     updated_tool, job_order, runtimeContext)
 
         # the last clause means: if it is a command line tool, and we
         # are going to wait for the result, and always_submit_runner
@@ -573,7 +590,7 @@ The 'jobs' API is no longer supported.
 
         # Upload direct dependencies of workflow steps, get back mapping of files to keep references.
         # Also uploads docker images.
-        merged_map = upload_workflow_deps(self, tool)
+        merged_map = upload_workflow_deps(self, tool, runtimeContext)
 
         # Recreate process object (ArvadosWorkflow or
         # ArvadosCommandTool) because tool document may have been
@@ -584,17 +601,17 @@ The 'jobs' API is no longer supported.
         loadingContext.metadata = tool.metadata
         tool = load_tool(tool.tool, loadingContext)
 
-        existing_uuid = runtimeContext.update_workflow
-        if existing_uuid or runtimeContext.create_workflow:
+        if runtimeContext.update_workflow or runtimeContext.create_workflow:
             # Create a pipeline template or workflow record and exit.
             if self.work_api == "containers":
                 uuid = upload_workflow(self, tool, job_order,
-                                        self.project_uuid,
-                                        uuid=existing_uuid,
-                                        submit_runner_ram=runtimeContext.submit_runner_ram,
-                                        name=runtimeContext.name,
-                                        merged_map=merged_map,
-                                        submit_runner_image=runtimeContext.submit_runner_image)
+                                       runtimeContext.project_uuid,
+                                       runtimeContext,
+                                       uuid=runtimeContext.update_workflow,
+                                       submit_runner_ram=runtimeContext.submit_runner_ram,
+                                       name=runtimeContext.name,
+                                       merged_map=merged_map,
+                                       submit_runner_image=runtimeContext.submit_runner_image)
                 self.stdout.write(uuid + "\n")
                 return (None, "success")
 
@@ -603,7 +620,6 @@ The 'jobs' API is no longer supported.
         self.ignore_docker_for_reuse = runtimeContext.ignore_docker_for_reuse
         self.eval_timeout = runtimeContext.eval_timeout
 
-        runtimeContext = runtimeContext.copy()
         runtimeContext.use_container = True
         runtimeContext.tmpdir_prefix = "tmp"
         runtimeContext.work_api = self.work_api
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 7d4310b0e0..644713bce2 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -39,6 +39,7 @@ from cwltool.builder import Builder
 import schema_salad.validate as validate
 
 import arvados.collection
+import arvados.util
 from .util import collectionUUID
 from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
@@ -128,6 +129,9 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov
             set_secondary(fsaccess, builder, i, secondaryspec, primary, discovered)
         return
 
+    if inputschema == "File":
+        inputschema = {"type": "File"}
+
     if isinstance(inputschema, basestring):
         sd = search_schemadef(inputschema, reversed(builder.hints+builder.requirements))
         if sd:
@@ -163,10 +167,13 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov
             set_secondary(fsaccess, builder, {"type": inputschema["items"]}, secondaryspec, p, discovered)
 
     elif (inputschema["type"] == "File" and
-          secondaryspec and
           isinstance(primary, Mapping) and
-          primary.get("class") == "File" and
-          "secondaryFiles" not in primary):
+          primary.get("class") == "File"):
+
+        if "secondaryFiles" in primary or not secondaryspec:
+            # Nothing to do.
+            return
+
         #
         # Found a file, check for secondaryFiles
         #
@@ -174,9 +181,9 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov
         primary["secondaryFiles"] = secondaryspec
         for i, sf in enumerate(aslist(secondaryspec)):
             if builder.cwlVersion == "v1.0":
-                pattern = builder.do_eval(sf, context=primary)
+                pattern = sf
             else:
-                pattern = builder.do_eval(sf["pattern"], context=primary)
+                pattern = sf["pattern"]
             if pattern is None:
                 continue
             if isinstance(pattern, list):
@@ -213,7 +220,20 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov
                     "Expression must return list, object, string or null")
 
             if pattern is not None:
-                sfpath = substitute(primary["location"], pattern)
+                if "${" in pattern or "$(" in pattern:
+                    sfname = builder.do_eval(pattern, context=primary)
+                else:
+                    sfname = substitute(primary["basename"], pattern)
+
+                if sfname is None:
+                    continue
+
+                p_location = primary["location"]
+                if "/" in p_location:
+                    sfpath = (
+                        p_location[0 : p_location.rindex("/") + 1]
+                        + sfname
+                    )
 
             required = builder.do_eval(required, context=primary)
 
@@ -229,7 +249,7 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov
         primary["secondaryFiles"] = cmap(found)
         if discovered is not None:
             discovered[primary["location"]] = primary["secondaryFiles"]
-    elif inputschema["type"] not in primitive_types_set:
+    elif inputschema["type"] not in primitive_types_set and inputschema["type"] not in ("File", "Directory"):
         set_secondary(fsaccess, builder, inputschema["type"], secondaryspec, primary, discovered)
 
 def discover_secondary_files(fsaccess, builder, inputs, job_order, discovered=None):
@@ -239,7 +259,7 @@ def discover_secondary_files(fsaccess, builder, inputs, job_order, discovered=No
             set_secondary(fsaccess, builder, inputschema, None, primary, discovered)
 
 def upload_dependencies(arvrunner, name, document_loader,
-                        workflowobj, uri, loadref_run,
+                        workflowobj, uri, loadref_run, runtimeContext,
                         include_primary=True, discovered_secondaryfiles=None):
     """Upload the dependencies of the workflowobj document to Keep.
 
@@ -399,10 +419,16 @@ def upload_dependencies(arvrunner, name, document_loader,
                            single_collection=True,
                            optional_deps=optional_deps)
 
+    keeprefs = set()
+    def addkeepref(k):
+        if k.startswith("keep:"):
+            keeprefs.add(collection_pdh_pattern.match(k).group(1))
+
     def setloc(p):
         loc = p.get("location")
         if loc and (not loc.startswith("_:")) and (not loc.startswith("keep:")):
             p["location"] = mapper.mapper(p["location"]).resolved
+            addkeepref(p["location"])
             return
 
         if not loc:
@@ -424,7 +450,10 @@ def upload_dependencies(arvrunner, name, document_loader,
 
         gp = collection_uuid_pattern.match(loc)
         if not gp:
+            # Not a uuid pattern (must be a pdh pattern)
+            addkeepref(p["location"])
             return
+
         uuid = gp.groups()[0]
         if uuid not in uuid_map:
             raise SourceLine(p, "location", validate.ValidationException).makeError(
@@ -439,6 +468,38 @@ def upload_dependencies(arvrunner, name, document_loader,
         for d in discovered:
             discovered_secondaryfiles[mapper.mapper(d).resolved] = discovered[d]
 
+    if runtimeContext.copy_deps:
+        # Find referenced collections and copy them into the
+        # destination project, for easy sharing.
+        already_present = list(arvados.util.keyset_list_all(arvrunner.api.collections().list,
+                                     filters=[["portable_data_hash", "in", list(keeprefs)],
+                                              ["owner_uuid", "=", runtimeContext.project_uuid]],
+                                     select=["uuid", "portable_data_hash", "created_at"]))
+
+        keeprefs = keeprefs - set(a["portable_data_hash"] for a in already_present)
+        for kr in keeprefs:
+            col = arvrunner.api.collections().list(filters=[["portable_data_hash", "=", kr]],
+                                                  order="created_at desc",
+                                                   select=["name", "description", "properties", "portable_data_hash", "manifest_text", "storage_classes_desired", "trash_at"],
+                                                   limit=1).execute()
+            if len(col["items"]) == 0:
+                logger.warning("Cannot find collection with portable data hash %s", kr)
+                continue
+            col = col["items"][0]
+            try:
+                arvrunner.api.collections().create(body={"collection": {
+                    "owner_uuid": runtimeContext.project_uuid,
+                    "name": col["name"],
+                    "description": col["description"],
+                    "properties": col["properties"],
+                    "portable_data_hash": col["portable_data_hash"],
+                    "manifest_text": col["manifest_text"],
+                    "storage_classes_desired": col["storage_classes_desired"],
+                    "trash_at": col["trash_at"]
+                }}, ensure_unique_name=True).execute()
+            except Exception as e:
+                logger.warning("Unable copy collection to destination: %s", e)
+
     if "$schemas" in workflowobj:
         sch = CommentedSeq()
         for s in workflowobj["$schemas"]:
@@ -449,32 +510,36 @@ def upload_dependencies(arvrunner, name, document_loader,
     return mapper
 
 
-def upload_docker(arvrunner, tool):
+def upload_docker(arvrunner, tool, runtimeContext):
     """Uploads Docker images used in CommandLineTool objects."""
 
     if isinstance(tool, CommandLineTool):
         (docker_req, docker_is_req) = tool.get_requirement("DockerRequirement")
         if docker_req:
             if docker_req.get("dockerOutputDirectory") and arvrunner.work_api != "containers":
-                # TODO: can be supported by containers API, but not jobs API.
                 raise SourceLine(docker_req, "dockerOutputDirectory", UnsupportedRequirement).makeError(
                     "Option 'dockerOutputDirectory' of DockerRequirement not supported.")
-            arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, docker_req, True, arvrunner.project_uuid,
-                                                       arvrunner.runtimeContext.force_docker_pull,
-                                                       arvrunner.runtimeContext.tmp_outdir_prefix,
-                                                       arvrunner.runtimeContext.match_local_docker)
+
+            arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, docker_req, True,
+                                                       runtimeContext.project_uuid,
+                                                       runtimeContext.force_docker_pull,
+                                                       runtimeContext.tmp_outdir_prefix,
+                                                       runtimeContext.match_local_docker,
+                                                       runtimeContext.copy_deps)
         else:
             arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": "arvados/jobs:"+__version__},
-                                                       True, arvrunner.project_uuid,
-                                                       arvrunner.runtimeContext.force_docker_pull,
-                                                       arvrunner.runtimeContext.tmp_outdir_prefix,
-                                                       arvrunner.runtimeContext.match_local_docker)
+                                                       True,
+                                                       runtimeContext.project_uuid,
+                                                       runtimeContext.force_docker_pull,
+                                                       runtimeContext.tmp_outdir_prefix,
+                                                       runtimeContext.match_local_docker,
+                                                       runtimeContext.copy_deps)
     elif isinstance(tool, cwltool.workflow.Workflow):
         for s in tool.steps:
-            upload_docker(arvrunner, s.embedded_tool)
+            upload_docker(arvrunner, s.embedded_tool, runtimeContext)
 
 
-def packed_workflow(arvrunner, tool, merged_map):
+def packed_workflow(arvrunner, tool, merged_map, runtimeContext):
     """Create a packed workflow.
 
     A "packed" workflow is one where all the components have been combined into a single document."""
@@ -503,10 +568,11 @@ def packed_workflow(arvrunner, tool, merged_map):
                     v["secondaryFiles"] = merged_map[cur_id].secondaryFiles[v["location"]]
             if v.get("class") == "DockerRequirement":
                 v["http://arvados.org/cwl#dockerCollectionPDH"] = arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, v, True,
-                                                                                                             arvrunner.project_uuid,
-                                                                                                             arvrunner.runtimeContext.force_docker_pull,
-                                                                                                             arvrunner.runtimeContext.tmp_outdir_prefix,
-                                                                                                             arvrunner.runtimeContext.match_local_docker)
+                                                                                                             runtimeContext.project_uuid,
+                                                                                                             runtimeContext.force_docker_pull,
+                                                                                                             runtimeContext.tmp_outdir_prefix,
+                                                                                                             runtimeContext.match_local_docker,
+                                                                                                             runtimeContext.copy_deps)
             for l in v:
                 visit(v[l], cur_id)
         if isinstance(v, list):
@@ -527,7 +593,7 @@ def tag_git_version(packed):
             packed["http://schema.org/version"] = githash
 
 
-def upload_job_order(arvrunner, name, tool, job_order):
+def upload_job_order(arvrunner, name, tool, job_order, runtimeContext):
     """Upload local files referenced in the input object and return updated input
     object with 'location' updated to the proper keep references.
     """
@@ -563,7 +629,8 @@ def upload_job_order(arvrunner, name, tool, job_order):
                                     tool.doc_loader,
                                     job_order,
                                     job_order.get("id", "#"),
-                                    False)
+                                    False,
+                                    runtimeContext)
 
     if "id" in job_order:
         del job_order["id"]
@@ -577,10 +644,10 @@ def upload_job_order(arvrunner, name, tool, job_order):
 
 FileUpdates = namedtuple("FileUpdates", ["resolved", "secondaryFiles"])
 
-def upload_workflow_deps(arvrunner, tool):
+def upload_workflow_deps(arvrunner, tool, runtimeContext):
     # Ensure that Docker images needed by this workflow are available
 
-    upload_docker(arvrunner, tool)
+    upload_docker(arvrunner, tool, runtimeContext)
 
     document_loader = tool.doc_loader
 
@@ -595,6 +662,7 @@ def upload_workflow_deps(arvrunner, tool):
                                      deptool,
                                      deptool["id"],
                                      False,
+                                     runtimeContext,
                                      include_primary=False,
                                      discovered_secondaryfiles=discovered_secondaryfiles)
             document_loader.idx[deptool["id"]] = deptool
@@ -607,19 +675,22 @@ def upload_workflow_deps(arvrunner, tool):
 
     return merged_map
 
-def arvados_jobs_image(arvrunner, img):
+def arvados_jobs_image(arvrunner, img, runtimeContext):
     """Determine if the right arvados/jobs image version is available.  If not, try to pull and upload it."""
 
     try:
-        return arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid,
-                                                          arvrunner.runtimeContext.force_docker_pull,
-                                                          arvrunner.runtimeContext.tmp_outdir_prefix,
-                                                          arvrunner.runtimeContext.match_local_docker)
+        return arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": img},
+                                                          True,
+                                                          runtimeContext.project_uuid,
+                                                          runtimeContext.force_docker_pull,
+                                                          runtimeContext.tmp_outdir_prefix,
+                                                          runtimeContext.match_local_docker,
+                                                          runtimeContext.copy_deps)
     except Exception as e:
         raise Exception("Docker image %s is not available\n%s" % (img, e) )
 
 
-def upload_workflow_collection(arvrunner, name, packed):
+def upload_workflow_collection(arvrunner, name, packed, runtimeContext):
     collection = arvados.collection.Collection(api_client=arvrunner.api,
                                                keep_client=arvrunner.keep_client,
                                                num_retries=arvrunner.num_retries)
@@ -628,15 +699,15 @@ def upload_workflow_collection(arvrunner, name, packed):
 
     filters = [["portable_data_hash", "=", collection.portable_data_hash()],
                ["name", "like", name+"%"]]
-    if arvrunner.project_uuid:
-        filters.append(["owner_uuid", "=", arvrunner.project_uuid])
+    if runtimeContext.project_uuid:
+        filters.append(["owner_uuid", "=", runtimeContext.project_uuid])
     exists = arvrunner.api.collections().list(filters=filters).execute(num_retries=arvrunner.num_retries)
 
     if exists["items"]:
         logger.info("Using collection %s", exists["items"][0]["uuid"])
     else:
         collection.save_new(name=name,
-                            owner_uuid=arvrunner.project_uuid,
+                            owner_uuid=runtimeContext.project_uuid,
                             ensure_unique_name=True,
                             num_retries=arvrunner.num_retries)
         logger.info("Uploaded to %s", collection.manifest_locator())
diff --git a/sdk/cwl/test_with_arvbox.sh b/sdk/cwl/test_with_arvbox.sh
index d38414fc81..354d6f0e56 100755
--- a/sdk/cwl/test_with_arvbox.sh
+++ b/sdk/cwl/test_with_arvbox.sh
@@ -83,6 +83,12 @@ fi
 
 arvbox start $config $tag
 
+# Copy the integration test suite from our local arvados clone instead
+# of using the one inside the container, so we can make changes to the
+# integration tests without necessarily having to rebuilding the
+# container image.
+docker cp -L $(readlink -f $(dirname $0)/tests) $ARVBOX_CONTAINER:/usr/src/arvados/sdk/cwl
+
 arvbox pipe <<EOF
 set -eu -o pipefail
 
@@ -134,25 +140,6 @@ export ARVADOS_API_TOKEN=\$(cat /var/lib/arvados-arvbox/superuser_token)
 
 if test -n "$build" ; then
   /usr/src/arvados/build/build-dev-docker-jobs-image.sh
-elif test "$tag" = "latest" ; then
-  arv-keepdocker --pull arvados/jobs $tag
-else
-  set +u
-  export WORKSPACE=/usr/src/arvados
-  . /usr/src/arvados/build/run-library.sh
-  TMPHERE=\$(pwd)
-  cd /usr/src/arvados
-
-  # This defines python_sdk_version and cwl_runner_version with python-style
-  # package suffixes (.dev/rc)
-  calculate_python_sdk_cwl_package_versions
-
-  cd \$TMPHERE
-  set -u
-
-  arv-keepdocker --pull arvados/jobs \$cwl_runner_version
-  docker tag arvados/jobs:\$cwl_runner_version arvados/jobs:latest
-  arv-keepdocker arvados/jobs latest
 fi
 
 EXTRA=--compute-checksum
@@ -162,13 +149,29 @@ if [[ $devcwl -eq 1 ]] ; then
 fi
 
 env
+
+# Skip docker_entrypoint test because it fails on singularity
+#
+# Skip timelimit_invalid_wf test because the timeout is very short
+# (5s) and singularity containers loading off an arv-mount take too long
+# to start and get incorrectly terminated
+#
+# Skip test 199 in the v1.1 suite because it has different output
+# depending on whether there is a pty associated with stdout (fixed in
+# the v1.2 suite)
+#
+# Skip test 307 in the v1.2 suite because the test relied on
+# secondary file behavior of cwltool that wasn't actually correct to specification
+
 if [[ "$suite" = "integration" ]] ; then
    cd /usr/src/arvados/sdk/cwl/tests
    exec ./arvados-tests.sh $@
 elif [[ "$suite" = "conformance-v1.2" ]] ; then
-   exec cwltest --tool arvados-cwl-runner --test conformance_tests.yaml -N307 $@ -- \$EXTRA
-else
-   exec ./run_test.sh RUNNER=arvados-cwl-runner EXTRA="\$EXTRA" $@
+   exec cwltest --tool arvados-cwl-runner --test conformance_tests.yaml -Sdocker_entrypoint,timelimit_invalid_wf -N307 $@ -- \$EXTRA
+elif [[ "$suite" = "conformance-v1.1" ]] ; then
+   exec cwltest --tool arvados-cwl-runner --test conformance_tests.yaml -Sdocker_entrypoint,timelimit_invalid_wf -N199 $@ -- \$EXTRA
+elif [[ "$suite" = "conformance-v1.0" ]] ; then
+   exec cwltest --tool arvados-cwl-runner --test v1.0/conformance_test_v1.0.yaml -Sdocker_entrypoint $@ -- \$EXTRA
 fi
 EOF
 
diff --git a/sdk/cwl/tests/19070-copy-deps.cwl b/sdk/cwl/tests/19070-copy-deps.cwl
new file mode 100644
index 0000000000..b0d61700ef
--- /dev/null
+++ b/sdk/cwl/tests/19070-copy-deps.cwl
@@ -0,0 +1,17 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.2
+class: CommandLineTool
+baseCommand: echo
+inputs:
+  message:
+    type: File
+    inputBinding:
+      position: 1
+    default:
+      class: File
+      location: keep:d7514270f356df848477718d58308cc4+94/b
+
+outputs: []
diff --git a/sdk/cwl/tests/19109-upload-secondary.cwl b/sdk/cwl/tests/19109-upload-secondary.cwl
new file mode 100644
index 0000000000..933496b6a5
--- /dev/null
+++ b/sdk/cwl/tests/19109-upload-secondary.cwl
@@ -0,0 +1,42 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.2
+class: Workflow
+
+requirements:
+  InlineJavascriptRequirement: {}
+
+inputs:
+  file1:
+    type: File?
+    secondaryFiles:
+      - pattern: .tbi
+        required: true
+  file2:
+    type: File
+    secondaryFiles:
+      - pattern: |
+          ${
+          return self.basename + '.tbi';
+          }
+        required: true
+outputs:
+  out:
+    type: File
+    outputSource: cat/out
+  out2:
+    type: File
+    outputSource: cat2/out
+steps:
+  cat:
+    in:
+      inp: file1
+    run: cat2.cwl
+    out: [out]
+  cat2:
+    in:
+      inp: file2
+    run: cat2.cwl
+    out: [out]
diff --git a/sdk/cwl/tests/19109-upload-secondary.yml b/sdk/cwl/tests/19109-upload-secondary.yml
new file mode 100644
index 0000000000..af4147ae9a
--- /dev/null
+++ b/sdk/cwl/tests/19109-upload-secondary.yml
@@ -0,0 +1,10 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+file1:
+  class: File
+  location: 19109-upload-secondary/file1.txt
+file2:
+  class: File
+  location: 19109-upload-secondary/file2.txt
diff --git a/sdk/cwl/tests/19109-upload-secondary/file1.txt b/sdk/cwl/tests/19109-upload-secondary/file1.txt
new file mode 100644
index 0000000000..41549dbf8c
--- /dev/null
+++ b/sdk/cwl/tests/19109-upload-secondary/file1.txt
@@ -0,0 +1 @@
+strawberry
diff --git a/sdk/cwl/tests/19109-upload-secondary/file1.txt.tbi b/sdk/cwl/tests/19109-upload-secondary/file1.txt.tbi
new file mode 100644
index 0000000000..c9438bd232
--- /dev/null
+++ b/sdk/cwl/tests/19109-upload-secondary/file1.txt.tbi
@@ -0,0 +1 @@
+blueberry
\ No newline at end of file
diff --git a/sdk/cwl/tests/19109-upload-secondary/file2.txt b/sdk/cwl/tests/19109-upload-secondary/file2.txt
new file mode 100644
index 0000000000..637a09b86a
--- /dev/null
+++ b/sdk/cwl/tests/19109-upload-secondary/file2.txt
@@ -0,0 +1 @@
+banana
diff --git a/sdk/cwl/tests/19109-upload-secondary/file2.txt.tbi b/sdk/cwl/tests/19109-upload-secondary/file2.txt.tbi
new file mode 100644
index 0000000000..4d5e62525a
--- /dev/null
+++ b/sdk/cwl/tests/19109-upload-secondary/file2.txt.tbi
@@ -0,0 +1 @@
+mango
\ No newline at end of file
diff --git a/sdk/cwl/tests/arvados-tests.sh b/sdk/cwl/tests/arvados-tests.sh
index 9cb5234cf0..7d27523d30 100755
--- a/sdk/cwl/tests/arvados-tests.sh
+++ b/sdk/cwl/tests/arvados-tests.sh
@@ -1,9 +1,14 @@
-#!/bin/sh
+#!/bin/bash
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
+#
+# This in an additional integration test suite for Arvados specific
+# bugs and features that are not covered by the unit tests or CWL
+# conformance tests.
+#
 
-set -e
+set -ex
 
 if ! arv-get d7514270f356df848477718d58308cc4+94 > /dev/null ; then
     arv-put --portable-data-hash testdir/*
@@ -18,6 +23,18 @@ if ! arv-get 20850f01122e860fb878758ac1320877+71 > /dev/null ; then
     arv-put --portable-data-hash samples/sample1_S01_R1_001.fastq.gz
 fi
 
+# Use the python executor associated with the installed OS package, if present.
+python=$(((ls /usr/share/python3*/dist/python3-arvados-cwl-runner/bin/python || echo python3) | head -n1) 2>/dev/null)
+
+# Test for #18888
+# This is a standalone test because the bug was observed with this
+# command line and was thought to be due to command line handling.
 arvados-cwl-runner 18888-download_def.cwl --scripts scripts/
 
+# Test for #19070
+# The most effective way to test this seemed to be to write an
+# integration test to check for the expected behavior.
+$python test_copy_deps.py
+
+# Run integration tests
 exec cwltest --test arvados-tests.yml --tool arvados-cwl-runner $@ -- --disable-reuse --compute-checksum --api=containers
diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index 9e691bdba5..2f309cfe81 100644
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -449,3 +449,23 @@
   output: {}
   tool: 18994-basename/wf_ren.cwl
   doc: "Test issue 18994 - correctly stage file with modified basename"
+
+- job: 19109-upload-secondary.yml
+  output: {
+    "out": {
+        "basename": "file1.catted",
+        "class": "File",
+        "location": "file1.catted",
+        "size": 20,
+        "checksum": "sha1$c4cead17cebdd829f38c48e18a28f1da72339ef7"
+    },
+    "out2": {
+        "basename": "file2.catted",
+        "checksum": "sha1$6f71c5d1512519ede45bedfdd624e05fd8037b0d",
+        "class": "File",
+        "location": "file2.catted",
+        "size": 12
+    }
+  }
+  tool: 19109-upload-secondary.cwl
+  doc: "Test issue 19109 - correctly discover & upload secondary files"
diff --git a/sdk/cwl/tests/cat2.cwl b/sdk/cwl/tests/cat2.cwl
new file mode 100644
index 0000000000..82d93efbf0
--- /dev/null
+++ b/sdk/cwl/tests/cat2.cwl
@@ -0,0 +1,17 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.2
+class: CommandLineTool
+inputs:
+  - id: inp
+    type: File
+    secondaryFiles:
+      - pattern: .tbi
+        required: true
+stdout: $(inputs.inp.nameroot).catted
+outputs:
+  out:
+    type: stdout
+arguments: [cat, '$(inputs.inp.path)', '$(inputs.inp.secondaryFiles[0].path)']
diff --git a/sdk/cwl/tests/scripts/download_all_data.sh b/sdk/cwl/tests/scripts/download_all_data.sh
index d3a9d78762..7c769b5848 100755
--- a/sdk/cwl/tests/scripts/download_all_data.sh
+++ b/sdk/cwl/tests/scripts/download_all_data.sh
@@ -1,7 +1,7 @@
+#!/bin/sh
+
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
 
-#!/bin/bash
-
 echo bubble
diff --git a/sdk/cwl/tests/test_copy_deps.py b/sdk/cwl/tests/test_copy_deps.py
new file mode 100644
index 0000000000..853a7d3609
--- /dev/null
+++ b/sdk/cwl/tests/test_copy_deps.py
@@ -0,0 +1,152 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import arvados
+import subprocess
+
+api = arvados.api()
+
+def check_contents(group, wf_uuid):
+    contents = api.groups().contents(uuid=group["uuid"]).execute()
+    if len(contents["items"]) != 3:
+        raise Exception("Expected 3 items in "+group["uuid"]+" was "+len(contents["items"]))
+
+    found = False
+    for c in contents["items"]:
+        if c["kind"] == "arvados#workflow" and c["uuid"] == wf_uuid:
+            found = True
+    if not found:
+        raise Exception("Couldn't find workflow in "+group["uuid"])
+
+    found = False
+    for c in contents["items"]:
+        if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
+            found = True
+    if not found:
+        raise Exception("Couldn't find collection dependency")
+
+    found = False
+    for c in contents["items"]:
+        if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
+            found = True
+    if not found:
+        raise Exception("Couldn't find jobs image dependency")
+
+
+def test_create():
+    group = api.groups().create(body={"group": {"name": "test-19070-project-1", "group_class": "project"}}, ensure_unique_name=True).execute()
+    try:
+        contents = api.groups().contents(uuid=group["uuid"]).execute()
+        if len(contents["items"]) != 0:
+            raise Exception("Expected 0 items")
+
+        # Create workflow, by default should also copy dependencies
+        cmd = ["arvados-cwl-runner", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]
+        print(" ".join(cmd))
+        wf_uuid = subprocess.check_output(cmd)
+        wf_uuid = wf_uuid.decode("utf-8").strip()
+        check_contents(group, wf_uuid)
+    finally:
+        api.groups().delete(uuid=group["uuid"]).execute()
+
+
+def test_update():
+    group = api.groups().create(body={"group": {"name": "test-19070-project-2", "group_class": "project"}}, ensure_unique_name=True).execute()
+    try:
+        contents = api.groups().contents(uuid=group["uuid"]).execute()
+        if len(contents["items"]) != 0:
+            raise Exception("Expected 0 items")
+
+        # Create workflow, but with --no-copy-deps it shouldn't copy anything
+        cmd = ["arvados-cwl-runner", "--no-copy-deps", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]
+        print(" ".join(cmd))
+        wf_uuid = subprocess.check_output(cmd)
+        wf_uuid = wf_uuid.decode("utf-8").strip()
+
+        contents = api.groups().contents(uuid=group["uuid"]).execute()
+        if len(contents["items"]) != 1:
+            raise Exception("Expected 1 items")
+
+        found = False
+        for c in contents["items"]:
+            if c["kind"] == "arvados#workflow" and c["uuid"] == wf_uuid:
+                found = True
+        if not found:
+            raise Exception("Couldn't find workflow")
+
+        # Updating by default will copy missing items
+        cmd = ["arvados-cwl-runner", "--update-workflow", wf_uuid, "19070-copy-deps.cwl"]
+        print(" ".join(cmd))
+        wf_uuid = subprocess.check_output(cmd)
+        wf_uuid = wf_uuid.decode("utf-8").strip()
+        check_contents(group, wf_uuid)
+
+    finally:
+        api.groups().delete(uuid=group["uuid"]).execute()
+
+
+def test_execute():
+    group = api.groups().create(body={"group": {"name": "test-19070-project-3", "group_class": "project"}}, ensure_unique_name=True).execute()
+    try:
+        contents = api.groups().contents(uuid=group["uuid"]).execute()
+        if len(contents["items"]) != 0:
+            raise Exception("Expected 0 items")
+
+        # Execute workflow, shouldn't copy anything.
+        cmd = ["arvados-cwl-runner", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]
+        print(" ".join(cmd))
+        wf_uuid = subprocess.check_output(cmd)
+        wf_uuid = wf_uuid.decode("utf-8").strip()
+
+        contents = api.groups().contents(uuid=group["uuid"]).execute()
+        # container request
+        # final output collection
+        # container log
+        # step output collection
+        # container request log
+        if len(contents["items"]) != 5:
+            raise Exception("Expected 5 items")
+
+        found = False
+        for c in contents["items"]:
+            if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
+                found = True
+        if found:
+            raise Exception("Didn't expect to find collection dependency")
+
+        found = False
+        for c in contents["items"]:
+            if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
+                found = True
+        if found:
+            raise Exception("Didn't expect to find jobs image dependency")
+
+        # Execute workflow with --copy-deps
+        cmd = ["arvados-cwl-runner", "--project-uuid", group["uuid"], "--copy-deps", "19070-copy-deps.cwl"]
+        print(" ".join(cmd))
+        wf_uuid = subprocess.check_output(cmd)
+        wf_uuid = wf_uuid.decode("utf-8").strip()
+
+        contents = api.groups().contents(uuid=group["uuid"]).execute()
+        found = False
+        for c in contents["items"]:
+            if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
+                found = True
+        if not found:
+            raise Exception("Couldn't find collection dependency")
+
+        found = False
+        for c in contents["items"]:
+            if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
+                found = True
+        if not found:
+            raise Exception("Couldn't find jobs image dependency")
+
+    finally:
+        api.groups().delete(uuid=group["uuid"]).execute()
+
+if __name__ == '__main__':
+    test_create()
+    test_update()
+    test_execute()
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 5092fc4575..305d51e144 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -47,12 +47,14 @@ _rootDesc = None
 
 def stubs(func):
     @functools.wraps(func)
+    @mock.patch("arvados_cwl.arvdocker.determine_image_id")
     @mock.patch("uuid.uuid4")
     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
     @mock.patch("arvados.collection.KeepClient")
     @mock.patch("arvados.keep.KeepClient")
     @mock.patch("arvados.events.subscribe")
-    def wrapped(self, events, keep_client1, keep_client2, keepdocker, uuid4, *args, **kwargs):
+    def wrapped(self, events, keep_client1, keep_client2, keepdocker,
+                uuid4, determine_image_id, *args, **kwargs):
         class Stubs(object):
             pass
         stubs = Stubs()
@@ -63,6 +65,8 @@ def stubs(func):
                              "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
                              "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5"]
 
+        determine_image_id.return_value = None
+
         def putstub(p, **kwargs):
             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
         keep_client1().put.side_effect = putstub
@@ -77,7 +81,7 @@ def stubs(func):
             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
         }
-        def kd(a, b, image_name=None, image_tag=None):
+        def kd(a, b, image_name=None, image_tag=None, project_uuid=None):
             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
         stubs.keepdocker.side_effect = kd
 
@@ -1077,6 +1081,18 @@ class TestSubmit(unittest.TestCase):
                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
                                                                         "owner_uuid": "",
                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
+                                                            {"items": [{"created_at": "",
+                                                                        "head_uuid": "",
+                                                                        "link_class": "docker_image_hash",
+                                                                        "name": "123456",
+                                                                        "owner_uuid": "",
+                                                                        "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
+                                                            {"items": [{"created_at": "",
+                                                                        "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
+                                                                        "link_class": "docker_image_repo+tag",
+                                                                        "name": "arvados/jobs:"+arvados_cwl.__version__,
+                                                                        "owner_uuid": "",
+                                                                        "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
                                                             {"items": [{"created_at": "",
                                                                         "head_uuid": "",
                                                                         "link_class": "docker_image_hash",
@@ -1090,12 +1106,18 @@ class TestSubmit(unittest.TestCase):
                                                                               "owner_uuid": "",
                                                                               "manifest_text": "",
                                                                               "properties": ""
-                                                                          }], "items_available": 1, "offset": 0},)
+                                                                              }], "items_available": 1, "offset": 0},
+                                                                  {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
+                                                                              "owner_uuid": "",
+                                                                              "manifest_text": "",
+                                                                              "properties": ""
+                                                                          }], "items_available": 1, "offset": 0})
         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
+
         self.assertEqual("9999999999999999999999999999999b+99",
-                         arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
+                         arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__, arvrunner.runtimeContext))
 
 
     @stubs
@@ -1599,6 +1621,9 @@ class TestCreateWorkflow(unittest.TestCase):
 
     @stubs
     def test_update(self, stubs):
+        project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
+        stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
+
         exited = arvados_cwl.main(
             ["--update-workflow", self.existing_workflow_uuid,
              "--debug",
@@ -1610,6 +1635,7 @@ class TestCreateWorkflow(unittest.TestCase):
                 "name": "submit_wf.cwl",
                 "description": "",
                 "definition": self.expect_workflow,
+                "owner_uuid": project_uuid
             }
         }
         stubs.api.workflows().update.assert_called_with(
@@ -1622,6 +1648,9 @@ class TestCreateWorkflow(unittest.TestCase):
 
     @stubs
     def test_update_name(self, stubs):
+        project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
+        stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
+
         exited = arvados_cwl.main(
             ["--update-workflow", self.existing_workflow_uuid,
              "--debug", "--name", "testing 123",
@@ -1633,6 +1662,7 @@ class TestCreateWorkflow(unittest.TestCase):
                 "name": "testing 123",
                 "description": "",
                 "definition": self.expect_workflow,
+                "owner_uuid": project_uuid
             }
         }
         stubs.api.workflows().update.assert_called_with(
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 6a90c30ce4..319fa1a38f 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -348,6 +348,7 @@ type Services struct {
 	Controller     Service
 	DispatchCloud  Service
 	DispatchLSF    Service
+	DispatchSLURM  Service
 	GitHTTP        Service
 	GitSSH         Service
 	Health         Service
@@ -605,6 +606,7 @@ const (
 	ServiceNameController    ServiceName = "arvados-controller"
 	ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 	ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
+	ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
 	ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
 	ServiceNameHealth        ServiceName = "arvados-health"
 	ServiceNameKeepbalance   ServiceName = "keep-balance"
@@ -624,6 +626,7 @@ func (svcs Services) Map() map[ServiceName]Service {
 		ServiceNameController:    svcs.Controller,
 		ServiceNameDispatchCloud: svcs.DispatchCloud,
 		ServiceNameDispatchLSF:   svcs.DispatchLSF,
+		ServiceNameDispatchSLURM: svcs.DispatchSLURM,
 		ServiceNameGitHTTP:       svcs.GitHTTP,
 		ServiceNameHealth:        svcs.Health,
 		ServiceNameKeepbalance:   svcs.Keepbalance,
diff --git a/sdk/go/health/aggregator_test.go b/sdk/go/health/aggregator_test.go
index 5f60cf67f3..481054c4de 100644
--- a/sdk/go/health/aggregator_test.go
+++ b/sdk/go/health/aggregator_test.go
@@ -327,6 +327,7 @@ func (s *AggregatorSuite) setAllServiceURLs(listen string) {
 		&svcs.Controller,
 		&svcs.DispatchCloud,
 		&svcs.DispatchLSF,
+		&svcs.DispatchSLURM,
 		&svcs.GitHTTP,
 		&svcs.Keepbalance,
 		&svcs.Keepproxy,
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 537ea3a945..db4edd2dfa 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -85,7 +85,8 @@ class DockerError(Exception):
 def popen_docker(cmd, *args, **kwargs):
     manage_stdin = ('stdin' not in kwargs)
     kwargs.setdefault('stdin', subprocess.PIPE)
-    kwargs.setdefault('stdout', sys.stderr)
+    kwargs.setdefault('stdout', subprocess.PIPE)
+    kwargs.setdefault('stderr', subprocess.PIPE)
     try:
         docker_proc = subprocess.Popen(['docker'] + cmd, *args, **kwargs)
     except OSError:  # No docker in $PATH, try docker.io
@@ -257,7 +258,7 @@ def _new_image_listing(link, dockerhash, repo='<none>', tag='<none>'):
         'tag': tag,
         }
 
-def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None):
+def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None, project_uuid=None):
     """List all Docker images known to the api_client with image_name and
     image_tag.  If no image_name is given, defaults to listing all
     Docker images.
@@ -272,13 +273,18 @@ def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None)
     search_filters = []
     repo_links = None
     hash_links = None
+
+    project_filter = []
+    if project_uuid is not None:
+        project_filter = [["owner_uuid", "=", project_uuid]]
+
     if image_name:
         # Find images with the name the user specified.
         search_links = _get_docker_links(
             api_client, num_retries,
             filters=[['link_class', '=', 'docker_image_repo+tag'],
                      ['name', '=',
-                      '{}:{}'.format(image_name, image_tag or 'latest')]])
+                      '{}:{}'.format(image_name, image_tag or 'latest')]]+project_filter)
         if search_links:
             repo_links = search_links
         else:
@@ -286,7 +292,7 @@ def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None)
             search_links = _get_docker_links(
                 api_client, num_retries,
                 filters=[['link_class', '=', 'docker_image_hash'],
-                         ['name', 'ilike', image_name + '%']])
+                         ['name', 'ilike', image_name + '%']]+project_filter)
             hash_links = search_links
         # Only list information about images that were found in the search.
         search_filters.append(['head_uuid', 'in',
@@ -298,7 +304,7 @@ def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None)
     if hash_links is None:
         hash_links = _get_docker_links(
             api_client, num_retries,
-            filters=search_filters + [['link_class', '=', 'docker_image_hash']])
+            filters=search_filters + [['link_class', '=', 'docker_image_hash']]+project_filter)
     hash_link_map = {link['head_uuid']: link for link in reversed(hash_links)}
 
     # Each collection may have more than one name (though again, one name
@@ -308,7 +314,7 @@ def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None)
         repo_links = _get_docker_links(
             api_client, num_retries,
             filters=search_filters + [['link_class', '=',
-                                       'docker_image_repo+tag']])
+                                       'docker_image_repo+tag']]+project_filter)
     seen_image_names = collections.defaultdict(set)
     images = []
     for link in repo_links:
@@ -336,7 +342,7 @@ def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None)
     # Remove any image listings that refer to unknown collections.
     existing_coll_uuids = {coll['uuid'] for coll in arvados.util.list_all(
             api_client.collections().list, num_retries,
-            filters=[['uuid', 'in', [im['collection'] for im in images]]],
+            filters=[['uuid', 'in', [im['collection'] for im in images]]]+project_filter,
             select=['uuid'])}
     return [(image['collection'], image) for image in images
             if image['collection'] in existing_coll_uuids]
@@ -385,18 +391,25 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None)
     if args.pull and not find_image_hashes(args.image):
         pull_image(args.image, args.tag)
 
+    images_in_arv = list_images_in_arv(api, args.retries, args.image, args.tag)
+
+    image_hash = None
     try:
         image_hash = find_one_image_hash(args.image, args.tag)
+        if not docker_image_compatible(api, image_hash):
+            if args.force_image_format:
+                logger.warning("forcing incompatible image")
+            else:
+                logger.error("refusing to store " \
+                    "incompatible format (use --force-image-format to override)")
+                sys.exit(1)
     except DockerError as error:
-        logger.error(str(error))
-        sys.exit(1)
-
-    if not docker_image_compatible(api, image_hash):
-        if args.force_image_format:
-            logger.warning("forcing incompatible image")
+        if images_in_arv:
+            # We don't have Docker / we don't have the image locally,
+            # use image that's already uploaded to Arvados
+            image_hash = images_in_arv[0][1]['dockerhash']
         else:
-            logger.error("refusing to store " \
-                "incompatible format (use --force-image-format to override)")
+            logger.error(str(error))
             sys.exit(1)
 
     image_repo_tag = '{}:{}'.format(args.image, args.tag) if not image_hash.startswith(args.image.lower()) else None
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index 4d1e06613a..543390004b 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -30,6 +30,10 @@ http {
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
       proxy_set_header X-Forwarded-Proto https;
       proxy_redirect off;
+      proxy_max_temp_file_size 0;
+      proxy_request_buffering off;
+      proxy_buffering off;
+      proxy_http_version 1.1;
     }
   }
   upstream arv-git-http {
diff --git a/sdk/python/tests/test_arv_keepdocker.py b/sdk/python/tests/test_arv_keepdocker.py
index fd3a69cae4..8fbfad4377 100644
--- a/sdk/python/tests/test_arv_keepdocker.py
+++ b/sdk/python/tests/test_arv_keepdocker.py
@@ -48,11 +48,13 @@ class ArvKeepdockerTestCase(unittest.TestCase, tutil.VersionChecker):
                 self.run_arv_keepdocker(['--version'], sys.stderr)
         self.assertVersionOutput(out, err)
 
+    @mock.patch('arvados.commands.keepdocker.list_images_in_arv',
+                return_value=[])
     @mock.patch('arvados.commands.keepdocker.find_image_hashes',
                 return_value=['abc123'])
     @mock.patch('arvados.commands.keepdocker.find_one_image_hash',
                 return_value='abc123')
-    def test_image_format_compatibility(self, _1, _2):
+    def test_image_format_compatibility(self, _1, _2, _3):
         old_id = hashlib.sha256(b'old').hexdigest()
         new_id = 'sha256:'+hashlib.sha256(b'new').hexdigest()
         for supported, img_id, expect_ok in [
@@ -152,11 +154,13 @@ class ArvKeepdockerTestCase(unittest.TestCase, tutil.VersionChecker):
             self.run_arv_keepdocker(['[::1]/repo/img'], sys.stderr)
         find_image_mock.assert_called_with('[::1]/repo/img', 'latest')
 
+    @mock.patch('arvados.commands.keepdocker.list_images_in_arv',
+                return_value=[])
     @mock.patch('arvados.commands.keepdocker.find_image_hashes',
                 return_value=['abc123'])
     @mock.patch('arvados.commands.keepdocker.find_one_image_hash',
                 return_value='abc123')
-    def test_collection_property_update(self, _1, _2):
+    def test_collection_property_update(self, _1, _2, _3):
         image_id = 'sha256:'+hashlib.sha256(b'image').hexdigest()
         fakeDD = arvados.api('v1')._rootDesc
         fakeDD['dockerImageFormats'] = ['v2']
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 84105e1fc7..c31d799752 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -2,32 +2,48 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
-
 // Dispatcher service for Crunch that submits containers to the slurm queue.
+package dispatchslurm
 
 import (
 	"context"
-	"flag"
 	"fmt"
 	"log"
 	"math"
+	"net/http"
 	"os"
 	"regexp"
 	"strings"
 	"time"
 
 	"git.arvados.org/arvados.git/lib/cmd"
-	"git.arvados.org/arvados.git/lib/config"
 	"git.arvados.org/arvados.git/lib/dispatchcloud"
+	"git.arvados.org/arvados.git/lib/service"
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"git.arvados.org/arvados.git/sdk/go/arvadosclient"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
 	"git.arvados.org/arvados.git/sdk/go/dispatch"
 	"github.com/coreos/go-systemd/daemon"
-	"github.com/ghodss/yaml"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 )
 
+var Command cmd.Handler = service.Command(arvados.ServiceNameDispatchSLURM, newHandler)
+
+func newHandler(ctx context.Context, cluster *arvados.Cluster, _ string, _ *prometheus.Registry) service.Handler {
+	logger := ctxlog.FromContext(ctx)
+	disp := &Dispatcher{logger: logger, cluster: cluster}
+	if err := disp.configure(); err != nil {
+		return service.ErrorHandler(ctx, cluster, err)
+	}
+	disp.setup()
+	go func() {
+		disp.err = disp.run()
+		close(disp.done)
+	}()
+	return disp
+}
+
 type logger interface {
 	dispatch.Logger
 	Fatalf(string, ...interface{})
@@ -35,10 +51,6 @@ type logger interface {
 
 const initialNiceValue int64 = 10000
 
-var (
-	version = "dev"
-)
-
 type Dispatcher struct {
 	*dispatch.Dispatcher
 	logger  logrus.FieldLogger
@@ -46,75 +58,32 @@ type Dispatcher struct {
 	sqCheck *SqueueChecker
 	slurm   Slurm
 
+	done chan struct{}
+	err  error
+
 	Client arvados.Client
 }
 
-func main() {
-	logger := logrus.StandardLogger()
-	if os.Getenv("DEBUG") != "" {
-		logger.SetLevel(logrus.DebugLevel)
-	}
-	logger.Formatter = &logrus.JSONFormatter{
-		TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
-	}
-	disp := &Dispatcher{logger: logger}
-	err := disp.Run(os.Args[0], os.Args[1:])
-	if err != nil {
-		logrus.Fatalf("%s", err)
-	}
+func (disp *Dispatcher) CheckHealth() error {
+	return disp.err
 }
 
-func (disp *Dispatcher) Run(prog string, args []string) error {
-	if err := disp.configure(prog, args); err != nil {
-		return err
-	}
-	disp.setup()
-	return disp.run()
+func (disp *Dispatcher) Done() <-chan struct{} {
+	return disp.done
+}
+
+func (disp *Dispatcher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	http.NotFound(w, r)
 }
 
-// configure() loads config files. Tests skip this.
-func (disp *Dispatcher) configure(prog string, args []string) error {
+// configure() loads config files. Some tests skip this (see
+// StubbedSuite).
+func (disp *Dispatcher) configure() error {
 	if disp.logger == nil {
 		disp.logger = logrus.StandardLogger()
 	}
-	flags := flag.NewFlagSet(prog, flag.ContinueOnError)
-	flags.Usage = func() { usage(flags) }
-
-	loader := config.NewLoader(nil, disp.logger)
-	loader.SetupFlags(flags)
-
-	dumpConfig := flag.Bool(
-		"dump-config",
-		false,
-		"write current configuration to stdout and exit")
-	getVersion := flags.Bool(
-		"version",
-		false,
-		"Print version information and exit.")
-
-	args = loader.MungeLegacyConfigArgs(disp.logger, args, "-legacy-crunch-dispatch-slurm-config")
-	if ok, code := cmd.ParseFlags(flags, prog, args, "", os.Stderr); !ok {
-		os.Exit(code)
-	}
-
-	// Print version information if requested
-	if *getVersion {
-		fmt.Printf("crunch-dispatch-slurm %s\n", version)
-		return nil
-	}
-
-	disp.logger.Printf("crunch-dispatch-slurm %s started", version)
-
-	cfg, err := loader.Load()
-	if err != nil {
-		return err
-	}
-
-	if disp.cluster, err = cfg.GetCluster(""); err != nil {
-		return fmt.Errorf("config error: %s", err)
-	}
-
 	disp.logger = disp.logger.WithField("ClusterID", disp.cluster.ClusterID)
+	disp.logger.Printf("crunch-dispatch-slurm %s started", cmd.Version.String())
 
 	disp.Client.APIHost = disp.cluster.Services.Controller.ExternalURL.Host
 	disp.Client.AuthToken = disp.cluster.SystemRootToken
@@ -137,23 +106,12 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
 	} else {
 		disp.logger.Warnf("Client credentials missing from config, so falling back on environment variables (deprecated).")
 	}
-
-	if *dumpConfig {
-		out, err := yaml.Marshal(cfg)
-		if err != nil {
-			return err
-		}
-		_, err = os.Stdout.Write(out)
-		if err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 
 // setup() initializes private fields after configure().
 func (disp *Dispatcher) setup() {
+	disp.done = make(chan struct{})
 	arv, err := arvadosclient.MakeArvadosClient()
 	if err != nil {
 		disp.logger.Fatalf("Error making Arvados client: %v", err)
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
index cf83257dad..fb433e65cd 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
@@ -2,12 +2,13 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package dispatchslurm
 
 import (
 	"bytes"
 	"context"
 	"errors"
+	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -19,10 +20,13 @@ import (
 	"testing"
 	"time"
 
+	"git.arvados.org/arvados.git/lib/cmd"
+	"git.arvados.org/arvados.git/lib/config"
 	"git.arvados.org/arvados.git/lib/dispatchcloud"
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"git.arvados.org/arvados.git/sdk/go/arvadosclient"
 	"git.arvados.org/arvados.git/sdk/go/arvadostest"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
 	"git.arvados.org/arvados.git/sdk/go/dispatch"
 	"github.com/sirupsen/logrus"
 	. "gopkg.in/check.v1"
@@ -387,6 +391,7 @@ func (s *StubbedSuite) TestSbatchPartition(c *C) {
 }
 
 func (s *StubbedSuite) TestLoadLegacyConfig(c *C) {
+	log := ctxlog.TestLogger(c)
 	content := []byte(`
 Client:
   APIHost: example.com
@@ -402,36 +407,42 @@ ReserveExtraRAM: 12345
 MinRetryPeriod: 13s
 BatchSize: 99
 `)
-	tmpfile, err := ioutil.TempFile("", "example")
-	if err != nil {
-		c.Error(err)
-	}
-
-	defer os.Remove(tmpfile.Name()) // clean up
-
-	if _, err := tmpfile.Write(content); err != nil {
-		c.Error(err)
-	}
-	if err := tmpfile.Close(); err != nil {
-		c.Error(err)
+	tmpfile := c.MkDir() + "/config.yml"
+	err := ioutil.WriteFile(tmpfile, content, 0777)
+	c.Assert(err, IsNil)
 
-	}
 	os.Setenv("ARVADOS_KEEP_SERVICES", "")
-	err = s.disp.configure("crunch-dispatch-slurm", []string{"-config", tmpfile.Name()})
-	c.Check(err, IsNil)
 
-	c.Check(s.disp.cluster.Services.Controller.ExternalURL, Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
-	c.Check(s.disp.cluster.SystemRootToken, Equals, "abcdefg")
-	c.Check(s.disp.cluster.Containers.SLURM.SbatchArgumentsList, DeepEquals, []string{"--foo", "bar"})
-	c.Check(s.disp.cluster.Containers.CloudVMs.PollInterval, Equals, arvados.Duration(12*time.Second))
-	c.Check(s.disp.cluster.Containers.SLURM.PrioritySpread, Equals, int64(42))
-	c.Check(s.disp.cluster.Containers.CrunchRunCommand, Equals, "x-crunch-run")
-	c.Check(s.disp.cluster.Containers.CrunchRunArgumentsList, DeepEquals, []string{"--cgroup-parent-subsystem=memory"})
-	c.Check(s.disp.cluster.Containers.ReserveExtraRAM, Equals, arvados.ByteSize(12345))
-	c.Check(s.disp.cluster.Containers.MinRetryPeriod, Equals, arvados.Duration(13*time.Second))
-	c.Check(s.disp.cluster.API.MaxItemsPerResponse, Equals, 99)
-	c.Check(s.disp.cluster.Containers.SLURM.SbatchEnvironmentVariables, DeepEquals, map[string]string{
+	flags := flag.NewFlagSet("", flag.ContinueOnError)
+	flags.SetOutput(os.Stderr)
+	loader := config.NewLoader(&bytes.Buffer{}, log)
+	loader.SetupFlags(flags)
+	args := loader.MungeLegacyConfigArgs(log, []string{"-config", tmpfile}, "-legacy-"+string(arvados.ServiceNameDispatchSLURM)+"-config")
+	ok, _ := cmd.ParseFlags(flags, "crunch-dispatch-slurm", args, "", os.Stderr)
+	c.Check(ok, Equals, true)
+	cfg, err := loader.Load()
+	c.Assert(err, IsNil)
+	cluster, err := cfg.GetCluster("")
+	c.Assert(err, IsNil)
+
+	c.Check(cluster.Services.Controller.ExternalURL, Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
+	c.Check(cluster.SystemRootToken, Equals, "abcdefg")
+	c.Check(cluster.Containers.SLURM.SbatchArgumentsList, DeepEquals, []string{"--foo", "bar"})
+	c.Check(cluster.Containers.CloudVMs.PollInterval, Equals, arvados.Duration(12*time.Second))
+	c.Check(cluster.Containers.SLURM.PrioritySpread, Equals, int64(42))
+	c.Check(cluster.Containers.CrunchRunCommand, Equals, "x-crunch-run")
+	c.Check(cluster.Containers.CrunchRunArgumentsList, DeepEquals, []string{"--cgroup-parent-subsystem=memory"})
+	c.Check(cluster.Containers.ReserveExtraRAM, Equals, arvados.ByteSize(12345))
+	c.Check(cluster.Containers.MinRetryPeriod, Equals, arvados.Duration(13*time.Second))
+	c.Check(cluster.API.MaxItemsPerResponse, Equals, 99)
+	c.Check(cluster.Containers.SLURM.SbatchEnvironmentVariables, DeepEquals, map[string]string{
 		"ARVADOS_KEEP_SERVICES": "https://example.com/keep1 https://example.com/keep2",
 	})
+
+	// Ensure configure() copies SbatchEnvironmentVariables into
+	// the current process's environment (that's how they end up
+	// getting passed to sbatch).
+	s.disp.cluster = cluster
+	s.disp.configure()
 	c.Check(os.Getenv("ARVADOS_KEEP_SERVICES"), Equals, "https://example.com/keep1 https://example.com/keep2")
 }
diff --git a/services/crunch-dispatch-slurm/node_type.go b/services/crunch-dispatch-slurm/node_type.go
index d31322f182..738426c92d 100644
--- a/services/crunch-dispatch-slurm/node_type.go
+++ b/services/crunch-dispatch-slurm/node_type.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package dispatchslurm
 
 import (
 	"log"
diff --git a/services/crunch-dispatch-slurm/priority.go b/services/crunch-dispatch-slurm/priority.go
index 2312ce5952..515a98d323 100644
--- a/services/crunch-dispatch-slurm/priority.go
+++ b/services/crunch-dispatch-slurm/priority.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package dispatchslurm
 
 const defaultSpread int64 = 10
 
diff --git a/services/crunch-dispatch-slurm/priority_test.go b/services/crunch-dispatch-slurm/priority_test.go
index e80984c0fc..df1c27def7 100644
--- a/services/crunch-dispatch-slurm/priority_test.go
+++ b/services/crunch-dispatch-slurm/priority_test.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package dispatchslurm
 
 import (
 	. "gopkg.in/check.v1"
diff --git a/services/crunch-dispatch-slurm/script.go b/services/crunch-dispatch-slurm/script.go
index f559104d14..fb16e593e5 100644
--- a/services/crunch-dispatch-slurm/script.go
+++ b/services/crunch-dispatch-slurm/script.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package dispatchslurm
 
 import (
 	"strings"
diff --git a/services/crunch-dispatch-slurm/script_test.go b/services/crunch-dispatch-slurm/script_test.go
index a21aeeddad..00d70190dd 100644
--- a/services/crunch-dispatch-slurm/script_test.go
+++ b/services/crunch-dispatch-slurm/script_test.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package dispatchslurm
 
 import (
 	. "gopkg.in/check.v1"
diff --git a/services/crunch-dispatch-slurm/slurm.go b/services/crunch-dispatch-slurm/slurm.go
index 791f294df1..e59826f763 100644
--- a/services/crunch-dispatch-slurm/slurm.go
+++ b/services/crunch-dispatch-slurm/slurm.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package dispatchslurm
 
 import (
 	"fmt"
diff --git a/services/crunch-dispatch-slurm/squeue.go b/services/crunch-dispatch-slurm/squeue.go
index eae21e62b6..d4e41ed1fb 100644
--- a/services/crunch-dispatch-slurm/squeue.go
+++ b/services/crunch-dispatch-slurm/squeue.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package dispatchslurm
 
 import (
 	"bytes"
diff --git a/services/crunch-dispatch-slurm/squeue_test.go b/services/crunch-dispatch-slurm/squeue_test.go
index ce74fe61cc..d41e1982b4 100644
--- a/services/crunch-dispatch-slurm/squeue_test.go
+++ b/services/crunch-dispatch-slurm/squeue_test.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package dispatchslurm
 
 import (
 	"time"
diff --git a/services/crunch-dispatch-slurm/usage.go b/services/crunch-dispatch-slurm/usage.go
index 68a2305f74..785843b198 100644
--- a/services/crunch-dispatch-slurm/usage.go
+++ b/services/crunch-dispatch-slurm/usage.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package dispatchslurm
 
 import (
 	"flag"
diff --git a/services/health/main.go b/services/health/main.go
deleted file mode 100644
index 92bd377c80..0000000000
--- a/services/health/main.go
+++ /dev/null
@@ -1,39 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
-	"context"
-	"os"
-
-	"git.arvados.org/arvados.git/lib/cmd"
-	"git.arvados.org/arvados.git/lib/service"
-	"git.arvados.org/arvados.git/sdk/go/arvados"
-	"git.arvados.org/arvados.git/sdk/go/health"
-	"github.com/prometheus/client_golang/prometheus"
-)
-
-var (
-	version             = "dev"
-	command cmd.Handler = service.Command(arvados.ServiceNameHealth, newHandler)
-)
-
-func newHandler(ctx context.Context, cluster *arvados.Cluster, _ string, reg *prometheus.Registry) service.Handler {
-	mClockSkew := prometheus.NewGauge(prometheus.GaugeOpts{
-		Namespace: "arvados",
-		Subsystem: "health",
-		Name:      "clock_skew_seconds",
-		Help:      "Clock skew observed in most recent health check",
-	})
-	reg.MustRegister(mClockSkew)
-	return &health.Aggregator{
-		Cluster:         cluster,
-		MetricClockSkew: mClockSkew,
-	}
-}
-
-func main() {
-	os.Exit(command.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
-}
diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.base b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
index b984aeb7f5..8f20850ef4 100644
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.base
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
@@ -108,11 +108,12 @@ ADD $workdir/8D81803C0EBFCD88.asc /tmp/
 RUN apt-key add --no-tty /tmp/8D81803C0EBFCD88.asc && \
     rm -f /tmp/8D81803C0EBFCD88.asc
 
-RUN mkdir -p /etc/apt/sources.list.d && \
-    echo deb https://download.docker.com/linux/debian/ buster stable > /etc/apt/sources.list.d/docker.list && \
-    apt-get update && \
-    apt-get -yq --no-install-recommends install docker-ce=5:20.10.6~3-0~debian-buster && \
-    apt-get clean
+# docker is now installed by arvados-server install
+# RUN mkdir -p /etc/apt/sources.list.d && \
+#     echo deb https://download.docker.com/linux/debian/ buster stable > /etc/apt/sources.list.d/docker.list && \
+#     apt-get update && \
+#     apt-get -yq --no-install-recommends install docker-ce=5:20.10.6~3-0~debian-buster && \
+#     apt-get clean
 
 # Set UTF-8 locale
 RUN echo en_US.UTF-8 UTF-8 > /etc/locale.gen && locale-gen
diff --git a/tools/arvbox/lib/arvbox/docker/service/workbench2/run-service b/tools/arvbox/lib/arvbox/docker/service/workbench2/run-service
index 2b68cadafd..5268c7e17e 100755
--- a/tools/arvbox/lib/arvbox/docker/service/workbench2/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/workbench2/run-service
@@ -22,11 +22,15 @@ if test "$1" = "--only-deps" ; then
     exit
 fi
 
+API_HOST=${localip}:${services[controller-ssl]}
+
+if test -f /usr/src/workbench2/public/API_HOST ; then
+    API_HOST=$(cat /usr/src/workbench2/public/API_HOST)
+fi
+
 cat <<EOF > /usr/src/workbench2/public/config.json
 {
-  "API_HOST": "${localip}:${services[controller-ssl]}",
-  "VOCABULARY_URL": "/vocabulary-example.json",
-  "FILE_VIEWERS_CONFIG_URL": "/file-viewers-example.json"
+  "API_HOST": "$API_HOST"
 }
 EOF
 
diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_controller_configuration.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_controller_configuration.sls
index 41d6e1365a..869cc596a3 100644
--- a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_controller_configuration.sls
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_controller_configuration.sls
@@ -55,6 +55,10 @@ nginx:
               - proxy_set_header: 'X-Real-IP $remote_addr'
               - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
               - proxy_set_header: 'X-External-Client $external_client'
+              - proxy_max_temp_file_size: 0
+              - proxy_request_buffering: 'off'
+              - proxy_buffering: 'off'
+              - proxy_http_version: '1.1'
             - include: snippets/ssl_hardening_default.conf
             - ssl_certificate: __CERT_PEM__
             - ssl_certificate_key: __CERT_KEY__
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls
index 22838fe14a..bc28fd8259 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls
@@ -54,6 +54,10 @@ nginx:
               - proxy_set_header: 'X-Real-IP $remote_addr'
               - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
               - proxy_set_header: 'X-External-Client $external_client'
+              - proxy_max_temp_file_size: 0
+              - proxy_request_buffering: 'off'
+              - proxy_buffering: 'off'
+              - proxy_http_version: '1.1'
             - include: snippets/ssl_hardening_default.conf
             - ssl_certificate: __CERT_PEM__
             - ssl_certificate_key: __CERT_KEY__
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_controller_configuration.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_controller_configuration.sls
index cfd1525924..3058367bc0 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_controller_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_controller_configuration.sls
@@ -54,6 +54,10 @@ nginx:
               - proxy_set_header: 'X-Real-IP $remote_addr'
               - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
               - proxy_set_header: 'X-External-Client $external_client'
+              - proxy_max_temp_file_size: 0
+              - proxy_request_buffering: 'off'
+              - proxy_buffering: 'off'
+              - proxy_http_version: '1.1'
             - include: snippets/ssl_hardening_default.conf
             - ssl_certificate: __CERT_PEM__
             - ssl_certificate_key: __CERT_KEY__