From: Peter Amstutz Date: Thu, 19 May 2022 13:45:37 +0000 (-0400) Subject: Merge branch '17004-properties-on-output' refs #17004 X-Git-Tag: 2.5.0~165 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/c36ec856598f214e340e3335ddd347d131335bf8?hp=f35aae3c6732a7e08253367517c61ab3071086d1 Merge branch '17004-properties-on-output' refs #17004 Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index 26705c0664..3e1ed6a94d 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -248,13 +248,13 @@ package_go_binary cmd/arvados-server arvados-git-httpd "$FORMAT" "$ARCH" \ "Provide authenticated http access to Arvados-hosted git repositories" package_go_binary services/crunch-dispatch-local crunch-dispatch-local "$FORMAT" "$ARCH" \ "Dispatch Crunch containers on the local system" -package_go_binary services/crunch-dispatch-slurm crunch-dispatch-slurm "$FORMAT" "$ARCH" \ +package_go_binary cmd/arvados-server crunch-dispatch-slurm "$FORMAT" "$ARCH" \ "Dispatch Crunch containers to a SLURM cluster" package_go_binary cmd/arvados-server crunch-run "$FORMAT" "$ARCH" \ "Supervise a single Crunch container" package_go_binary services/crunchstat crunchstat "$FORMAT" "$ARCH" \ "Gather cpu/memory/network statistics of running Crunch jobs" -package_go_binary services/health arvados-health "$FORMAT" "$ARCH" \ +package_go_binary cmd/arvados-server arvados-health "$FORMAT" "$ARCH" \ "Check health of all Arvados cluster services" package_go_binary cmd/arvados-server keep-balance "$FORMAT" "$ARCH" \ "Rebalance and garbage-collect data blocks stored in Arvados Keep" diff --git a/services/health/arvados-health.service b/cmd/arvados-server/arvados-health.service similarity index 92% rename from services/health/arvados-health.service rename to cmd/arvados-server/arvados-health.service index 4b8745d154..cf246b0ee2 100644 --- a/services/health/arvados-health.service +++ b/cmd/arvados-server/arvados-health.service @@ -12,7 +12,8 @@ AssertPathExists=/etc/arvados/config.yml StartLimitIntervalSec=0 [Service] -Type=simple +Type=notify +EnvironmentFile=-/etc/arvados/environment ExecStart=/usr/bin/arvados-health # Set a reasonable default for the open file limit LimitNOFILE=65536 diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go index ae1e3fbeee..3a1fcd4c64 100644 --- a/cmd/arvados-server/cmd.go +++ b/cmd/arvados-server/cmd.go @@ -5,6 +5,7 @@ package main import ( + "context" "encoding/json" "fmt" "io" @@ -21,6 +22,8 @@ import ( "git.arvados.org/arvados.git/lib/install" "git.arvados.org/arvados.git/lib/lsf" "git.arvados.org/arvados.git/lib/recovercollection" + "git.arvados.org/arvados.git/lib/service" + "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/health" "git.arvados.org/arvados.git/services/githttpd" keepbalance "git.arvados.org/arvados.git/services/keep-balance" @@ -28,6 +31,7 @@ import ( "git.arvados.org/arvados.git/services/keepproxy" "git.arvados.org/arvados.git/services/keepstore" "git.arvados.org/arvados.git/services/ws" + "github.com/prometheus/client_golang/prometheus" ) var ( @@ -47,6 +51,7 @@ var ( "dispatch-cloud": dispatchcloud.Command, "dispatch-lsf": lsf.DispatchCommand, "git-httpd": githttpd.Command, + "health": healthCommand, "install": install.Command, "init": install.InitCommand, "keep-balance": keepbalance.Command, @@ -90,3 +95,17 @@ func (wb2command) RunCommand(prog string, args []string, stdin io.Reader, stdout } return 0 } + +var healthCommand cmd.Handler = service.Command(arvados.ServiceNameHealth, func(ctx context.Context, cluster *arvados.Cluster, _ string, reg *prometheus.Registry) service.Handler { + mClockSkew := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "arvados", + Subsystem: "health", + Name: "clock_skew_seconds", + Help: "Clock skew observed in most recent health check", + }) + reg.MustRegister(mClockSkew) + return &health.Aggregator{ + Cluster: cluster, + MetricClockSkew: mClockSkew, + } +}) diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service b/cmd/arvados-server/crunch-dispatch-slurm.service similarity index 88% rename from services/crunch-dispatch-slurm/crunch-dispatch-slurm.service rename to cmd/arvados-server/crunch-dispatch-slurm.service index 86830f3a7f..51b4e58c35 100644 --- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service +++ b/cmd/arvados-server/crunch-dispatch-slurm.service @@ -6,18 +6,19 @@ Description=Arvados Crunch Dispatcher for SLURM Documentation=https://doc.arvados.org/ After=network.target +AssertPathExists=/etc/arvados/config.yml # systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section StartLimitIntervalSec=0 [Service] Type=notify +EnvironmentFile=-/etc/arvados/environment ExecStart=/usr/bin/crunch-dispatch-slurm # Set a reasonable default for the open file limit LimitNOFILE=65536 Restart=always RestartSec=1 -LimitNOFILE=1000000 # systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section StartLimitInterval=0 diff --git a/doc/admin/config-urls.html.textile.liquid b/doc/admin/config-urls.html.textile.liquid index 1358fd81e1..e518ea1bf7 100644 --- a/doc/admin/config-urls.html.textile.liquid +++ b/doc/admin/config-urls.html.textile.liquid @@ -255,10 +255,14 @@ server { client_max_body_size 128m; location / { - proxy_pass http://controller; - proxy_redirect off; - proxy_connect_timeout 90s; - proxy_read_timeout 300s; + proxy_pass http://controller; + proxy_redirect off; + proxy_connect_timeout 90s; + proxy_read_timeout 300s; + proxy_max_temp_file_size 0; + proxy_request_buffering off; + proxy_buffering off; + proxy_http_version 1.1; proxy_set_header Host $http_host; proxy_set_header Upgrade $http_upgrade; diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid index 2d10c870d5..3f6009a803 100644 --- a/doc/admin/upgrading.html.textile.liquid +++ b/doc/admin/upgrading.html.textile.liquid @@ -32,6 +32,23 @@ h2(#main). development main (as of 2022-04-08) "previous: Upgrading to 2.4.0":#v2_4_0 +h3. Slurm dispatcher requires configuration update + +If you use the Slurm dispatcher (@crunch-dispatch-slurm@) you must add a @Services.DispatchSLURM.InternalURLs@ section to your configuration file, as shown on the "updated install page":{{site.baseurl}}/install/crunch2-slurm/install-dispatch.html. + +h3. New proxy parameters for arvados-controller + +We now recommend disabling nginx proxy caching for arvados-controller, to avoid truncation of large responses. + +In your Nginx configuration file (@/etc/nginx/conf.d/arvados-api-and-controller.conf@), add the following lines to the @location /@ block with @http://controller@ (see "Update nginx configuration":{{site.baseurl}}/install/install-api-server.html#update-nginx for an example) and reload/restart Nginx (@sudo nginx -s reload@). + +
+    proxy_max_temp_file_size 0;
+    proxy_request_buffering  off;
+    proxy_buffering          off;
+    proxy_http_version       1.1;
+
+ h3. Now recommending Singularity 3.9.9 The compute image "build script":{{site.baseurl}}/install/crunch2-cloud/install-compute-node.html now installs Singularity 3.9.9 instead of 3.7.4. The newer version includes a bugfix that should resolve "intermittent loopback device errors":https://dev.arvados.org/issues/18489 when running containers. diff --git a/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid b/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid index 52553a35e7..9b664ec9ef 100644 --- a/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid +++ b/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid @@ -26,9 +26,18 @@ This assumes you already have a Slurm cluster, and have set up all of your compu The Arvados Slurm dispatcher can run on any node that can submit requests to both the Arvados API server and the Slurm controller (via @sbatch@). It is not resource-intensive, so you can run it on the API server node. -h2(#update-config). Update config.yml (optional) +h2(#update-config). Update config.yml -Crunch-dispatch-slurm reads the common configuration file at @config.yml@. +Crunch-dispatch-slurm reads the common configuration file at @/etc/arvados/config.yml@. + +Add a DispatchSLURM entry to the Services section, using the hostname where @crunch-dispatch-slurm@ will run, and an available port: + + +
    Services:
+      DispatchSLURM:
+        InternalURLs:
+          "http://hostname.zzzzz.arvadosapi.com:9007": {}
+ The following configuration parameters are optional. diff --git a/doc/install/install-api-server.html.textile.liquid b/doc/install/install-api-server.html.textile.liquid index 6c3eabba4f..4c9f168e82 100644 --- a/doc/install/install-api-server.html.textile.liquid +++ b/doc/install/install-api-server.html.textile.liquid @@ -148,10 +148,14 @@ server { client_max_body_size 128m; location / { - proxy_pass http://controller; - proxy_redirect off; - proxy_connect_timeout 90s; - proxy_read_timeout 300s; + proxy_pass http://controller; + proxy_redirect off; + proxy_connect_timeout 90s; + proxy_read_timeout 300s; + proxy_max_temp_file_size 0; + proxy_request_buffering off; + proxy_buffering off; + proxy_http_version 1.1; proxy_set_header Host $http_host; proxy_set_header Upgrade $http_upgrade; diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go index a5d61b4a1c..4b15266006 100644 --- a/lib/boot/supervisor.go +++ b/lib/boot/supervisor.go @@ -365,7 +365,7 @@ func (super *Supervisor) runCluster() error { runNginx{}, runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{seedDatabase{}}}, runServiceCommand{name: "git-httpd", svc: super.cluster.Services.GitHTTP}, - runGoProgram{src: "services/health", svc: super.cluster.Services.Health}, + runServiceCommand{name: "health", svc: super.cluster.Services.Health}, runServiceCommand{name: "keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}}, runServiceCommand{name: "keepstore", svc: super.cluster.Services.Keepstore}, runServiceCommand{name: "keep-web", svc: super.cluster.Services.WebDAV}, diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml index e60880c217..893542df18 100644 --- a/lib/config/config.default.yml +++ b/lib/config/config.default.yml @@ -55,6 +55,9 @@ Clusters: DispatchLSF: InternalURLs: {SAMPLE: {}} ExternalURL: "" + DispatchSLURM: + InternalURLs: {SAMPLE: {}} + ExternalURL: "" Keepproxy: InternalURLs: {SAMPLE: {}} ExternalURL: "" diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go index 474fbf4ade..0253ac3fa8 100644 --- a/lib/crunchrun/crunchrun.go +++ b/lib/crunchrun/crunchrun.go @@ -422,7 +422,7 @@ func (runner *ContainerRunner) SetupMounts() (map[string]bindmount, error) { "--storage-classes", strings.Join(runner.Container.OutputStorageClasses, ","), fmt.Sprintf("--crunchstat-interval=%v", runner.statInterval.Seconds())} - if runner.executor.Runtime() == "docker" { + if _, isdocker := runner.executor.(*dockerExecutor); isdocker { arvMountCmd = append(arvMountCmd, "--allow-other") } @@ -1479,7 +1479,10 @@ func (runner *ContainerRunner) NewArvLogWriter(name string) (io.WriteCloser, err func (runner *ContainerRunner) Run() (err error) { runner.CrunchLog.Printf("crunch-run %s started", cmd.Version.String()) runner.CrunchLog.Printf("%s", currentUserAndGroups()) - runner.CrunchLog.Printf("Executing container '%s' using %s runtime", runner.Container.UUID, runner.executor.Runtime()) + v, _ := exec.Command("arv-mount", "--version").CombinedOutput() + runner.CrunchLog.Printf("Using FUSE mount: %s", v) + runner.CrunchLog.Printf("Using container runtime: %s", runner.executor.Runtime()) + runner.CrunchLog.Printf("Executing container: %s", runner.Container.UUID) hostname, hosterr := os.Hostname() if hosterr != nil { diff --git a/lib/crunchrun/crunchrun_test.go b/lib/crunchrun/crunchrun_test.go index 1d2c7b09fd..347703a95b 100644 --- a/lib/crunchrun/crunchrun_test.go +++ b/lib/crunchrun/crunchrun_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "git.arvados.org/arvados.git/lib/cmd" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/arvadosclient" "git.arvados.org/arvados.git/sdk/go/arvadostest" @@ -128,6 +129,7 @@ func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvad return e.loadErr } func (e *stubExecutor) Runtime() string { return "stub" } +func (e *stubExecutor) Version() string { return "stub " + cmd.Version.String() } func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr } func (e *stubExecutor) Start() error { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr } func (e *stubExecutor) CgroupID() string { return "cgroupid" } @@ -885,7 +887,8 @@ func (s *TestSuite) TestLogVersionAndRuntime(c *C) { c.Assert(s.api.Logs["crunch-run"], NotNil) c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`) c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`) - c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' using stub runtime.*`) + c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container: zzzzz-zzzzz-zzzzzzzzzzzzzzz.*`) + c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Using container runtime: stub.*`) } func (s *TestSuite) TestContainerRecordLog(c *C) { diff --git a/lib/crunchrun/docker.go b/lib/crunchrun/docker.go index e62f2a39ba..f3808cb357 100644 --- a/lib/crunchrun/docker.go +++ b/lib/crunchrun/docker.go @@ -46,7 +46,20 @@ func newDockerExecutor(containerUUID string, logf func(string, ...interface{}), }, err } -func (e *dockerExecutor) Runtime() string { return "docker" } +func (e *dockerExecutor) Runtime() string { + v, _ := e.dockerclient.ServerVersion(context.Background()) + info := "" + for _, cv := range v.Components { + if info != "" { + info += ", " + } + info += cv.Name + " " + cv.Version + } + if info == "" { + info = "(unknown version)" + } + return "docker " + info +} func (e *dockerExecutor) LoadImage(imageID string, imageTarballPath string, container arvados.Container, arvMountPoint string, containerClient *arvados.Client) error { diff --git a/lib/crunchrun/executor.go b/lib/crunchrun/executor.go index dc1bc20b7c..0a65f4d634 100644 --- a/lib/crunchrun/executor.go +++ b/lib/crunchrun/executor.go @@ -60,6 +60,6 @@ type containerExecutor interface { // Release resources (temp dirs, stopped containers) Close() - // Name of runtime engine ("docker", "singularity") + // Name and version of runtime engine ("docker 20.10.16", "singularity-ce version 3.9.9") Runtime() string } diff --git a/lib/crunchrun/executor_test.go b/lib/crunchrun/executor_test.go index 99af0530ff..3301a6e63b 100644 --- a/lib/crunchrun/executor_test.go +++ b/lib/crunchrun/executor_test.go @@ -80,6 +80,7 @@ func (s *executorSuite) TearDownTest(c *C) { } func (s *executorSuite) TestExecTrivialContainer(c *C) { + c.Logf("Using container runtime: %s", s.executor.Runtime()) s.spec.Command = []string{"echo", "ok"} s.checkRun(c, 0) c.Check(s.stdout.String(), Equals, "ok\n") diff --git a/lib/crunchrun/integration_test.go b/lib/crunchrun/integration_test.go index 0b139dd97d..ec08937283 100644 --- a/lib/crunchrun/integration_test.go +++ b/lib/crunchrun/integration_test.go @@ -162,11 +162,13 @@ func (s *integrationSuite) setup(c *C) { func (s *integrationSuite) TestRunTrivialContainerWithDocker(c *C) { s.engine = "docker" s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*Using container runtime: docker Engine \d+\.\d+.*`) } func (s *integrationSuite) TestRunTrivialContainerWithSingularity(c *C) { s.engine = "singularity" s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*Using container runtime: singularity.* version 3\.\d+.*`) } func (s *integrationSuite) TestRunTrivialContainerWithLocalKeepstore(c *C) { diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go index 64a3773250..1af0d420e4 100644 --- a/lib/crunchrun/singularity.go +++ b/lib/crunchrun/singularity.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "sort" + "strings" "syscall" "time" @@ -36,7 +37,13 @@ func newSingularityExecutor(logf func(string, ...interface{})) (*singularityExec }, nil } -func (e *singularityExecutor) Runtime() string { return "singularity" } +func (e *singularityExecutor) Runtime() string { + buf, err := exec.Command("singularity", "--version").CombinedOutput() + if err != nil { + return "singularity (unknown version)" + } + return strings.TrimSuffix(string(buf), "\n") +} func (e *singularityExecutor) getOrCreateProject(ownerUuid string, name string, containerClient *arvados.Client) (*arvados.Group, error) { var gp arvados.GroupList @@ -292,6 +299,14 @@ func (e *singularityExecutor) execCmd(path string) *exec.Cmd { // us to select specific devices we need to propagate that. env = append(env, "SINGULARITYENV_CUDA_VISIBLE_DEVICES="+cudaVisibleDevices) } + // Singularity's default behavior is to evaluate each + // SINGULARITYENV_* env var with a shell as a double-quoted + // string and pass the result to the contained + // process. Singularity 3.10+ has an option to pass env vars + // through literally without evaluating, which is what we + // want. See https://github.com/sylabs/singularity/pull/704 + // and https://dev.arvados.org/issues/19081 + env = append(env, "SINGULARITY_NO_EVAL=1") args = append(args, e.imageFilename) args = append(args, e.spec.Command...) diff --git a/lib/crunchrun/singularity_test.go b/lib/crunchrun/singularity_test.go index cdeafee882..bad2abef33 100644 --- a/lib/crunchrun/singularity_test.go +++ b/lib/crunchrun/singularity_test.go @@ -48,5 +48,5 @@ func (s *singularityStubSuite) TestSingularityExecArgs(c *C) { e.imageFilename = "/fake/image.sif" cmd := e.execCmd("./singularity") c.Check(cmd.Args, DeepEquals, []string{"./singularity", "exec", "--containall", "--cleanenv", "--pwd", "/WorkingDir", "--net", "--network=none", "--nv", "--bind", "/hostpath:/mnt:ro", "/fake/image.sif"}) - c.Check(cmd.Env, DeepEquals, []string{"SINGULARITYENV_FOO=bar"}) + c.Check(cmd.Env, DeepEquals, []string{"SINGULARITYENV_FOO=bar", "SINGULARITY_NO_EVAL=1"}) } diff --git a/lib/install/deps.go b/lib/install/deps.go index cdf28e09c6..ed71709363 100644 --- a/lib/install/deps.go +++ b/lib/install/deps.go @@ -199,6 +199,7 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read } if dev || test { pkgs = append(pkgs, "squashfs-tools") // for singularity + pkgs = append(pkgs, "gnupg") // for docker install recipe } switch { case osv.Debian && osv.Major >= 11: @@ -223,6 +224,37 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read } } + if dev || test { + if havedockerversion, err := exec.Command("docker", "--version").CombinedOutput(); err == nil { + logger.Printf("%s installed, assuming that version is ok", bytes.TrimSuffix(havedockerversion, []byte("\n"))) + } else if osv.Debian { + var codename string + switch osv.Major { + case 10: + codename = "buster" + case 11: + codename = "bullseye" + default: + err = fmt.Errorf("don't know how to install docker-ce for debian %d", osv.Major) + return 1 + } + err = inst.runBash(` +rm -f /usr/share/keyrings/docker-archive-keyring.gpg +curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg +echo 'deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian/ `+codename+` stable' | \ + tee /etc/apt/sources.list.d/docker.list +apt-get update +DEBIAN_FRONTEND=noninteractive apt-get --yes --no-install-recommends install docker-ce +`, stdout, stderr) + if err != nil { + return 1 + } + } else { + err = fmt.Errorf("don't know how to install docker for osversion %v", osv) + return 1 + } + } + os.Mkdir("/var/lib/arvados", 0755) os.Mkdir("/var/lib/arvados/tmp", 0700) if prod || pkg { diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py index f232178c5d..644713bce2 100644 --- a/sdk/cwl/arvados_cwl/runner.py +++ b/sdk/cwl/arvados_cwl/runner.py @@ -249,7 +249,7 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov primary["secondaryFiles"] = cmap(found) if discovered is not None: discovered[primary["location"]] = primary["secondaryFiles"] - elif inputschema["type"] not in primitive_types_set: + elif inputschema["type"] not in primitive_types_set and inputschema["type"] not in ("File", "Directory"): set_secondary(fsaccess, builder, inputschema["type"], secondaryspec, primary, discovered) def discover_secondary_files(fsaccess, builder, inputs, job_order, discovered=None): diff --git a/sdk/cwl/test_with_arvbox.sh b/sdk/cwl/test_with_arvbox.sh index d38414fc81..354d6f0e56 100755 --- a/sdk/cwl/test_with_arvbox.sh +++ b/sdk/cwl/test_with_arvbox.sh @@ -83,6 +83,12 @@ fi arvbox start $config $tag +# Copy the integration test suite from our local arvados clone instead +# of using the one inside the container, so we can make changes to the +# integration tests without necessarily having to rebuilding the +# container image. +docker cp -L $(readlink -f $(dirname $0)/tests) $ARVBOX_CONTAINER:/usr/src/arvados/sdk/cwl + arvbox pipe < /dev/null ; then arv-put --portable-data-hash testdir/* @@ -18,6 +23,9 @@ if ! arv-get 20850f01122e860fb878758ac1320877+71 > /dev/null ; then arv-put --portable-data-hash samples/sample1_S01_R1_001.fastq.gz fi +# Use the python executor associated with the installed OS package, if present. +python=$(((ls /usr/share/python3*/dist/python3-arvados-cwl-runner/bin/python || echo python3) | head -n1) 2>/dev/null) + # Test for #18888 # This is a standalone test because the bug was observed with this # command line and was thought to be due to command line handling. @@ -26,7 +34,7 @@ arvados-cwl-runner 18888-download_def.cwl --scripts scripts/ # Test for #19070 # The most effective way to test this seemed to be to write an # integration test to check for the expected behavior. -python test_copy_deps.py +$python test_copy_deps.py # Test for #17004 # Checks that the final output collection has the expected properties. diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go index 6a90c30ce4..319fa1a38f 100644 --- a/sdk/go/arvados/config.go +++ b/sdk/go/arvados/config.go @@ -348,6 +348,7 @@ type Services struct { Controller Service DispatchCloud Service DispatchLSF Service + DispatchSLURM Service GitHTTP Service GitSSH Service Health Service @@ -605,6 +606,7 @@ const ( ServiceNameController ServiceName = "arvados-controller" ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud" ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf" + ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm" ServiceNameGitHTTP ServiceName = "arvados-git-httpd" ServiceNameHealth ServiceName = "arvados-health" ServiceNameKeepbalance ServiceName = "keep-balance" @@ -624,6 +626,7 @@ func (svcs Services) Map() map[ServiceName]Service { ServiceNameController: svcs.Controller, ServiceNameDispatchCloud: svcs.DispatchCloud, ServiceNameDispatchLSF: svcs.DispatchLSF, + ServiceNameDispatchSLURM: svcs.DispatchSLURM, ServiceNameGitHTTP: svcs.GitHTTP, ServiceNameHealth: svcs.Health, ServiceNameKeepbalance: svcs.Keepbalance, diff --git a/sdk/go/health/aggregator_test.go b/sdk/go/health/aggregator_test.go index 5f60cf67f3..481054c4de 100644 --- a/sdk/go/health/aggregator_test.go +++ b/sdk/go/health/aggregator_test.go @@ -327,6 +327,7 @@ func (s *AggregatorSuite) setAllServiceURLs(listen string) { &svcs.Controller, &svcs.DispatchCloud, &svcs.DispatchLSF, + &svcs.DispatchSLURM, &svcs.GitHTTP, &svcs.Keepbalance, &svcs.Keepproxy, diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf index 4d1e06613a..543390004b 100644 --- a/sdk/python/tests/nginx.conf +++ b/sdk/python/tests/nginx.conf @@ -30,6 +30,10 @@ http { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto https; proxy_redirect off; + proxy_max_temp_file_size 0; + proxy_request_buffering off; + proxy_buffering off; + proxy_http_version 1.1; } } upstream arv-git-http { diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go index 84105e1fc7..c31d799752 100644 --- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go +++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go @@ -2,32 +2,48 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main - // Dispatcher service for Crunch that submits containers to the slurm queue. +package dispatchslurm import ( "context" - "flag" "fmt" "log" "math" + "net/http" "os" "regexp" "strings" "time" "git.arvados.org/arvados.git/lib/cmd" - "git.arvados.org/arvados.git/lib/config" "git.arvados.org/arvados.git/lib/dispatchcloud" + "git.arvados.org/arvados.git/lib/service" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/ctxlog" "git.arvados.org/arvados.git/sdk/go/dispatch" "github.com/coreos/go-systemd/daemon" - "github.com/ghodss/yaml" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" ) +var Command cmd.Handler = service.Command(arvados.ServiceNameDispatchSLURM, newHandler) + +func newHandler(ctx context.Context, cluster *arvados.Cluster, _ string, _ *prometheus.Registry) service.Handler { + logger := ctxlog.FromContext(ctx) + disp := &Dispatcher{logger: logger, cluster: cluster} + if err := disp.configure(); err != nil { + return service.ErrorHandler(ctx, cluster, err) + } + disp.setup() + go func() { + disp.err = disp.run() + close(disp.done) + }() + return disp +} + type logger interface { dispatch.Logger Fatalf(string, ...interface{}) @@ -35,10 +51,6 @@ type logger interface { const initialNiceValue int64 = 10000 -var ( - version = "dev" -) - type Dispatcher struct { *dispatch.Dispatcher logger logrus.FieldLogger @@ -46,75 +58,32 @@ type Dispatcher struct { sqCheck *SqueueChecker slurm Slurm + done chan struct{} + err error + Client arvados.Client } -func main() { - logger := logrus.StandardLogger() - if os.Getenv("DEBUG") != "" { - logger.SetLevel(logrus.DebugLevel) - } - logger.Formatter = &logrus.JSONFormatter{ - TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00", - } - disp := &Dispatcher{logger: logger} - err := disp.Run(os.Args[0], os.Args[1:]) - if err != nil { - logrus.Fatalf("%s", err) - } +func (disp *Dispatcher) CheckHealth() error { + return disp.err } -func (disp *Dispatcher) Run(prog string, args []string) error { - if err := disp.configure(prog, args); err != nil { - return err - } - disp.setup() - return disp.run() +func (disp *Dispatcher) Done() <-chan struct{} { + return disp.done +} + +func (disp *Dispatcher) ServeHTTP(w http.ResponseWriter, r *http.Request) { + http.NotFound(w, r) } -// configure() loads config files. Tests skip this. -func (disp *Dispatcher) configure(prog string, args []string) error { +// configure() loads config files. Some tests skip this (see +// StubbedSuite). +func (disp *Dispatcher) configure() error { if disp.logger == nil { disp.logger = logrus.StandardLogger() } - flags := flag.NewFlagSet(prog, flag.ContinueOnError) - flags.Usage = func() { usage(flags) } - - loader := config.NewLoader(nil, disp.logger) - loader.SetupFlags(flags) - - dumpConfig := flag.Bool( - "dump-config", - false, - "write current configuration to stdout and exit") - getVersion := flags.Bool( - "version", - false, - "Print version information and exit.") - - args = loader.MungeLegacyConfigArgs(disp.logger, args, "-legacy-crunch-dispatch-slurm-config") - if ok, code := cmd.ParseFlags(flags, prog, args, "", os.Stderr); !ok { - os.Exit(code) - } - - // Print version information if requested - if *getVersion { - fmt.Printf("crunch-dispatch-slurm %s\n", version) - return nil - } - - disp.logger.Printf("crunch-dispatch-slurm %s started", version) - - cfg, err := loader.Load() - if err != nil { - return err - } - - if disp.cluster, err = cfg.GetCluster(""); err != nil { - return fmt.Errorf("config error: %s", err) - } - disp.logger = disp.logger.WithField("ClusterID", disp.cluster.ClusterID) + disp.logger.Printf("crunch-dispatch-slurm %s started", cmd.Version.String()) disp.Client.APIHost = disp.cluster.Services.Controller.ExternalURL.Host disp.Client.AuthToken = disp.cluster.SystemRootToken @@ -137,23 +106,12 @@ func (disp *Dispatcher) configure(prog string, args []string) error { } else { disp.logger.Warnf("Client credentials missing from config, so falling back on environment variables (deprecated).") } - - if *dumpConfig { - out, err := yaml.Marshal(cfg) - if err != nil { - return err - } - _, err = os.Stdout.Write(out) - if err != nil { - return err - } - } - return nil } // setup() initializes private fields after configure(). func (disp *Dispatcher) setup() { + disp.done = make(chan struct{}) arv, err := arvadosclient.MakeArvadosClient() if err != nil { disp.logger.Fatalf("Error making Arvados client: %v", err) diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go index cf83257dad..fb433e65cd 100644 --- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go +++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go @@ -2,12 +2,13 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package dispatchslurm import ( "bytes" "context" "errors" + "flag" "fmt" "io" "io/ioutil" @@ -19,10 +20,13 @@ import ( "testing" "time" + "git.arvados.org/arvados.git/lib/cmd" + "git.arvados.org/arvados.git/lib/config" "git.arvados.org/arvados.git/lib/dispatchcloud" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/arvadosclient" "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/ctxlog" "git.arvados.org/arvados.git/sdk/go/dispatch" "github.com/sirupsen/logrus" . "gopkg.in/check.v1" @@ -387,6 +391,7 @@ func (s *StubbedSuite) TestSbatchPartition(c *C) { } func (s *StubbedSuite) TestLoadLegacyConfig(c *C) { + log := ctxlog.TestLogger(c) content := []byte(` Client: APIHost: example.com @@ -402,36 +407,42 @@ ReserveExtraRAM: 12345 MinRetryPeriod: 13s BatchSize: 99 `) - tmpfile, err := ioutil.TempFile("", "example") - if err != nil { - c.Error(err) - } - - defer os.Remove(tmpfile.Name()) // clean up - - if _, err := tmpfile.Write(content); err != nil { - c.Error(err) - } - if err := tmpfile.Close(); err != nil { - c.Error(err) + tmpfile := c.MkDir() + "/config.yml" + err := ioutil.WriteFile(tmpfile, content, 0777) + c.Assert(err, IsNil) - } os.Setenv("ARVADOS_KEEP_SERVICES", "") - err = s.disp.configure("crunch-dispatch-slurm", []string{"-config", tmpfile.Name()}) - c.Check(err, IsNil) - c.Check(s.disp.cluster.Services.Controller.ExternalURL, Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"}) - c.Check(s.disp.cluster.SystemRootToken, Equals, "abcdefg") - c.Check(s.disp.cluster.Containers.SLURM.SbatchArgumentsList, DeepEquals, []string{"--foo", "bar"}) - c.Check(s.disp.cluster.Containers.CloudVMs.PollInterval, Equals, arvados.Duration(12*time.Second)) - c.Check(s.disp.cluster.Containers.SLURM.PrioritySpread, Equals, int64(42)) - c.Check(s.disp.cluster.Containers.CrunchRunCommand, Equals, "x-crunch-run") - c.Check(s.disp.cluster.Containers.CrunchRunArgumentsList, DeepEquals, []string{"--cgroup-parent-subsystem=memory"}) - c.Check(s.disp.cluster.Containers.ReserveExtraRAM, Equals, arvados.ByteSize(12345)) - c.Check(s.disp.cluster.Containers.MinRetryPeriod, Equals, arvados.Duration(13*time.Second)) - c.Check(s.disp.cluster.API.MaxItemsPerResponse, Equals, 99) - c.Check(s.disp.cluster.Containers.SLURM.SbatchEnvironmentVariables, DeepEquals, map[string]string{ + flags := flag.NewFlagSet("", flag.ContinueOnError) + flags.SetOutput(os.Stderr) + loader := config.NewLoader(&bytes.Buffer{}, log) + loader.SetupFlags(flags) + args := loader.MungeLegacyConfigArgs(log, []string{"-config", tmpfile}, "-legacy-"+string(arvados.ServiceNameDispatchSLURM)+"-config") + ok, _ := cmd.ParseFlags(flags, "crunch-dispatch-slurm", args, "", os.Stderr) + c.Check(ok, Equals, true) + cfg, err := loader.Load() + c.Assert(err, IsNil) + cluster, err := cfg.GetCluster("") + c.Assert(err, IsNil) + + c.Check(cluster.Services.Controller.ExternalURL, Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"}) + c.Check(cluster.SystemRootToken, Equals, "abcdefg") + c.Check(cluster.Containers.SLURM.SbatchArgumentsList, DeepEquals, []string{"--foo", "bar"}) + c.Check(cluster.Containers.CloudVMs.PollInterval, Equals, arvados.Duration(12*time.Second)) + c.Check(cluster.Containers.SLURM.PrioritySpread, Equals, int64(42)) + c.Check(cluster.Containers.CrunchRunCommand, Equals, "x-crunch-run") + c.Check(cluster.Containers.CrunchRunArgumentsList, DeepEquals, []string{"--cgroup-parent-subsystem=memory"}) + c.Check(cluster.Containers.ReserveExtraRAM, Equals, arvados.ByteSize(12345)) + c.Check(cluster.Containers.MinRetryPeriod, Equals, arvados.Duration(13*time.Second)) + c.Check(cluster.API.MaxItemsPerResponse, Equals, 99) + c.Check(cluster.Containers.SLURM.SbatchEnvironmentVariables, DeepEquals, map[string]string{ "ARVADOS_KEEP_SERVICES": "https://example.com/keep1 https://example.com/keep2", }) + + // Ensure configure() copies SbatchEnvironmentVariables into + // the current process's environment (that's how they end up + // getting passed to sbatch). + s.disp.cluster = cluster + s.disp.configure() c.Check(os.Getenv("ARVADOS_KEEP_SERVICES"), Equals, "https://example.com/keep1 https://example.com/keep2") } diff --git a/services/crunch-dispatch-slurm/node_type.go b/services/crunch-dispatch-slurm/node_type.go index d31322f182..738426c92d 100644 --- a/services/crunch-dispatch-slurm/node_type.go +++ b/services/crunch-dispatch-slurm/node_type.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package dispatchslurm import ( "log" diff --git a/services/crunch-dispatch-slurm/priority.go b/services/crunch-dispatch-slurm/priority.go index 2312ce5952..515a98d323 100644 --- a/services/crunch-dispatch-slurm/priority.go +++ b/services/crunch-dispatch-slurm/priority.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package dispatchslurm const defaultSpread int64 = 10 diff --git a/services/crunch-dispatch-slurm/priority_test.go b/services/crunch-dispatch-slurm/priority_test.go index e80984c0fc..df1c27def7 100644 --- a/services/crunch-dispatch-slurm/priority_test.go +++ b/services/crunch-dispatch-slurm/priority_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package dispatchslurm import ( . "gopkg.in/check.v1" diff --git a/services/crunch-dispatch-slurm/script.go b/services/crunch-dispatch-slurm/script.go index f559104d14..fb16e593e5 100644 --- a/services/crunch-dispatch-slurm/script.go +++ b/services/crunch-dispatch-slurm/script.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package dispatchslurm import ( "strings" diff --git a/services/crunch-dispatch-slurm/script_test.go b/services/crunch-dispatch-slurm/script_test.go index a21aeeddad..00d70190dd 100644 --- a/services/crunch-dispatch-slurm/script_test.go +++ b/services/crunch-dispatch-slurm/script_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package dispatchslurm import ( . "gopkg.in/check.v1" diff --git a/services/crunch-dispatch-slurm/slurm.go b/services/crunch-dispatch-slurm/slurm.go index 791f294df1..e59826f763 100644 --- a/services/crunch-dispatch-slurm/slurm.go +++ b/services/crunch-dispatch-slurm/slurm.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package dispatchslurm import ( "fmt" diff --git a/services/crunch-dispatch-slurm/squeue.go b/services/crunch-dispatch-slurm/squeue.go index eae21e62b6..d4e41ed1fb 100644 --- a/services/crunch-dispatch-slurm/squeue.go +++ b/services/crunch-dispatch-slurm/squeue.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package dispatchslurm import ( "bytes" diff --git a/services/crunch-dispatch-slurm/squeue_test.go b/services/crunch-dispatch-slurm/squeue_test.go index ce74fe61cc..d41e1982b4 100644 --- a/services/crunch-dispatch-slurm/squeue_test.go +++ b/services/crunch-dispatch-slurm/squeue_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package dispatchslurm import ( "time" diff --git a/services/crunch-dispatch-slurm/usage.go b/services/crunch-dispatch-slurm/usage.go index 68a2305f74..785843b198 100644 --- a/services/crunch-dispatch-slurm/usage.go +++ b/services/crunch-dispatch-slurm/usage.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package dispatchslurm import ( "flag" diff --git a/services/health/main.go b/services/health/main.go deleted file mode 100644 index 92bd377c80..0000000000 --- a/services/health/main.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (C) The Arvados Authors. All rights reserved. -// -// SPDX-License-Identifier: AGPL-3.0 - -package main - -import ( - "context" - "os" - - "git.arvados.org/arvados.git/lib/cmd" - "git.arvados.org/arvados.git/lib/service" - "git.arvados.org/arvados.git/sdk/go/arvados" - "git.arvados.org/arvados.git/sdk/go/health" - "github.com/prometheus/client_golang/prometheus" -) - -var ( - version = "dev" - command cmd.Handler = service.Command(arvados.ServiceNameHealth, newHandler) -) - -func newHandler(ctx context.Context, cluster *arvados.Cluster, _ string, reg *prometheus.Registry) service.Handler { - mClockSkew := prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: "arvados", - Subsystem: "health", - Name: "clock_skew_seconds", - Help: "Clock skew observed in most recent health check", - }) - reg.MustRegister(mClockSkew) - return &health.Aggregator{ - Cluster: cluster, - MetricClockSkew: mClockSkew, - } -} - -func main() { - os.Exit(command.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr)) -} diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.base b/tools/arvbox/lib/arvbox/docker/Dockerfile.base index b984aeb7f5..8f20850ef4 100644 --- a/tools/arvbox/lib/arvbox/docker/Dockerfile.base +++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.base @@ -108,11 +108,12 @@ ADD $workdir/8D81803C0EBFCD88.asc /tmp/ RUN apt-key add --no-tty /tmp/8D81803C0EBFCD88.asc && \ rm -f /tmp/8D81803C0EBFCD88.asc -RUN mkdir -p /etc/apt/sources.list.d && \ - echo deb https://download.docker.com/linux/debian/ buster stable > /etc/apt/sources.list.d/docker.list && \ - apt-get update && \ - apt-get -yq --no-install-recommends install docker-ce=5:20.10.6~3-0~debian-buster && \ - apt-get clean +# docker is now installed by arvados-server install +# RUN mkdir -p /etc/apt/sources.list.d && \ +# echo deb https://download.docker.com/linux/debian/ buster stable > /etc/apt/sources.list.d/docker.list && \ +# apt-get update && \ +# apt-get -yq --no-install-recommends install docker-ce=5:20.10.6~3-0~debian-buster && \ +# apt-get clean # Set UTF-8 locale RUN echo en_US.UTF-8 UTF-8 > /etc/locale.gen && locale-gen diff --git a/tools/arvbox/lib/arvbox/docker/service/workbench2/run-service b/tools/arvbox/lib/arvbox/docker/service/workbench2/run-service index 2b68cadafd..5268c7e17e 100755 --- a/tools/arvbox/lib/arvbox/docker/service/workbench2/run-service +++ b/tools/arvbox/lib/arvbox/docker/service/workbench2/run-service @@ -22,11 +22,15 @@ if test "$1" = "--only-deps" ; then exit fi +API_HOST=${localip}:${services[controller-ssl]} + +if test -f /usr/src/workbench2/public/API_HOST ; then + API_HOST=$(cat /usr/src/workbench2/public/API_HOST) +fi + cat < /usr/src/workbench2/public/config.json { - "API_HOST": "${localip}:${services[controller-ssl]}", - "VOCABULARY_URL": "/vocabulary-example.json", - "FILE_VIEWERS_CONFIG_URL": "/file-viewers-example.json" + "API_HOST": "$API_HOST" } EOF diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_controller_configuration.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_controller_configuration.sls index 41d6e1365a..869cc596a3 100644 --- a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_controller_configuration.sls +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_controller_configuration.sls @@ -55,6 +55,10 @@ nginx: - proxy_set_header: 'X-Real-IP $remote_addr' - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' - proxy_set_header: 'X-External-Client $external_client' + - proxy_max_temp_file_size: 0 + - proxy_request_buffering: 'off' + - proxy_buffering: 'off' + - proxy_http_version: '1.1' - include: snippets/ssl_hardening_default.conf - ssl_certificate: __CERT_PEM__ - ssl_certificate_key: __CERT_KEY__ diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls index 22838fe14a..bc28fd8259 100644 --- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls +++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls @@ -54,6 +54,10 @@ nginx: - proxy_set_header: 'X-Real-IP $remote_addr' - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' - proxy_set_header: 'X-External-Client $external_client' + - proxy_max_temp_file_size: 0 + - proxy_request_buffering: 'off' + - proxy_buffering: 'off' + - proxy_http_version: '1.1' - include: snippets/ssl_hardening_default.conf - ssl_certificate: __CERT_PEM__ - ssl_certificate_key: __CERT_KEY__ diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_controller_configuration.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_controller_configuration.sls index cfd1525924..3058367bc0 100644 --- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_controller_configuration.sls +++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_controller_configuration.sls @@ -54,6 +54,10 @@ nginx: - proxy_set_header: 'X-Real-IP $remote_addr' - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' - proxy_set_header: 'X-External-Client $external_client' + - proxy_max_temp_file_size: 0 + - proxy_request_buffering: 'off' + - proxy_buffering: 'off' + - proxy_http_version: '1.1' - include: snippets/ssl_hardening_default.conf - ssl_certificate: __CERT_PEM__ - ssl_certificate_key: __CERT_KEY__