Merge branch '18947-keep-balance'

refs #18947

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index adcbab8a09b9d50e2083ca05383c2312a82ac52a..26705c06646b732fdd9258c383e530a352ad8568 100755 (executable)
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -256,7 +256,7 @@ package_go_binary services/crunchstat crunchstat "$FORMAT" "$ARCH" \
     "Gather cpu/memory/network statistics of running Crunch jobs"
 package_go_binary services/health arvados-health "$FORMAT" "$ARCH" \
     "Check health of all Arvados cluster services"
-package_go_binary services/keep-balance keep-balance "$FORMAT" "$ARCH" \
+package_go_binary cmd/arvados-server keep-balance "$FORMAT" "$ARCH" \
     "Rebalance and garbage-collect data blocks stored in Arvados Keep"
 package_go_binary cmd/arvados-server keepproxy "$FORMAT" "$ARCH" \
     "Make a Keep cluster accessible to clients that are not on the LAN"

diff --git a/build/run-tests.sh b/build/run-tests.sh
index d517e7e2c4d921bbc91d8edd31f1ecfcee9bb881..0f996f77e927b4c4d6e0eae6458c4d80f3229be7 100755 (executable)
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -158,6 +158,8 @@ only_install=
 temp=
 temp_preserve=
 
+ignore_sigint=
+
 clear_temp() {
     if [[ -z "$temp" ]]; then
         # we did not even get as far as making a temp dir
@@ -473,6 +475,10 @@ stop_services() {
 }
 
 interrupt() {
+    if [[ -n "$ignore_sigint" ]]; then
+        echo >&2 "ignored SIGINT"
+        return
+    fi
     failures+=("($(basename $0) interrupted)")
     exit_cleanly
 }
@@ -1216,6 +1222,7 @@ else
     setnextcmd
     HISTFILE="$WORKSPACE/tmp/.history"
     history -r
+    ignore_sigint=1
     while read -p 'What next? ' -e -i "$nextcmd" nextcmd; do
         history -s "$nextcmd"
         history -w

diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go
index e4bd39002aafaab29a5ed690fc09a3174f30effc..c9a91c872160cb74c10d098c4ae2f0f9c16ce667 100644 (file)
--- a/cmd/arvados-server/cmd.go
+++ b/cmd/arvados-server/cmd.go
@@ -22,6 +22,7 @@ import (
        "git.arvados.org/arvados.git/lib/lsf"
        "git.arvados.org/arvados.git/lib/recovercollection"
        "git.arvados.org/arvados.git/services/githttpd"
+       keepbalance "git.arvados.org/arvados.git/services/keep-balance"
        keepweb "git.arvados.org/arvados.git/services/keep-web"
        "git.arvados.org/arvados.git/services/keepproxy"
        "git.arvados.org/arvados.git/services/keepstore"
@@ -46,6 +47,7 @@ var (
                "git-httpd":          githttpd.Command,
                "install":            install.Command,
                "init":               install.InitCommand,
+               "keep-balance":       keepbalance.Command,
                "keep-web":           keepweb.Command,
                "keepproxy":          keepproxy.Command,
                "keepstore":          keepstore.Command,

diff --git a/services/keep-balance/keep-balance.service b/cmd/arvados-server/keep-balance.service
similarity index 87%
rename from services/keep-balance/keep-balance.service
rename to cmd/arvados-server/keep-balance.service
index 859d70724106f815ab9a990e0a9615c09a4a5189..1c5808288b38a4e0a1b5b4706cfa446f6a224841 100644 (file)
--- a/services/keep-balance/keep-balance.service
+++ b/cmd/arvados-server/keep-balance.service
@@ -6,12 +6,14 @@
 Description=Arvados Keep Balance
 Documentation=https://doc.arvados.org/
 After=network.target
+AssertPathExists=/etc/arvados/config.yml
 
 # systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
-Type=simple
+Type=notify
+EnvironmentFile=-/etc/arvados/environment
 ExecStart=/usr/bin/keep-balance -commit-pulls -commit-trash
 # Set a reasonable default for the open file limit
 LimitNOFILE=65536

diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 143529487b315391aa07901427deb7cd19085491..a5d61b4a1c25e75cae8aa0b0b035dab0ba7e08e1 100644 (file)
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -388,7 +388,7 @@ func (super *Supervisor) runCluster() error {
        if super.ClusterType != "test" {
                tasks = append(tasks,
                        runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.DispatchCloud},
-                       runGoProgram{src: "services/keep-balance", svc: super.cluster.Services.Keepbalance},
+                       runServiceCommand{name: "keep-balance", svc: super.cluster.Services.Keepbalance},
                )
        }
        super.tasksReady = map[string]chan bool{}

diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 74722b256e4e7c1dadb0148d3b507bcbc3b0d08f..6f591b74a5a20f51b1d87e798c07eebf83606f3f 100644 (file)
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -957,6 +957,9 @@ if __name__ == "__main__":
               format(args.action, actions),
               file=sys.stderr)
         sys.exit(1)
+    # Create a new process group so our child processes don't exit on
+    # ^C in run-tests.sh interactive mode.
+    os.setpgid(0, 0)
     if args.action == 'start':
         stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
         run(leave_running_atexit=True)

diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index eb6f580f435c56072ad381f984703e229c524f29..1dedb409a4a2de5c4f414959b024e291007d42b1 100644 (file)
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package keepbalance
 
 import (
        "bytes"

diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go
index 0d1b6b5912d2820be5a40f1000871b02c1321e0e..2db7bea173c17dc41f6943b4fe579cbc7d15a24f 100644 (file)
--- a/services/keep-balance/balance_run_test.go
+++ b/services/keep-balance/balance_run_test.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package keepbalance
 
 import (
        "bytes"

diff --git a/services/keep-balance/balance_test.go b/services/keep-balance/balance_test.go
index df04145b9dd15e3d1066b0a1d5a71c57ba27a5f8..6626609b5769f55bdb7d32385afffc443df8712c 100644 (file)
--- a/services/keep-balance/balance_test.go
+++ b/services/keep-balance/balance_test.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package keepbalance
 
 import (
        "crypto/md5"

diff --git a/services/keep-balance/block_state.go b/services/keep-balance/block_state.go
index e30b4ff7943d4c4a041ec71924000a48a856c4d4..07c9952f906d7e57dfca21d0007c01d96f42294b 100644 (file)
--- a/services/keep-balance/block_state.go
+++ b/services/keep-balance/block_state.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package keepbalance
 
 import (
        "sync"

diff --git a/services/keep-balance/block_state_test.go b/services/keep-balance/block_state_test.go
index aaf2c18e297c206b2aa8f3de54df6878b6c7235b..8a58be288ff1832a3799224510d04d5d581d2455 100644 (file)
--- a/services/keep-balance/block_state_test.go
+++ b/services/keep-balance/block_state_test.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package keepbalance
 
 import (
        "time"

diff --git a/services/keep-balance/change_set.go b/services/keep-balance/change_set.go
index 85d03d409b681626bdee480f7b7c2260047385c2..8e0ba028acd801e182a9b475f47b658c86e250e1 100644 (file)
--- a/services/keep-balance/change_set.go
+++ b/services/keep-balance/change_set.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package keepbalance
 
 import (
        "encoding/json"

diff --git a/services/keep-balance/change_set_test.go b/services/keep-balance/change_set_test.go
index baf7ab6dc45665b6fc4e7690da2085a6c7452a9c..5474d29fb57e2d64a67286382b1d53907afe3ae7 100644 (file)
--- a/services/keep-balance/change_set_test.go
+++ b/services/keep-balance/change_set_test.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package keepbalance
 
 import (
        "encoding/json"

diff --git a/services/keep-balance/collection.go b/services/keep-balance/collection.go
index 1e1e51abe7ebcd55e5d0ed7ca55eae02a8b56504..ccb01bdd10c5fb7f777d868a2aeefa175d494ace 100644 (file)
--- a/services/keep-balance/collection.go
+++ b/services/keep-balance/collection.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package keepbalance
 
 import (
        "context"

diff --git a/services/keep-balance/collection_test.go b/services/keep-balance/collection_test.go
index f749bad6ad1865a30670d0fe2978dfe8ebd2764c..ca67d1617d6fb110a6c90646f46ca8977c5569d8 100644 (file)
--- a/services/keep-balance/collection_test.go
+++ b/services/keep-balance/collection_test.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package keepbalance
 
 import (
        "context"

diff --git a/services/keep-balance/integration_test.go b/services/keep-balance/integration_test.go
index a6cc3281042739191921dcc8cda04d3a4e938d59..3cfb5cdeda5039fb37f414f5cd0b095eea0e772d 100644 (file)
--- a/services/keep-balance/integration_test.go
+++ b/services/keep-balance/integration_test.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package keepbalance
 
 import (
        "bytes"

diff --git a/services/keep-balance/keep_service.go b/services/keep-balance/keep_service.go
index 17f8418f622f992a7025db9b9214e60c5a39f2ca..f50fbfd946775e86fa667de2a89eddb1e728acc4 100644 (file)
--- a/services/keep-balance/keep_service.go
+++ b/services/keep-balance/keep_service.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package keepbalance
 
 import (
        "context"

diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go
index 8a95d389c8292f04c8479f0c21abdcf5c84932a3..f0b0df5bd331d6a97a2cdaab0a8d968cfdbfc550 100644 (file)
--- a/services/keep-balance/main.go
+++ b/services/keep-balance/main.go
@@ -2,16 +2,15 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package keepbalance
 
 import (
+       "bytes"
        "context"
        "flag"
        "fmt"
        "io"
-       "net/http"
        _ "net/http/pprof"
-       "os"
 
        "git.arvados.org/arvados.git/lib/cmd"
        "git.arvados.org/arvados.git/lib/config"
@@ -22,16 +21,13 @@ import (
        "github.com/jmoiron/sqlx"
        _ "github.com/lib/pq"
        "github.com/prometheus/client_golang/prometheus"
-       "github.com/sirupsen/logrus"
 )
 
-func main() {
-       os.Exit(runCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
-}
+type command struct{}
 
-func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
-       logger := ctxlog.FromContext(context.Background())
+var Command = command{}
 
+func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
        var options RunOptions
        flags := flag.NewFlagSet(prog, flag.ContinueOnError)
        flags.BoolVar(&options.Once, "once", false,
@@ -42,34 +38,21 @@ func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.W
                "send trash requests (delete unreferenced old blocks, and excess replicas of overreplicated blocks)")
        flags.BoolVar(&options.CommitConfirmedFields, "commit-confirmed-fields", true,
                "update collection fields (replicas_confirmed, storage_classes_confirmed, etc.)")
-       dumpFlag := flags.Bool("dump", false, "dump details for each block to stdout")
-       pprofAddr := flags.String("pprof", "", "serve Go profile data at `[addr]:port`")
-       // "show version" is implemented by service.Command, so we
-       // don't need the var here -- we just need the -version flag
+       // These options are implemented by service.Command, so we
+       // don't need the vars here -- we just need the flags
        // to pass flags.Parse().
+       flags.Bool("dump", false, "dump details for each block to stdout")
+       flags.String("pprof", "", "serve Go profile data at `[addr]:port`")
        flags.Bool("version", false, "Write version information to stdout and exit 0")
 
-       if *pprofAddr != "" {
-               go func() {
-                       logrus.Println(http.ListenAndServe(*pprofAddr, nil))
-               }()
-       }
-
-       loader := config.NewLoader(os.Stdin, logger)
+       logger := ctxlog.New(stderr, "json", "info")
+       loader := config.NewLoader(&bytes.Buffer{}, logger)
        loader.SetupFlags(flags)
-
        munged := loader.MungeLegacyConfigArgs(logger, args, "-legacy-keepbalance-config")
        if ok, code := cmd.ParseFlags(flags, prog, munged, "", stderr); !ok {
                return code
        }
 
-       if *dumpFlag {
-               dumper := logrus.New()
-               dumper.Out = os.Stdout
-               dumper.Formatter = &logrus.TextFormatter{}
-               options.Dumper = dumper
-       }
-
        // Drop our custom args that would be rejected by the generic
        // service.Command
        args = nil

diff --git a/services/keep-balance/main_test.go b/services/keep-balance/main_test.go
index 820f35216637626d7e543fb5781b613294f2dd91..26002eca986c3d2c60c7c20e035e7b32b474c22c 100644 (file)
--- a/services/keep-balance/main_test.go
+++ b/services/keep-balance/main_test.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package keepbalance
 
 import (
        "bytes"
@@ -25,7 +25,7 @@ type mainSuite struct{}
 
 func (s *mainSuite) TestVersionFlag(c *check.C) {
        var stdout, stderr bytes.Buffer
-       runCommand("keep-balance", []string{"-version"}, nil, &stdout, &stderr)
+       Command.RunCommand("keep-balance", []string{"-version"}, nil, &stdout, &stderr)
        c.Check(stderr.String(), check.Equals, "")
        c.Log(stdout.String())
        c.Check(stdout.String(), check.Matches, `keep-balance.*\(go1.*\)\n`)
@@ -51,7 +51,7 @@ func (s *mainSuite) TestHTTPServer(c *check.C) {
        c.Assert(err, check.IsNil)
 
        var stdout bytes.Buffer
-       go runCommand("keep-balance", []string{"-config", "-"}, bytes.NewBuffer(config), &stdout, &stdout)
+       go Command.RunCommand("keep-balance", []string{"-config", "-"}, bytes.NewBuffer(config), &stdout, &stdout)
        done := make(chan struct{})
        go func() {
                defer close(done)

diff --git a/services/keep-balance/metrics.go b/services/keep-balance/metrics.go
index ce1b1811cc69f28f3fad955a5525b35a666baf3a..4683b67b9860052d97d8fa77e92141ae29bdcef1 100644 (file)
--- a/services/keep-balance/metrics.go
+++ b/services/keep-balance/metrics.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package keepbalance
 
 import (
        "fmt"

diff --git a/services/keep-balance/server.go b/services/keep-balance/server.go
index 5299b96c1caf2ac3aaa28c639e71d501ddbbd637..e485f5b2061f28134306d1d897b22cb62e4190e9 100644 (file)
--- a/services/keep-balance/server.go
+++ b/services/keep-balance/server.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package keepbalance
 
 import (
        "net/http"