From a05465b46a2b91879596d30b5e8051c937dfa8e9 Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Thu, 30 Sep 2021 08:54:31 -0400 Subject: [PATCH] 18216: Move keepstore into arvados-server binary. Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- build/run-build-packages.sh | 2 +- build/run-tests.sh | 2 -- cmd/arvados-server/cmd.go | 2 ++ .../keepstore => cmd/arvados-server}/keepstore.service | 1 + lib/boot/supervisor.go | 2 +- sdk/python/tests/run_test_server.py | 2 +- services/keepstore/azure_blob_volume.go | 2 +- services/keepstore/azure_blob_volume_test.go | 2 +- services/keepstore/bufferpool.go | 2 +- services/keepstore/bufferpool_test.go | 2 +- services/keepstore/collision.go | 2 +- services/keepstore/collision_test.go | 2 +- services/keepstore/command.go | 6 +----- services/keepstore/command_test.go | 2 +- services/keepstore/count.go | 2 +- services/keepstore/gocheck_test.go | 2 +- services/keepstore/handler_test.go | 2 +- services/keepstore/handlers.go | 2 +- services/keepstore/keepstore.go | 2 +- services/keepstore/metrics.go | 2 +- services/keepstore/mock_mutex_for_test.go | 2 +- services/keepstore/mounts_test.go | 2 +- services/keepstore/perms.go | 2 +- services/keepstore/perms_test.go | 2 +- services/keepstore/pipe_adapters.go | 2 +- services/keepstore/proxy_remote.go | 2 +- services/keepstore/proxy_remote_test.go | 2 +- services/keepstore/pull_worker.go | 2 +- services/keepstore/pull_worker_integration_test.go | 2 +- services/keepstore/pull_worker_test.go | 2 +- services/keepstore/s3_volume.go | 2 +- services/keepstore/s3_volume_test.go | 2 +- services/keepstore/s3aws_volume.go | 2 +- services/keepstore/s3aws_volume_test.go | 2 +- services/keepstore/stats_ticker.go | 2 +- services/keepstore/status_test.go | 2 +- services/keepstore/trash_worker.go | 2 +- services/keepstore/trash_worker_test.go | 2 +- services/keepstore/unix_volume.go | 2 +- services/keepstore/unix_volume_test.go | 2 +- services/keepstore/volume.go | 2 +- services/keepstore/volume_generic_test.go | 2 +- services/keepstore/volume_test.go | 2 +- services/keepstore/work_queue.go | 2 +- services/keepstore/work_queue_test.go | 2 +- 45 files changed, 45 insertions(+), 48 deletions(-) rename {services/keepstore => cmd/arvados-server}/keepstore.service (95%) diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index 7829c8c6cd..26ed168aa6 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -295,7 +295,7 @@ package_go_binary services/keep-balance keep-balance \ "Rebalance and garbage-collect data blocks stored in Arvados Keep" package_go_binary services/keepproxy keepproxy \ "Make a Keep cluster accessible to clients that are not on the LAN" -package_go_binary services/keepstore keepstore \ +package_go_binary cmd/arvados-server keepstore \ "Keep storage daemon, accessible to clients on the LAN" package_go_binary services/keep-web keep-web \ "Static web hosting service for user data stored in Arvados Keep" diff --git a/build/run-tests.sh b/build/run-tests.sh index 71da30ce43..d318bc60de 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -1082,9 +1082,7 @@ install_deps() { do_install services/api do_install services/arv-git-httpd go do_install services/keepproxy go - do_install services/keepstore go do_install services/keep-web go - do_install services/ws go } install_all() { diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go index 4b94a78138..c8b945bea4 100644 --- a/cmd/arvados-server/cmd.go +++ b/cmd/arvados-server/cmd.go @@ -17,6 +17,7 @@ import ( "git.arvados.org/arvados.git/lib/install" "git.arvados.org/arvados.git/lib/lsf" "git.arvados.org/arvados.git/lib/recovercollection" + "git.arvados.org/arvados.git/services/keepstore" "git.arvados.org/arvados.git/services/ws" ) @@ -37,6 +38,7 @@ var ( "dispatch-lsf": lsf.DispatchCommand, "install": install.Command, "init": install.InitCommand, + "keepstore": keepstore.Command, "recover-collection": recovercollection.Command, "ws": ws.Command, }) diff --git a/services/keepstore/keepstore.service b/cmd/arvados-server/keepstore.service similarity index 95% rename from services/keepstore/keepstore.service rename to cmd/arvados-server/keepstore.service index 1f14c3f464..bcfde3a788 100644 --- a/services/keepstore/keepstore.service +++ b/cmd/arvados-server/keepstore.service @@ -6,6 +6,7 @@ Description=Arvados Keep Storage Daemon Documentation=https://doc.arvados.org/ After=network.target +AssertPathExists=/etc/arvados/config.yml # systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section StartLimitIntervalSec=0 diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go index 2026b8c843..2c89ccdb00 100644 --- a/lib/boot/supervisor.go +++ b/lib/boot/supervisor.go @@ -245,7 +245,7 @@ func (super *Supervisor) run(cfg *arvados.Config) error { runGoProgram{src: "services/arv-git-httpd", svc: super.cluster.Services.GitHTTP}, runGoProgram{src: "services/health", svc: super.cluster.Services.Health}, runGoProgram{src: "services/keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}}, - runGoProgram{src: "services/keepstore", svc: super.cluster.Services.Keepstore}, + runServiceCommand{name: "keepstore", svc: super.cluster.Services.Keepstore}, runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV}, runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}}, installPassenger{src: "services/api"}, diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index c022e6c874..6d2643a967 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -473,7 +473,7 @@ def _start_keep(n, blob_signing=False): confdata['Clusters']['zzzzz']['Collections']['BlobSigning'] = blob_signing with open(conf, 'w') as f: yaml.safe_dump(confdata, f) - keep_cmd = ["keepstore", "-config", conf] + keep_cmd = ["arvados-server", "keepstore", "-config", conf] with open(_logfilename('keep{}'.format(n)), WRITE_MODE) as logf: with open('/dev/null') as _stdin: diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go index 6e8def8222..cf655c2a5a 100644 --- a/services/keepstore/azure_blob_volume.go +++ b/services/keepstore/azure_blob_volume.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "bytes" diff --git a/services/keepstore/azure_blob_volume_test.go b/services/keepstore/azure_blob_volume_test.go index 3f395e40ca..48d58ee9bf 100644 --- a/services/keepstore/azure_blob_volume_test.go +++ b/services/keepstore/azure_blob_volume_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "bytes" diff --git a/services/keepstore/bufferpool.go b/services/keepstore/bufferpool.go index 623693cd12..b4cc5d38e1 100644 --- a/services/keepstore/bufferpool.go +++ b/services/keepstore/bufferpool.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "sync" diff --git a/services/keepstore/bufferpool_test.go b/services/keepstore/bufferpool_test.go index 72b719ba28..13e1cb4f33 100644 --- a/services/keepstore/bufferpool_test.go +++ b/services/keepstore/bufferpool_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "context" diff --git a/services/keepstore/collision.go b/services/keepstore/collision.go index 4d6583b26b..16f2d09232 100644 --- a/services/keepstore/collision.go +++ b/services/keepstore/collision.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "bytes" diff --git a/services/keepstore/collision_test.go b/services/keepstore/collision_test.go index 0d6fd62833..aa8f0cbaa1 100644 --- a/services/keepstore/collision_test.go +++ b/services/keepstore/collision_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "bytes" diff --git a/services/keepstore/command.go b/services/keepstore/command.go index bf3bf1722c..2a426936ed 100644 --- a/services/keepstore/command.go +++ b/services/keepstore/command.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "context" @@ -30,10 +30,6 @@ var ( Command = service.Command(arvados.ServiceNameKeepstore, newHandlerOrErrorHandler) ) -func main() { - os.Exit(runCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr)) -} - func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { args, ok := convertKeepstoreFlagsToServiceFlags(args, ctxlog.FromContext(context.Background())) if !ok { diff --git a/services/keepstore/command_test.go b/services/keepstore/command_test.go index ad2aa09571..bbfae52f69 100644 --- a/services/keepstore/command_test.go +++ b/services/keepstore/command_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "bytes" diff --git a/services/keepstore/count.go b/services/keepstore/count.go index ccba4f1355..272b5017cb 100644 --- a/services/keepstore/count.go +++ b/services/keepstore/count.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "io" diff --git a/services/keepstore/gocheck_test.go b/services/keepstore/gocheck_test.go index 89d680fc12..90076db5b2 100644 --- a/services/keepstore/gocheck_test.go +++ b/services/keepstore/gocheck_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "gopkg.in/check.v1" diff --git a/services/keepstore/handler_test.go b/services/keepstore/handler_test.go index 16dcd2aaf6..cbb7f38bb1 100644 --- a/services/keepstore/handler_test.go +++ b/services/keepstore/handler_test.go @@ -11,7 +11,7 @@ // The HTTP handlers are responsible for enforcing permission policy, // so these tests must exercise all possible permission permutations. -package main +package keepstore import ( "bytes" diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go index 910033ebb1..29e7b2ca9c 100644 --- a/services/keepstore/handlers.go +++ b/services/keepstore/handlers.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "container/list" diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go index 3c9d5d15e8..b9dbe2777e 100644 --- a/services/keepstore/keepstore.go +++ b/services/keepstore/keepstore.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "time" diff --git a/services/keepstore/metrics.go b/services/keepstore/metrics.go index b2f0aa6638..d04601fbec 100644 --- a/services/keepstore/metrics.go +++ b/services/keepstore/metrics.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "fmt" diff --git a/services/keepstore/mock_mutex_for_test.go b/services/keepstore/mock_mutex_for_test.go index 484b177230..daf0ef05f7 100644 --- a/services/keepstore/mock_mutex_for_test.go +++ b/services/keepstore/mock_mutex_for_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore type MockMutex struct { AllowLock chan struct{} diff --git a/services/keepstore/mounts_test.go b/services/keepstore/mounts_test.go index b60bf9658b..e8c248219f 100644 --- a/services/keepstore/mounts_test.go +++ b/services/keepstore/mounts_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "bytes" diff --git a/services/keepstore/perms.go b/services/keepstore/perms.go index 7b962641e6..7205a4594d 100644 --- a/services/keepstore/perms.go +++ b/services/keepstore/perms.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "time" diff --git a/services/keepstore/perms_test.go b/services/keepstore/perms_test.go index 5460f8eda6..1322374706 100644 --- a/services/keepstore/perms_test.go +++ b/services/keepstore/perms_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "strconv" diff --git a/services/keepstore/pipe_adapters.go b/services/keepstore/pipe_adapters.go index 69ed6d2ff5..6b555054b6 100644 --- a/services/keepstore/pipe_adapters.go +++ b/services/keepstore/pipe_adapters.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "bytes" diff --git a/services/keepstore/proxy_remote.go b/services/keepstore/proxy_remote.go index 8c88a406f4..526bc25299 100644 --- a/services/keepstore/proxy_remote.go +++ b/services/keepstore/proxy_remote.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "context" diff --git a/services/keepstore/proxy_remote_test.go b/services/keepstore/proxy_remote_test.go index 00161bf236..534371cc0e 100644 --- a/services/keepstore/proxy_remote_test.go +++ b/services/keepstore/proxy_remote_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "context" diff --git a/services/keepstore/pull_worker.go b/services/keepstore/pull_worker.go index 57b9469244..abe3dc3857 100644 --- a/services/keepstore/pull_worker.go +++ b/services/keepstore/pull_worker.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "context" diff --git a/services/keepstore/pull_worker_integration_test.go b/services/keepstore/pull_worker_integration_test.go index eb7fe5fd67..3855b4ecd3 100644 --- a/services/keepstore/pull_worker_integration_test.go +++ b/services/keepstore/pull_worker_integration_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "bytes" diff --git a/services/keepstore/pull_worker_test.go b/services/keepstore/pull_worker_test.go index 2013c8c04d..2626e66d88 100644 --- a/services/keepstore/pull_worker_test.go +++ b/services/keepstore/pull_worker_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "bytes" diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go index 4c43b3f469..ee89b156f7 100644 --- a/services/keepstore/s3_volume.go +++ b/services/keepstore/s3_volume.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "bufio" diff --git a/services/keepstore/s3_volume_test.go b/services/keepstore/s3_volume_test.go index 5cb8a668a3..a820983568 100644 --- a/services/keepstore/s3_volume_test.go +++ b/services/keepstore/s3_volume_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "bytes" diff --git a/services/keepstore/s3aws_volume.go b/services/keepstore/s3aws_volume.go index fb2e97efb7..cb0b73cb8b 100644 --- a/services/keepstore/s3aws_volume.go +++ b/services/keepstore/s3aws_volume.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "bytes" diff --git a/services/keepstore/s3aws_volume_test.go b/services/keepstore/s3aws_volume_test.go index fa9e270e0b..c7e2d485df 100644 --- a/services/keepstore/s3aws_volume_test.go +++ b/services/keepstore/s3aws_volume_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "bytes" diff --git a/services/keepstore/stats_ticker.go b/services/keepstore/stats_ticker.go index 342b9e3205..520d4530c2 100644 --- a/services/keepstore/stats_ticker.go +++ b/services/keepstore/stats_ticker.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "sync" diff --git a/services/keepstore/status_test.go b/services/keepstore/status_test.go index cafe9f72ff..80f98adb22 100644 --- a/services/keepstore/status_test.go +++ b/services/keepstore/status_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "encoding/json" diff --git a/services/keepstore/trash_worker.go b/services/keepstore/trash_worker.go index 4063765726..3909d90d92 100644 --- a/services/keepstore/trash_worker.go +++ b/services/keepstore/trash_worker.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "errors" diff --git a/services/keepstore/trash_worker_test.go b/services/keepstore/trash_worker_test.go index 2b1ee2be13..4e20c3feb4 100644 --- a/services/keepstore/trash_worker_test.go +++ b/services/keepstore/trash_worker_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "container/list" diff --git a/services/keepstore/unix_volume.go b/services/keepstore/unix_volume.go index 1706473cc8..a746166048 100644 --- a/services/keepstore/unix_volume.go +++ b/services/keepstore/unix_volume.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "context" diff --git a/services/keepstore/unix_volume_test.go b/services/keepstore/unix_volume_test.go index 1660858098..75d9b22de5 100644 --- a/services/keepstore/unix_volume_test.go +++ b/services/keepstore/unix_volume_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "bytes" diff --git a/services/keepstore/volume.go b/services/keepstore/volume.go index cf0b7a3902..c3b8cd6283 100644 --- a/services/keepstore/volume.go +++ b/services/keepstore/volume.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "context" diff --git a/services/keepstore/volume_generic_test.go b/services/keepstore/volume_generic_test.go index a31b861236..0dd34e3af1 100644 --- a/services/keepstore/volume_generic_test.go +++ b/services/keepstore/volume_generic_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "bytes" diff --git a/services/keepstore/volume_test.go b/services/keepstore/volume_test.go index cc2d21e5a9..950b3989aa 100644 --- a/services/keepstore/volume_test.go +++ b/services/keepstore/volume_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "bytes" diff --git a/services/keepstore/work_queue.go b/services/keepstore/work_queue.go index 56c6376ad3..4c46ec8e65 100644 --- a/services/keepstore/work_queue.go +++ b/services/keepstore/work_queue.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore /* A WorkQueue is an asynchronous thread-safe queue manager. It provides a channel from which items can be read off the queue, and diff --git a/services/keepstore/work_queue_test.go b/services/keepstore/work_queue_test.go index 8a26c090c9..254f96cb2d 100644 --- a/services/keepstore/work_queue_test.go +++ b/services/keepstore/work_queue_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepstore import ( "container/list" -- 2.30.2