From: Tom Clegg Date: Wed, 18 May 2022 18:01:21 +0000 (-0400) Subject: 15370: Merge branch 'main' into 15370-loopback-dispatchcloud X-Git-Tag: 2.5.0~142^2~9 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/bcad695db9a1c3aac5807faa153086e653107f51?hp=-c 15370: Merge branch 'main' into 15370-loopback-dispatchcloud Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- bcad695db9a1c3aac5807faa153086e653107f51 diff --combined lib/boot/supervisor.go index d69cc9f18c,4b15266006..7e641c62dd --- a/lib/boot/supervisor.go +++ b/lib/boot/supervisor.go @@@ -365,19 -365,19 +365,19 @@@ func (super *Supervisor) runCluster() e runNginx{}, runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{seedDatabase{}}}, runServiceCommand{name: "git-httpd", svc: super.cluster.Services.GitHTTP}, - runGoProgram{src: "services/health", svc: super.cluster.Services.Health}, + runServiceCommand{name: "health", svc: super.cluster.Services.Health}, runServiceCommand{name: "keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}}, runServiceCommand{name: "keepstore", svc: super.cluster.Services.Keepstore}, runServiceCommand{name: "keep-web", svc: super.cluster.Services.WebDAV}, runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}}, - installPassenger{src: "services/api"}, - runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}}, + installPassenger{src: "services/api", varlibdir: "railsapi"}, + runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api", varlibdir: "railsapi"}}}, seedDatabase{}, } if !super.NoWorkbench1 { tasks = append(tasks, - installPassenger{src: "apps/workbench", depends: []supervisedTask{seedDatabase{}}}, // dependency ensures workbench doesn't delay api install/startup - runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}}, + installPassenger{src: "apps/workbench", varlibdir: "workbench1", depends: []supervisedTask{seedDatabase{}}}, // dependency ensures workbench doesn't delay api install/startup + runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench", varlibdir: "workbench1"}}}, ) } if !super.NoWorkbench2 { @@@ -387,14 -387,10 +387,14 @@@ } if super.ClusterType != "test" { tasks = append(tasks, - runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.DispatchCloud}, runServiceCommand{name: "keep-balance", svc: super.cluster.Services.Keepbalance}, ) } + if super.cluster.Containers.CloudVMs.Enable { + tasks = append(tasks, + runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.DispatchCloud}, + ) + } super.tasksReady = map[string]chan bool{} for _, task := range tasks { super.tasksReady[task.String()] = make(chan bool) @@@ -828,6 -824,9 +828,6 @@@ func (super *Supervisor) autofillConfig &super.cluster.Services.Workbench1, &super.cluster.Services.Workbench2, } { - if svc == &super.cluster.Services.DispatchCloud && super.ClusterType == "test" { - continue - } if svc.ExternalURL.Host == "" { port, err := nextPort(defaultExtHost) if err != nil { diff --combined lib/config/config.default.yml index b09c1ecb9f,893542df18..d965633055 --- a/lib/config/config.default.yml +++ b/lib/config/config.default.yml @@@ -55,6 -55,9 +55,9 @@@ Clusters DispatchLSF: InternalURLs: {SAMPLE: {}} ExternalURL: "" + DispatchSLURM: + InternalURLs: {SAMPLE: {}} + ExternalURL: "" Keepproxy: InternalURLs: {SAMPLE: {}} ExternalURL: "" @@@ -1278,9 -1281,7 +1281,9 @@@ # need to be detected and cleaned up manually. TagKeyPrefix: Arvados - # Cloud driver: "azure" (Microsoft Azure) or "ec2" (Amazon AWS). + # Cloud driver: "azure" (Microsoft Azure), "ec2" (Amazon AWS), + # or "loopback" (run containers on dispatch host for testing + # purposes). Driver: ec2 # Cloud-specific driver parameters. diff --combined lib/crunchrun/executor_test.go index 1833fc8ac4,3301a6e63b..5b146a6321 --- a/lib/crunchrun/executor_test.go +++ b/lib/crunchrun/executor_test.go @@@ -7,15 -7,43 +7,15 @@@ package crunchru import ( "bytes" "io" - "io/ioutil" - "net/http" - "os" "strings" "time" "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadostest" "golang.org/x/net/context" . "gopkg.in/check.v1" ) -func busyboxDockerImage(c *C) string { - fnm := "busybox_uclibc.tar" - cachedir := c.MkDir() - cachefile := cachedir + "/" + fnm - if _, err := os.Stat(cachefile); err == nil { - return cachefile - } - - f, err := ioutil.TempFile(cachedir, "") - c.Assert(err, IsNil) - defer f.Close() - defer os.Remove(f.Name()) - - resp, err := http.Get("https://cache.arvados.org/" + fnm) - c.Assert(err, IsNil) - defer resp.Body.Close() - _, err = io.Copy(f, resp.Body) - c.Assert(err, IsNil) - err = f.Close() - c.Assert(err, IsNil) - err = os.Rename(f.Name(), cachefile) - c.Assert(err, IsNil) - - return cachefile -} - type nopWriteCloser struct{ io.Writer } func (nopWriteCloser) Close() error { return nil } @@@ -43,7 -71,7 +43,7 @@@ func (s *executorSuite) SetUpTest(c *C Stdout: nopWriteCloser{&s.stdout}, Stderr: nopWriteCloser{&s.stderr}, } - err := s.executor.LoadImage("", busyboxDockerImage(c), arvados.Container{}, "", nil) + err := s.executor.LoadImage("", arvadostest.BusyboxDockerImage(c), arvados.Container{}, "", nil) c.Assert(err, IsNil) } @@@ -52,6 -80,7 +52,7 @@@ func (s *executorSuite) TearDownTest(c } func (s *executorSuite) TestExecTrivialContainer(c *C) { + c.Logf("Using container runtime: %s", s.executor.Runtime()) s.spec.Command = []string{"echo", "ok"} s.checkRun(c, 0) c.Check(s.stdout.String(), Equals, "ok\n") diff --combined lib/crunchrun/integration_test.go index ff2165338a,ec08937283..ce92a9b807 --- a/lib/crunchrun/integration_test.go +++ b/lib/crunchrun/integration_test.go @@@ -51,7 -51,7 +51,7 @@@ func (s *integrationSuite) SetUpSuite( arvadostest.StartKeep(2, true) - out, err := exec.Command("docker", "load", "--input", busyboxDockerImage(c)).CombinedOutput() + out, err := exec.Command("docker", "load", "--input", arvadostest.BusyboxDockerImage(c)).CombinedOutput() c.Log(string(out)) c.Assert(err, IsNil) out, err = exec.Command("arv-keepdocker", "--no-resume", "busybox:uclibc").Output() @@@ -162,11 -162,13 +162,13 @@@ func (s *integrationSuite) setup(c *C) func (s *integrationSuite) TestRunTrivialContainerWithDocker(c *C) { s.engine = "docker" s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*Using container runtime: docker Engine \d+\.\d+.*`) } func (s *integrationSuite) TestRunTrivialContainerWithSingularity(c *C) { s.engine = "singularity" s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*Using container runtime: singularity.* version 3\.\d+.*`) } func (s *integrationSuite) TestRunTrivialContainerWithLocalKeepstore(c *C) {