1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
17 "git.arvados.org/arvados.git/lib/config"
18 "git.arvados.org/arvados.git/sdk/go/arvados"
19 "git.arvados.org/arvados.git/sdk/go/arvadostest"
20 "git.arvados.org/arvados.git/sdk/go/ctxlog"
21 "github.com/prometheus/client_golang/prometheus"
25 func Test(t *testing.T) {
29 var _ = check.Suite(&suite{})
35 func (s *suite) TearDownTest(c *check.C) {
36 arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil)
39 func (s *suite) SetUpTest(c *check.C) {
40 cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
41 c.Assert(err, check.IsNil)
42 cluster, err := cfg.GetCluster("")
43 c.Assert(err, check.IsNil)
44 cluster.Containers.CloudVMs.PollInterval = arvados.Duration(time.Second)
45 s.disp = newHandler(context.Background(), cluster, arvadostest.Dispatch1Token, prometheus.NewRegistry()).(*dispatcher)
46 s.disp.lsfcli.stubCommand = func(string, ...string) *exec.Cmd {
47 return exec.Command("bash", "-c", "echo >&2 unimplemented stub; false")
56 func (stub lsfstub) stubCommand(s *suite, c *check.C) func(prog string, args ...string) *exec.Cmd {
59 fakejobq := map[int]string{}
60 return func(prog string, args ...string) *exec.Cmd {
61 c.Logf("stubCommand: %q %q", prog, args)
62 if rand.Float64() < stub.errorRate {
63 return exec.Command("bash", "-c", "echo >&2 'stub random failure' && false")
65 if stub.sudoUser != "" && len(args) > 3 &&
69 args[2] == stub.sudoUser {
70 prog, args = args[3], args[4:]
74 defaultArgs := s.disp.Cluster.Containers.LSF.BsubArgumentsList
75 c.Assert(len(args), check.Equals, len(defaultArgs))
76 // %%J must have been rewritten to %J
77 c.Check(args[1], check.Equals, "/tmp/crunch-run.%J.out")
80 case arvadostest.LockedContainerUUID:
81 c.Check(args, check.DeepEquals, []string{
82 "-J", arvadostest.LockedContainerUUID,
85 "-R", "rusage[mem=11701MB:tmp=0MB] span[hosts=1]"})
87 fakejobq[nextjobid] = args[1]
90 case arvadostest.QueuedContainerUUID:
91 c.Check(args, check.DeepEquals, []string{
92 "-J", arvadostest.QueuedContainerUUID,
95 "-R", "rusage[mem=11701MB:tmp=45777MB] span[hosts=1]"})
97 fakejobq[nextjobid] = args[1]
101 c.Errorf("unexpected uuid passed to bsub: args %q", args)
102 return exec.Command("false")
104 return exec.Command("echo", "submitted job")
106 c.Check(args, check.DeepEquals, []string{"-u", "all", "-noheader", "-o", "jobid stat job_name:30"})
108 for jobid, uuid := range fakejobq {
109 out += fmt.Sprintf(`%d %s %s\n`, jobid, "RUN", uuid)
111 c.Logf("bjobs out: %q", out)
112 return exec.Command("printf", out)
114 killid, _ := strconv.Atoi(args[0])
115 if uuid, ok := fakejobq[killid]; !ok {
116 return exec.Command("bash", "-c", fmt.Sprintf("printf >&2 'Job <%d>: No matching job found\n'", killid))
117 } else if uuid == "" {
118 return exec.Command("bash", "-c", fmt.Sprintf("printf >&2 'Job <%d>: Job has already finished\n'", killid))
121 time.Sleep(time.Millisecond)
123 delete(fakejobq, killid)
126 return exec.Command("bash", "-c", fmt.Sprintf("printf 'Job <%d> is being terminated\n'", killid))
129 return exec.Command("bash", "-c", fmt.Sprintf("echo >&2 'stub: command not found: %+q'", prog))
134 func (s *suite) TestSubmit(c *check.C) {
135 s.disp.lsfcli.stubCommand = lsfstub{
137 sudoUser: s.disp.Cluster.Containers.LSF.BsubSudoUser,
140 deadline := time.Now().Add(20 * time.Second)
141 for range time.NewTicker(time.Second).C {
142 if time.Now().After(deadline) {
146 // "queuedcontainer" should be running
147 if _, ok := s.disp.lsfqueue.JobID(arvadostest.QueuedContainerUUID); !ok {
150 // "lockedcontainer" should be cancelled because it
151 // has priority 0 (no matching container requests)
152 if _, ok := s.disp.lsfqueue.JobID(arvadostest.LockedContainerUUID); ok {
155 var ctr arvados.Container
156 if err := s.disp.arvDispatcher.Arv.Get("containers", arvadostest.LockedContainerUUID, nil, &ctr); err != nil {
157 c.Logf("error getting container state for %s: %s", arvadostest.LockedContainerUUID, err)
160 if ctr.State != arvados.ContainerStateQueued {
161 c.Logf("LockedContainer is not in the LSF queue but its arvados record has not been updated to state==Queued (state is %q)", ctr.State)
164 c.Log("reached desired state")