1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
23 "git.arvados.org/arvados.git/lib/cmd"
24 "git.arvados.org/arvados.git/lib/config"
25 "git.arvados.org/arvados.git/sdk/go/arvados"
26 "git.arvados.org/arvados.git/sdk/go/ctxlog"
27 "git.arvados.org/arvados.git/sdk/go/health"
28 "github.com/sirupsen/logrus"
33 func (Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
35 f := flag.NewFlagSet(prog, flag.ContinueOnError)
36 f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects")
37 f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
38 f.StringVar(&diag.dockerImage, "docker-image", "", "image to use when running a test container (default: use embedded hello-world image)")
39 f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client")
40 f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client")
41 f.BoolVar(&diag.verbose, "v", false, "verbose: include more information in report")
42 f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000, or 0 to skip)")
43 f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests")
44 if ok, code := cmd.ParseFlags(f, prog, args, "", stderr); !ok {
47 diag.logger = ctxlog.New(stdout, "text", diag.logLevel)
48 diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true, PadLevelText: true})
50 if len(diag.errors) == 0 {
51 diag.logger.Info("--- no errors ---")
54 if diag.logger.Level > logrus.ErrorLevel {
55 fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n")
56 for _, e := range diag.errors {
64 // docker save hello-world > hello-world.tar
66 //go:embed hello-world.tar
67 var HelloWorldDockerImage []byte
69 type diagnoser struct {
85 func (diag *diagnoser) debugf(f string, args ...interface{}) {
86 diag.logger.Debugf(" ... "+f, args...)
89 func (diag *diagnoser) infof(f string, args ...interface{}) {
90 diag.logger.Infof(" ... "+f, args...)
93 func (diag *diagnoser) verbosef(f string, args ...interface{}) {
95 diag.logger.Infof(" ... "+f, args...)
99 func (diag *diagnoser) warnf(f string, args ...interface{}) {
100 diag.logger.Warnf(" ... "+f, args...)
103 func (diag *diagnoser) errorf(f string, args ...interface{}) {
104 diag.logger.Errorf(f, args...)
105 diag.errors = append(diag.errors, fmt.Sprintf(f, args...))
108 // Run the given func, logging appropriate messages before and after,
109 // adding timing info, etc.
111 // The id argument should be unique among tests, and shouldn't change
112 // when other tests are added/removed.
113 func (diag *diagnoser) dotest(id int, title string, fn func() error) {
114 if diag.done == nil {
115 diag.done = map[int]bool{}
116 } else if diag.done[id] {
117 diag.errorf("(bug) reused test id %d", id)
121 diag.logger.Infof("%4d: %s", id, title)
124 elapsed := fmt.Sprintf("%d ms", time.Now().Sub(t0)/time.Millisecond)
126 diag.errorf("%4d: %s (%s): %s", id, title, elapsed, err)
128 diag.logger.Debugf("%4d: %s (%s): ok", id, title, elapsed)
132 func (diag *diagnoser) runtests() {
133 client := arvados.NewClientFromEnv()
134 // Disable auto-retry, use context instead
137 if client.APIHost == "" || client.AuthToken == "" {
138 diag.errorf("ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables are not set -- aborting without running any tests")
142 hostname, err := os.Hostname()
144 diag.warnf("error getting hostname: %s")
146 diag.verbosef("hostname = %s", hostname)
149 diag.dotest(5, "running health check (same as `arvados-server check`)", func() error {
150 ldr := config.NewLoader(&bytes.Buffer{}, ctxlog.New(&bytes.Buffer{}, "text", "info"))
151 ldr.SetupFlags(flag.NewFlagSet("diagnostics", flag.ContinueOnError))
152 cfg, err := ldr.Load()
154 diag.infof("skipping because config could not be loaded: %s", err)
157 cluster, err := cfg.GetCluster("")
161 if cluster.SystemRootToken != os.Getenv("ARVADOS_API_TOKEN") {
162 return fmt.Errorf("diagnostics usage error: %s is readable but SystemRootToken does not match $ARVADOS_API_TOKEN (to fix, either run 'arvados-client sudo diagnostics' to load everything from config file, or set ARVADOS_CONFIG=- to load nothing from config file)", ldr.Path)
164 agg := &health.Aggregator{Cluster: cluster}
165 resp := agg.ClusterHealth()
166 for _, e := range resp.Errors {
167 diag.errorf("health check: %s", e)
169 if len(resp.Errors) > 0 {
170 diag.infof("consider running `arvados-server check -yaml` for a comprehensive report")
172 diag.verbosef("reported clock skew = %v", resp.ClockSkew)
173 reported := map[string]bool{}
174 for _, result := range resp.Checks {
175 version := strings.SplitN(result.Metrics.Version, " (go", 2)[0]
176 if version != "" && !reported[version] {
177 diag.verbosef("arvados version = %s", version)
178 reported[version] = true
181 reported = map[string]bool{}
182 for _, result := range resp.Checks {
183 if result.Server != "" && !reported[result.Server] {
184 diag.verbosef("http frontend version = %s", result.Server)
185 reported[result.Server] = true
188 reported = map[string]bool{}
189 for _, result := range resp.Checks {
190 if sha := result.ConfigSourceSHA256; sha != "" && !reported[sha] {
191 diag.verbosef("config file sha256 = %s", sha)
198 var dd arvados.DiscoveryDocument
199 ddpath := "discovery/v1/apis/arvados/v1/rest"
200 diag.dotest(10, fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath), func() error {
201 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
203 err := client.RequestAndDecodeContext(ctx, &dd, "GET", ddpath, nil, nil)
207 diag.verbosef("BlobSignatureTTL = %d", dd.BlobSignatureTTL)
211 var cluster arvados.Cluster
212 cfgpath := "arvados/v1/config"
214 diag.dotest(20, fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath), func() error {
215 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
217 err := client.RequestAndDecodeContext(ctx, &cluster, "GET", cfgpath, nil, nil)
221 diag.verbosef("Collections.BlobSigning = %v", cluster.Collections.BlobSigning)
226 var user arvados.User
227 diag.dotest(30, "getting current user record", func() error {
228 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
230 err := client.RequestAndDecodeContext(ctx, &user, "GET", "arvados/v1/users/current", nil, nil)
234 diag.verbosef("user uuid = %s", user.UUID)
239 diag.errorf("cannot proceed without cluster config -- aborting without running any further tests")
243 // uncomment to create some spurious errors
244 // cluster.Services.WebDAVDownload.ExternalURL.Host = "0.0.0.0:9"
246 // TODO: detect routing errors here, like finding wb2 at the
248 for i, svc := range []*arvados.Service{
249 &cluster.Services.Keepproxy,
250 &cluster.Services.WebDAV,
251 &cluster.Services.WebDAVDownload,
252 &cluster.Services.Websocket,
253 &cluster.Services.Workbench1,
254 &cluster.Services.Workbench2,
256 diag.dotest(40+i, fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL), func() error {
257 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
260 if strings.HasPrefix(u.Scheme, "ws") {
261 // We can do a real websocket test elsewhere,
262 // but for now we'll just check the https
264 u.Scheme = "http" + u.Scheme[2:]
266 if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
267 u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
269 req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
273 resp, err := http.DefaultClient.Do(req)
282 for i, url := range []string{
283 cluster.Services.Controller.ExternalURL.String(),
284 cluster.Services.Keepproxy.ExternalURL.String() + "d41d8cd98f00b204e9800998ecf8427e+0",
285 cluster.Services.WebDAVDownload.ExternalURL.String(),
287 diag.dotest(50+i, fmt.Sprintf("checking CORS headers at %s", url), func() error {
288 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
290 req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
294 req.Header.Set("Origin", "https://example.com")
295 resp, err := http.DefaultClient.Do(req)
299 if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" {
300 return fmt.Errorf("expected \"Access-Control-Allow-Origin: *\", got %q", hdr)
306 var keeplist arvados.KeepServiceList
307 diag.dotest(60, "checking internal/external client detection", func() error {
308 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
310 err := client.RequestAndDecodeContext(ctx, &keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: 999999})
312 return fmt.Errorf("error getting keep services list: %s", err)
313 } else if len(keeplist.Items) == 0 {
314 return fmt.Errorf("controller did not return any keep services")
316 found := map[string]int{}
317 for _, ks := range keeplist.Items {
318 found[ks.ServiceType]++
320 isInternal := found["proxy"] == 0 && len(keeplist.Items) > 0
321 isExternal := found["proxy"] > 0 && found["proxy"] == len(keeplist.Items)
323 diag.infof("controller returned only proxy services, this host is treated as \"external\"")
324 } else if isInternal {
325 diag.infof("controller returned only non-proxy services, this host is treated as \"internal\"")
327 if (diag.checkInternal && !isInternal) || (diag.checkExternal && !isExternal) {
328 return fmt.Errorf("expecting internal=%v external=%v, but found internal=%v external=%v", diag.checkInternal, diag.checkExternal, isInternal, isExternal)
333 for i, ks := range keeplist.Items {
336 Host: net.JoinHostPort(ks.ServiceHost, fmt.Sprintf("%d", ks.ServicePort)),
339 if ks.ServiceSSLFlag {
342 diag.dotest(61+i, fmt.Sprintf("reading+writing via keep service at %s", u.String()), func() error {
343 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
345 req, err := http.NewRequestWithContext(ctx, "PUT", u.String()+"d41d8cd98f00b204e9800998ecf8427e", nil)
349 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
350 resp, err := http.DefaultClient.Do(req)
354 defer resp.Body.Close()
355 body, err := ioutil.ReadAll(resp.Body)
357 return fmt.Errorf("reading response body: %s", err)
359 loc := strings.TrimSpace(string(body))
360 if !strings.HasPrefix(loc, "d41d8") {
361 return fmt.Errorf("unexpected response from write: %q", body)
364 req, err = http.NewRequestWithContext(ctx, "GET", u.String()+loc, nil)
368 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
369 resp, err = http.DefaultClient.Do(req)
373 defer resp.Body.Close()
374 body, err = ioutil.ReadAll(resp.Body)
376 return fmt.Errorf("reading response body: %s", err)
379 return fmt.Errorf("unexpected response from read: %q", body)
386 var project arvados.Group
387 diag.dotest(80, fmt.Sprintf("finding/creating %q project", diag.projectName), func() error {
388 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
390 var grplist arvados.GroupList
391 err := client.RequestAndDecodeContext(ctx, &grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
392 Filters: []arvados.Filter{
393 {"name", "=", diag.projectName},
394 {"group_class", "=", "project"},
395 {"owner_uuid", "=", user.UUID}},
398 return fmt.Errorf("list groups: %s", err)
400 if len(grplist.Items) > 0 {
401 project = grplist.Items[0]
402 diag.verbosef("using existing project, uuid = %s", project.UUID)
405 diag.debugf("list groups: ok, no results")
406 err = client.RequestAndDecodeContext(ctx, &project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
407 "name": diag.projectName,
408 "group_class": "project",
411 return fmt.Errorf("create project: %s", err)
413 diag.verbosef("created project, uuid = %s", project.UUID)
417 var collection arvados.Collection
418 diag.dotest(90, "creating temporary collection", func() error {
419 if project.UUID == "" {
420 return fmt.Errorf("skipping, no project to work in")
422 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
424 err := client.RequestAndDecodeContext(ctx, &collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
425 "ensure_unique_name": true,
426 "collection": map[string]interface{}{
427 "owner_uuid": project.UUID,
428 "name": "test collection",
429 "trash_at": time.Now().Add(time.Hour)}})
433 diag.verbosef("ok, uuid = %s", collection.UUID)
437 if collection.UUID != "" {
439 diag.dotest(9990, "deleting temporary collection", func() error {
440 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
442 return client.RequestAndDecodeContext(ctx, nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
447 // Read hello-world.tar to find image ID, so we can upload it
448 // as "sha256:{...}.tar"
451 tr := tar.NewReader(bytes.NewReader(HelloWorldDockerImage))
453 hdr, err := tr.Next()
458 diag.errorf("internal error/bug: cannot read embedded docker image tar file: %s", err)
461 if s := strings.TrimSuffix(hdr.Name, ".json"); len(s) == 64 && s != hdr.Name {
466 diag.errorf("internal error/bug: cannot find {sha256}.json file in embedded docker image tar file")
470 tarfilename := "sha256:" + imageSHA2 + ".tar"
472 diag.dotest(100, "uploading file via webdav", func() error {
473 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
475 if collection.UUID == "" {
476 return fmt.Errorf("skipping, no test collection")
478 req, err := http.NewRequestWithContext(ctx, "PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/"+tarfilename, bytes.NewReader(HelloWorldDockerImage))
480 return fmt.Errorf("BUG? http.NewRequest: %s", err)
482 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
483 resp, err := http.DefaultClient.Do(req)
485 return fmt.Errorf("error performing http request: %s", err)
488 if resp.StatusCode != http.StatusCreated {
489 return fmt.Errorf("status %s", resp.Status)
491 diag.debugf("ok, status %s", resp.Status)
492 err = client.RequestAndDecodeContext(ctx, &collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
494 return fmt.Errorf("get updated collection: %s", err)
496 diag.debugf("ok, pdh %s", collection.PortableDataHash)
500 davurl := cluster.Services.WebDAV.ExternalURL
501 davWildcard := strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.")
502 diag.dotest(110, fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl), func() error {
503 if davurl.Host == "" {
504 return fmt.Errorf("host missing - content previews will not work")
506 if !davWildcard && !cluster.Collections.TrustAllContent {
507 diag.warnf("WebDAV ExternalURL has no leading wildcard and TrustAllContent==false - content previews will not work")
512 for i, trial := range []struct {
518 {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
519 {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + tarfilename},
520 {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
521 {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/" + tarfilename},
522 {true, true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + tarfilename},
523 {true, false, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/" + tarfilename},
525 diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error {
526 if trial.needWildcard && !davWildcard {
527 diag.warnf("skipping collection-id-in-vhost test because WebDAV ExternalURL has no leading wildcard")
530 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
532 if trial.needcoll && collection.UUID == "" {
533 return fmt.Errorf("skipping, no test collection")
535 req, err := http.NewRequestWithContext(ctx, "GET", trial.fileurl, nil)
539 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
540 resp, err := http.DefaultClient.Do(req)
544 defer resp.Body.Close()
545 body, err := ioutil.ReadAll(resp.Body)
547 return fmt.Errorf("reading response: %s", err)
549 if resp.StatusCode != trial.status {
550 return fmt.Errorf("unexpected response status: %s", resp.Status)
552 if trial.status == http.StatusOK && !bytes.Equal(body, HelloWorldDockerImage) {
554 if len(excerpt) > 128 {
555 excerpt = append([]byte(nil), body[:128]...)
556 excerpt = append(excerpt, []byte("[...]")...)
558 return fmt.Errorf("unexpected response content: len %d, %q", len(body), excerpt)
564 var vm arvados.VirtualMachine
565 diag.dotest(130, "getting list of virtual machines", func() error {
566 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
568 var vmlist arvados.VirtualMachineList
569 err := client.RequestAndDecodeContext(ctx, &vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
573 if len(vmlist.Items) < 1 {
574 diag.warnf("no VMs found")
581 diag.dotest(140, "getting workbench1 webshell page", func() error {
582 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
585 diag.warnf("skipping, no vm available")
588 webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername"
589 diag.debugf("url %s", webshelltermurl)
590 req, err := http.NewRequestWithContext(ctx, "GET", webshelltermurl, nil)
594 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
595 resp, err := http.DefaultClient.Do(req)
599 defer resp.Body.Close()
600 body, err := ioutil.ReadAll(resp.Body)
602 return fmt.Errorf("reading response: %s", err)
604 if resp.StatusCode != http.StatusOK {
605 return fmt.Errorf("unexpected response status: %s %q", resp.Status, body)
610 diag.dotest(150, "connecting to webshell service", func() error {
611 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
614 diag.warnf("skipping, no vm available")
617 u := cluster.Services.WebShell.ExternalURL
618 webshellurl := u.String() + vm.Hostname + "?"
619 if strings.HasPrefix(u.Host, "*") {
620 u.Host = vm.Hostname + u.Host[1:]
621 webshellurl = u.String() + "?"
623 diag.debugf("url %s", webshellurl)
624 req, err := http.NewRequestWithContext(ctx, "POST", webshellurl, bytes.NewBufferString(url.Values{
627 "session": {"xyzzy"},
628 "rooturl": {webshellurl},
633 req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
634 resp, err := http.DefaultClient.Do(req)
638 defer resp.Body.Close()
639 diag.debugf("response status %s", resp.Status)
640 body, err := ioutil.ReadAll(resp.Body)
642 return fmt.Errorf("reading response: %s", err)
644 diag.debugf("response body %q", body)
645 // We don't speak the protocol, so we get a 400 error
646 // from the webshell server even if everything is
647 // OK. Anything else (404, 502, ???) indicates a
649 if resp.StatusCode != http.StatusBadRequest {
650 return fmt.Errorf("unexpected response status: %s, %q", resp.Status, body)
655 diag.dotest(160, "running a container", func() error {
656 if diag.priority < 1 {
657 diag.infof("skipping (use priority > 0 if you want to run a container)")
660 if project.UUID == "" {
661 return fmt.Errorf("skipping, no project to work in")
664 timestamp := time.Now().Format(time.RFC3339)
665 ctrCommand := []string{"echo", timestamp}
666 if diag.dockerImage == "" {
667 if collection.UUID == "" {
668 return fmt.Errorf("skipping, no test collection to use as docker image")
670 diag.dockerImage = collection.PortableDataHash
671 ctrCommand = []string{"/hello"}
674 var cr arvados.ContainerRequest
675 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
678 err := client.RequestAndDecodeContext(ctx, &cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{
679 "owner_uuid": project.UUID,
680 "name": fmt.Sprintf("diagnostics container request %s", timestamp),
681 "container_image": diag.dockerImage,
682 "command": ctrCommand,
683 "use_existing": false,
684 "output_path": "/mnt/output",
685 "output_name": fmt.Sprintf("diagnostics output %s", timestamp),
686 "priority": diag.priority,
687 "state": arvados.ContainerRequestStateCommitted,
688 "mounts": map[string]map[string]interface{}{
690 "kind": "collection",
694 "runtime_constraints": arvados.RuntimeConstraints{
697 KeepCacheRAM: 1 << 26,
703 diag.verbosef("container request uuid = %s", cr.UUID)
704 diag.verbosef("container uuid = %s", cr.ContainerUUID)
706 timeout := 10 * time.Minute
707 diag.infof("container request submitted, waiting up to %v for container to run", arvados.Duration(timeout))
708 deadline := time.Now().Add(timeout)
710 var c arvados.Container
711 for ; cr.State != arvados.ContainerRequestStateFinal && time.Now().Before(deadline); time.Sleep(2 * time.Second) {
712 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
715 crStateWas := cr.State
716 err := client.RequestAndDecodeContext(ctx, &cr, "GET", "arvados/v1/container_requests/"+cr.UUID, nil, nil)
720 if cr.State != crStateWas {
721 diag.debugf("container request state = %s", cr.State)
725 err = client.RequestAndDecodeContext(ctx, &c, "GET", "arvados/v1/containers/"+cr.ContainerUUID, nil, nil)
729 if c.State != cStateWas {
730 diag.debugf("container state = %s", c.State)
736 if cr.State != arvados.ContainerRequestStateFinal {
737 err := client.RequestAndDecodeContext(context.Background(), &cr, "PATCH", "arvados/v1/container_requests/"+cr.UUID, nil, map[string]interface{}{
738 "container_request": map[string]interface{}{
742 diag.infof("error canceling container request %s: %s", cr.UUID, err)
744 diag.debugf("canceled container request %s", cr.UUID)
746 return fmt.Errorf("timed out waiting for container to finish; container request %s state was %q, container %s state was %q", cr.UUID, cr.State, c.UUID, c.State)
748 if c.State != arvados.ContainerStateComplete {
749 return fmt.Errorf("container request %s is final but container %s did not complete: container state = %q", cr.UUID, cr.ContainerUUID, c.State)
752 return fmt.Errorf("container exited %d", c.ExitCode)