lib/diagnostics/cmd.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package diagnostics
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "flag"
  11         "fmt"
  12         "io"
  13         "io/ioutil"
  14         "net"
  15         "net/http"
  16         "net/url"
  17         "strings"
  18         "time"
  19
  20         "git.arvados.org/arvados.git/sdk/go/arvados"
  21         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  22         "github.com/sirupsen/logrus"
  23 )
  24
  25 type Command struct{}
  26
  27 func (cmd Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
  28         var diag diagnoser
  29         f := flag.NewFlagSet(prog, flag.ContinueOnError)
  30         f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects")
  31         f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
  32         f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client")
  33         f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client")
  34         f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000)")
  35         f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests")
  36         err := f.Parse(args)
  37         if err == flag.ErrHelp {
  38                 return 0
  39         } else if err != nil {
  40                 fmt.Fprintln(stderr, err)
  41                 return 2
  42         }
  43         diag.logger = ctxlog.New(stdout, "text", diag.logLevel)
  44         diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true, PadLevelText: true})
  45         diag.runtests()
  46         if len(diag.errors) == 0 {
  47                 diag.logger.Info("--- no errors ---")
  48                 return 0
  49         } else {
  50                 if diag.logger.Level > logrus.ErrorLevel {
  51                         fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n")
  52                         for _, e := range diag.errors {
  53                                 diag.logger.Error(e)
  54                         }
  55                 }
  56                 return 1
  57         }
  58 }
  59
  60 type diagnoser struct {
  61         stdout        io.Writer
  62         stderr        io.Writer
  63         logLevel      string
  64         priority      int
  65         projectName   string
  66         checkInternal bool
  67         checkExternal bool
  68         timeout       time.Duration
  69         logger        *logrus.Logger
  70         errors        []string
  71         done          map[int]bool
  72 }
  73
  74 func (diag *diagnoser) debugf(f string, args ...interface{}) {
  75         diag.logger.Debugf(f, args...)
  76 }
  77
  78 func (diag *diagnoser) infof(f string, args ...interface{}) {
  79         diag.logger.Infof(f, args...)
  80 }
  81
  82 func (diag *diagnoser) warnf(f string, args ...interface{}) {
  83         diag.logger.Warnf(f, args...)
  84 }
  85
  86 func (diag *diagnoser) errorf(f string, args ...interface{}) {
  87         diag.logger.Errorf(f, args...)
  88         diag.errors = append(diag.errors, fmt.Sprintf(f, args...))
  89 }
  90
  91 // Run the given func, logging appropriate messages before and after,
  92 // adding timing info, etc.
  93 //
  94 // The id argument should be unique among tests, and shouldn't change
  95 // when other tests are added/removed.
  96 func (diag *diagnoser) dotest(id int, title string, fn func() error) {
  97         if diag.done == nil {
  98                 diag.done = map[int]bool{}
  99         } else if diag.done[id] {
 100                 diag.errorf("(bug) reused test id %d", id)
 101         }
 102         diag.done[id] = true
 103
 104         diag.infof("%4d: %s", id, title)
 105         t0 := time.Now()
 106         err := fn()
 107         elapsed := fmt.Sprintf("%.0dms", time.Now().Sub(t0)/time.Millisecond)
 108         if err != nil {
 109                 diag.errorf("%4d: %s (%s): %s", id, title, elapsed, err)
 110         } else {
 111                 diag.debugf("%4d: %s (%s): ok", id, title, elapsed)
 112         }
 113 }
 114
 115 func (diag *diagnoser) runtests() {
 116         client := arvados.NewClientFromEnv()
 117
 118         if client.APIHost == "" || client.AuthToken == "" {
 119                 diag.errorf("ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables are not set -- aborting without running any tests")
 120                 return
 121         }
 122
 123         var dd arvados.DiscoveryDocument
 124         ddpath := "discovery/v1/apis/arvados/v1/rest"
 125         diag.dotest(10, fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath), func() error {
 126                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 127                 defer cancel()
 128                 err := client.RequestAndDecodeContext(ctx, &dd, "GET", ddpath, nil, nil)
 129                 if err != nil {
 130                         return err
 131                 }
 132                 diag.debugf("BlobSignatureTTL = %d", dd.BlobSignatureTTL)
 133                 return nil
 134         })
 135
 136         var cluster arvados.Cluster
 137         cfgpath := "arvados/v1/config"
 138         diag.dotest(20, fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath), func() error {
 139                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 140                 defer cancel()
 141                 err := client.RequestAndDecodeContext(ctx, &cluster, "GET", cfgpath, nil, nil)
 142                 if err != nil {
 143                         return err
 144                 }
 145                 diag.debugf("Collections.BlobSigning = %v", cluster.Collections.BlobSigning)
 146                 return nil
 147         })
 148
 149         var user arvados.User
 150         diag.dotest(30, "getting current user record", func() error {
 151                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 152                 defer cancel()
 153                 err := client.RequestAndDecodeContext(ctx, &user, "GET", "arvados/v1/users/current", nil, nil)
 154                 if err != nil {
 155                         return err
 156                 }
 157                 diag.debugf("user uuid = %s", user.UUID)
 158                 return nil
 159         })
 160
 161         // uncomment to create some spurious errors
 162         // cluster.Services.WebDAVDownload.ExternalURL.Host = "0.0.0.0:9"
 163
 164         // TODO: detect routing errors here, like finding wb2 at the
 165         // wb1 address.
 166         for i, svc := range []*arvados.Service{
 167                 &cluster.Services.Keepproxy,
 168                 &cluster.Services.WebDAV,
 169                 &cluster.Services.WebDAVDownload,
 170                 &cluster.Services.Websocket,
 171                 &cluster.Services.Workbench1,
 172                 &cluster.Services.Workbench2,
 173         } {
 174                 diag.dotest(40+i, fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL), func() error {
 175                         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 176                         defer cancel()
 177                         u := svc.ExternalURL
 178                         if strings.HasPrefix(u.Scheme, "ws") {
 179                                 // We can do a real websocket test elsewhere,
 180                                 // but for now we'll just check the https
 181                                 // connection.
 182                                 u.Scheme = "http" + u.Scheme[2:]
 183                         }
 184                         if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
 185                                 u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
 186                         }
 187                         req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
 188                         if err != nil {
 189                                 return err
 190                         }
 191                         resp, err := http.DefaultClient.Do(req)
 192                         if err != nil {
 193                                 return err
 194                         }
 195                         resp.Body.Close()
 196                         return nil
 197                 })
 198         }
 199
 200         for i, url := range []string{
 201                 cluster.Services.Controller.ExternalURL.String(),
 202                 cluster.Services.Keepproxy.ExternalURL.String() + "d41d8cd98f00b204e9800998ecf8427e+0",
 203                 cluster.Services.WebDAVDownload.ExternalURL.String(),
 204         } {
 205                 diag.dotest(50+i, fmt.Sprintf("checking CORS headers at %s", url), func() error {
 206                         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 207                         defer cancel()
 208                         req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 209                         if err != nil {
 210                                 return err
 211                         }
 212                         req.Header.Set("Origin", "https://example.com")
 213                         resp, err := http.DefaultClient.Do(req)
 214                         if err != nil {
 215                                 return err
 216                         }
 217                         if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" {
 218                                 return fmt.Errorf("expected \"Access-Control-Allow-Origin: *\", got %q", hdr)
 219                         }
 220                         return nil
 221                 })
 222         }
 223
 224         var keeplist arvados.KeepServiceList
 225         diag.dotest(60, "checking internal/external client detection", func() error {
 226                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 227                 defer cancel()
 228                 err := client.RequestAndDecodeContext(ctx, &keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: 999999})
 229                 if err != nil {
 230                         return fmt.Errorf("error getting keep services list: %s", err)
 231                 } else if len(keeplist.Items) == 0 {
 232                         return fmt.Errorf("controller did not return any keep services")
 233                 }
 234                 found := map[string]int{}
 235                 for _, ks := range keeplist.Items {
 236                         found[ks.ServiceType]++
 237                 }
 238                 isInternal := found["proxy"] == 0 && len(keeplist.Items) > 0
 239                 isExternal := found["proxy"] > 0 && found["proxy"] == len(keeplist.Items)
 240                 if isExternal {
 241                         diag.debugf("controller returned only proxy services, this host is treated as \"external\"")
 242                 } else if isInternal {
 243                         diag.debugf("controller returned only non-proxy services, this host is treated as \"internal\"")
 244                 }
 245                 if (diag.checkInternal && !isInternal) || (diag.checkExternal && !isExternal) {
 246                         return fmt.Errorf("expecting internal=%v external=%v, but found internal=%v external=%v", diag.checkInternal, diag.checkExternal, isInternal, isExternal)
 247                 }
 248                 return nil
 249         })
 250
 251         for i, ks := range keeplist.Items {
 252                 u := url.URL{
 253                         Scheme: "http",
 254                         Host:   net.JoinHostPort(ks.ServiceHost, fmt.Sprintf("%d", ks.ServicePort)),
 255                         Path:   "/",
 256                 }
 257                 if ks.ServiceSSLFlag {
 258                         u.Scheme = "https"
 259                 }
 260                 diag.dotest(61+i, fmt.Sprintf("reading+writing via keep service at %s", u.String()), func() error {
 261                         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 262                         defer cancel()
 263                         req, err := http.NewRequestWithContext(ctx, "PUT", u.String()+"d41d8cd98f00b204e9800998ecf8427e", nil)
 264                         if err != nil {
 265                                 return err
 266                         }
 267                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 268                         resp, err := http.DefaultClient.Do(req)
 269                         if err != nil {
 270                                 return err
 271                         }
 272                         defer resp.Body.Close()
 273                         body, err := ioutil.ReadAll(resp.Body)
 274                         if err != nil {
 275                                 return fmt.Errorf("reading response body: %s", err)
 276                         }
 277                         loc := strings.TrimSpace(string(body))
 278                         if !strings.HasPrefix(loc, "d41d8") {
 279                                 return fmt.Errorf("unexpected response from write: %q", body)
 280                         }
 281
 282                         req, err = http.NewRequestWithContext(ctx, "GET", u.String()+loc, nil)
 283                         if err != nil {
 284                                 return err
 285                         }
 286                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 287                         resp, err = http.DefaultClient.Do(req)
 288                         if err != nil {
 289                                 return err
 290                         }
 291                         defer resp.Body.Close()
 292                         body, err = ioutil.ReadAll(resp.Body)
 293                         if err != nil {
 294                                 return fmt.Errorf("reading response body: %s", err)
 295                         }
 296                         if len(body) != 0 {
 297                                 return fmt.Errorf("unexpected response from read: %q", body)
 298                         }
 299
 300                         return nil
 301                 })
 302         }
 303
 304         var project arvados.Group
 305         diag.dotest(80, fmt.Sprintf("finding/creating %q project", diag.projectName), func() error {
 306                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 307                 defer cancel()
 308                 var grplist arvados.GroupList
 309                 err := client.RequestAndDecodeContext(ctx, &grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
 310                         Filters: []arvados.Filter{
 311                                 {"name", "=", diag.projectName},
 312                                 {"group_class", "=", "project"},
 313                                 {"owner_uuid", "=", user.UUID}},
 314                         Limit: 999999})
 315                 if err != nil {
 316                         return fmt.Errorf("list groups: %s", err)
 317                 }
 318                 if len(grplist.Items) > 0 {
 319                         project = grplist.Items[0]
 320                         diag.debugf("using existing project, uuid = %s", project.UUID)
 321                         return nil
 322                 }
 323                 diag.debugf("list groups: ok, no results")
 324                 err = client.RequestAndDecodeContext(ctx, &project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
 325                         "name":        diag.projectName,
 326                         "group_class": "project",
 327                 }})
 328                 if err != nil {
 329                         return fmt.Errorf("create project: %s", err)
 330                 }
 331                 diag.debugf("created project, uuid = %s", project.UUID)
 332                 return nil
 333         })
 334
 335         var collection arvados.Collection
 336         diag.dotest(90, "creating temporary collection", func() error {
 337                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 338                 defer cancel()
 339                 err := client.RequestAndDecodeContext(ctx, &collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 340                         "ensure_unique_name": true,
 341                         "collection": map[string]interface{}{
 342                                 "name":     "test collection",
 343                                 "trash_at": time.Now().Add(time.Hour)}})
 344                 if err != nil {
 345                         return err
 346                 }
 347                 diag.debugf("ok, uuid = %s", collection.UUID)
 348                 return nil
 349         })
 350
 351         if collection.UUID != "" {
 352                 defer func() {
 353                         diag.dotest(9990, "deleting temporary collection", func() error {
 354                                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 355                                 defer cancel()
 356                                 return client.RequestAndDecodeContext(ctx, nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
 357                         })
 358                 }()
 359         }
 360
 361         diag.dotest(100, "uploading file via webdav", func() error {
 362                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 363                 defer cancel()
 364                 if collection.UUID == "" {
 365                         return fmt.Errorf("skipping, no test collection")
 366                 }
 367                 req, err := http.NewRequestWithContext(ctx, "PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata"))
 368                 if err != nil {
 369                         return fmt.Errorf("BUG? http.NewRequest: %s", err)
 370                 }
 371                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 372                 resp, err := http.DefaultClient.Do(req)
 373                 if err != nil {
 374                         return fmt.Errorf("error performing http request: %s", err)
 375                 }
 376                 resp.Body.Close()
 377                 if resp.StatusCode != http.StatusCreated {
 378                         return fmt.Errorf("status %s", resp.Status)
 379                 }
 380                 diag.debugf("ok, status %s", resp.Status)
 381                 err = client.RequestAndDecodeContext(ctx, &collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
 382                 if err != nil {
 383                         return fmt.Errorf("get updated collection: %s", err)
 384                 }
 385                 diag.debugf("ok, pdh %s", collection.PortableDataHash)
 386                 return nil
 387         })
 388
 389         davurl := cluster.Services.WebDAV.ExternalURL
 390         diag.dotest(110, fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl), func() error {
 391                 if davurl.Host == "" {
 392                         return fmt.Errorf("host missing - content previews will not work")
 393                 }
 394                 if !strings.HasPrefix(davurl.Host, "*--") && !strings.HasPrefix(davurl.Host, "*.") && !cluster.Collections.TrustAllContent {
 395                         diag.warnf("WebDAV ExternalURL has no leading wildcard and TrustAllContent==false - content previews will not work")
 396                 }
 397                 return nil
 398         })
 399
 400         for i, trial := range []struct {
 401                 needcoll bool
 402                 status   int
 403                 fileurl  string
 404         }{
 405                 {false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
 406                 {false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"},
 407                 {false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
 408                 {false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"},
 409                 {true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"},
 410                 {true, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
 411         } {
 412                 diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error {
 413                         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 414                         defer cancel()
 415                         if trial.needcoll && collection.UUID == "" {
 416                                 return fmt.Errorf("skipping, no test collection")
 417                         }
 418                         req, err := http.NewRequestWithContext(ctx, "GET", trial.fileurl, nil)
 419                         if err != nil {
 420                                 return err
 421                         }
 422                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 423                         resp, err := http.DefaultClient.Do(req)
 424                         if err != nil {
 425                                 return err
 426                         }
 427                         defer resp.Body.Close()
 428                         body, err := ioutil.ReadAll(resp.Body)
 429                         if err != nil {
 430                                 return fmt.Errorf("reading response: %s", err)
 431                         }
 432                         if resp.StatusCode != trial.status {
 433                                 return fmt.Errorf("unexpected response status: %s", resp.Status)
 434                         }
 435                         if trial.status == http.StatusOK && string(body) != "testfiledata" {
 436                                 return fmt.Errorf("unexpected response content: %q", body)
 437                         }
 438                         return nil
 439                 })
 440         }
 441
 442         var vm arvados.VirtualMachine
 443         diag.dotest(130, "getting list of virtual machines", func() error {
 444                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 445                 defer cancel()
 446                 var vmlist arvados.VirtualMachineList
 447                 err := client.RequestAndDecodeContext(ctx, &vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
 448                 if err != nil {
 449                         return err
 450                 }
 451                 if len(vmlist.Items) < 1 {
 452                         return fmt.Errorf("no VMs found")
 453                 }
 454                 vm = vmlist.Items[0]
 455                 return nil
 456         })
 457
 458         diag.dotest(140, "getting workbench1 webshell page", func() error {
 459                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 460                 defer cancel()
 461                 if vm.UUID == "" {
 462                         return fmt.Errorf("skipping, no vm available")
 463                 }
 464                 webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername"
 465                 diag.debugf("url %s", webshelltermurl)
 466                 req, err := http.NewRequestWithContext(ctx, "GET", webshelltermurl, nil)
 467                 if err != nil {
 468                         return err
 469                 }
 470                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 471                 resp, err := http.DefaultClient.Do(req)
 472                 if err != nil {
 473                         return err
 474                 }
 475                 defer resp.Body.Close()
 476                 body, err := ioutil.ReadAll(resp.Body)
 477                 if err != nil {
 478                         return fmt.Errorf("reading response: %s", err)
 479                 }
 480                 if resp.StatusCode != http.StatusOK {
 481                         return fmt.Errorf("unexpected response status: %s %q", resp.Status, body)
 482                 }
 483                 return nil
 484         })
 485
 486         diag.dotest(150, "connecting to webshell service", func() error {
 487                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 488                 defer cancel()
 489                 if vm.UUID == "" {
 490                         return fmt.Errorf("skipping, no vm available")
 491                 }
 492                 u := cluster.Services.WebShell.ExternalURL
 493                 webshellurl := u.String() + vm.Hostname + "?"
 494                 if strings.HasPrefix(u.Host, "*") {
 495                         u.Host = vm.Hostname + u.Host[1:]
 496                         webshellurl = u.String() + "?"
 497                 }
 498                 diag.debugf("url %s", webshellurl)
 499                 req, err := http.NewRequestWithContext(ctx, "POST", webshellurl, bytes.NewBufferString(url.Values{
 500                         "width":   {"80"},
 501                         "height":  {"25"},
 502                         "session": {"xyzzy"},
 503                         "rooturl": {webshellurl},
 504                 }.Encode()))
 505                 if err != nil {
 506                         return err
 507                 }
 508                 req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
 509                 resp, err := http.DefaultClient.Do(req)
 510                 if err != nil {
 511                         return err
 512                 }
 513                 defer resp.Body.Close()
 514                 diag.debugf("response status %s", resp.Status)
 515                 body, err := ioutil.ReadAll(resp.Body)
 516                 if err != nil {
 517                         return fmt.Errorf("reading response: %s", err)
 518                 }
 519                 diag.debugf("response body %q", body)
 520                 // We don't speak the protocol, so we get a 400 error
 521                 // from the webshell server even if everything is
 522                 // OK. Anything else (404, 502, ???) indicates a
 523                 // problem.
 524                 if resp.StatusCode != http.StatusBadRequest {
 525                         return fmt.Errorf("unexpected response status: %s, %q", resp.Status, body)
 526                 }
 527                 return nil
 528         })
 529
 530         diag.dotest(160, "running a container", func() error {
 531                 if diag.priority < 1 {
 532                         diag.debugf("skipping, caller requested priority<1 (%d)", diag.priority)
 533                         return nil
 534                 }
 535
 536                 var cr arvados.ContainerRequest
 537                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 538                 defer cancel()
 539
 540                 timestamp := time.Now().Format(time.RFC3339)
 541                 err := client.RequestAndDecodeContext(ctx, &cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{
 542                         "owner_uuid":      project.UUID,
 543                         "name":            fmt.Sprintf("diagnostics container request %s", timestamp),
 544                         "container_image": "arvados/jobs",
 545                         "command":         []string{"echo", timestamp},
 546                         "use_existing":    false,
 547                         "output_path":     "/mnt/output",
 548                         "output_name":     fmt.Sprintf("diagnostics output %s", timestamp),
 549                         "priority":        diag.priority,
 550                         "state":           arvados.ContainerRequestStateCommitted,
 551                         "mounts": map[string]map[string]interface{}{
 552                                 "/mnt/output": {
 553                                         "kind":     "collection",
 554                                         "writable": true,
 555                                 },
 556                         },
 557                         "runtime_constraints": arvados.RuntimeConstraints{
 558                                 VCPUs:        1,
 559                                 RAM:          1 << 26,
 560                                 KeepCacheRAM: 1 << 26,
 561                         },
 562                 }})
 563                 if err != nil {
 564                         return err
 565                 }
 566                 diag.debugf("container request uuid = %s", cr.UUID)
 567                 diag.debugf("container uuid = %s", cr.ContainerUUID)
 568
 569                 timeout := 10 * time.Minute
 570                 diag.infof("container request submitted, waiting up to %v for container to run", arvados.Duration(timeout))
 571                 ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(timeout))
 572                 defer cancel()
 573
 574                 var c arvados.Container
 575                 for ; cr.State != arvados.ContainerRequestStateFinal; time.Sleep(2 * time.Second) {
 576                         ctx, cancel := context.WithDeadline(ctx, time.Now().Add(diag.timeout))
 577                         defer cancel()
 578
 579                         crStateWas := cr.State
 580                         err := client.RequestAndDecodeContext(ctx, &cr, "GET", "arvados/v1/container_requests/"+cr.UUID, nil, nil)
 581                         if err != nil {
 582                                 return err
 583                         }
 584                         if cr.State != crStateWas {
 585                                 diag.debugf("container request state = %s", cr.State)
 586                         }
 587
 588                         cStateWas := c.State
 589                         err = client.RequestAndDecodeContext(ctx, &c, "GET", "arvados/v1/containers/"+cr.ContainerUUID, nil, nil)
 590                         if err != nil {
 591                                 return err
 592                         }
 593                         if c.State != cStateWas {
 594                                 diag.debugf("container state = %s", c.State)
 595                         }
 596                 }
 597
 598                 if c.State != arvados.ContainerStateComplete {
 599                         return fmt.Errorf("container request %s is final but container %s did not complete: container state = %q", cr.UUID, cr.ContainerUUID, c.State)
 600                 } else if c.ExitCode != 0 {
 601                         return fmt.Errorf("container exited %d", c.ExitCode)
 602                 }
 603                 return nil
 604         })
 605 }