lib/diagnostics/cmd.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package diagnostics
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "flag"
  11         "fmt"
  12         "io"
  13         "io/ioutil"
  14         "net"
  15         "net/http"
  16         "net/url"
  17         "strings"
  18         "time"
  19
  20         "git.arvados.org/arvados.git/lib/cmd"
  21         "git.arvados.org/arvados.git/sdk/go/arvados"
  22         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  23         "github.com/sirupsen/logrus"
  24 )
  25
  26 type Command struct{}
  27
  28 func (Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
  29         var diag diagnoser
  30         f := flag.NewFlagSet(prog, flag.ContinueOnError)
  31         f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects")
  32         f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
  33         f.StringVar(&diag.dockerImage, "docker-image", "alpine:latest", "image to use when running a test container")
  34         f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client")
  35         f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client")
  36         f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000, or 0 to skip)")
  37         f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests")
  38         if ok, code := cmd.ParseFlags(f, prog, args, "", stderr); !ok {
  39                 return code
  40         }
  41         diag.logger = ctxlog.New(stdout, "text", diag.logLevel)
  42         diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true, PadLevelText: true})
  43         diag.runtests()
  44         if len(diag.errors) == 0 {
  45                 diag.logger.Info("--- no errors ---")
  46                 return 0
  47         } else {
  48                 if diag.logger.Level > logrus.ErrorLevel {
  49                         fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n")
  50                         for _, e := range diag.errors {
  51                                 diag.logger.Error(e)
  52                         }
  53                 }
  54                 return 1
  55         }
  56 }
  57
  58 type diagnoser struct {
  59         stdout        io.Writer
  60         stderr        io.Writer
  61         logLevel      string
  62         priority      int
  63         projectName   string
  64         dockerImage   string
  65         checkInternal bool
  66         checkExternal bool
  67         timeout       time.Duration
  68         logger        *logrus.Logger
  69         errors        []string
  70         done          map[int]bool
  71 }
  72
  73 func (diag *diagnoser) debugf(f string, args ...interface{}) {
  74         diag.logger.Debugf("  ... "+f, args...)
  75 }
  76
  77 func (diag *diagnoser) infof(f string, args ...interface{}) {
  78         diag.logger.Infof("  ... "+f, args...)
  79 }
  80
  81 func (diag *diagnoser) warnf(f string, args ...interface{}) {
  82         diag.logger.Warnf("  ... "+f, args...)
  83 }
  84
  85 func (diag *diagnoser) errorf(f string, args ...interface{}) {
  86         diag.logger.Errorf(f, args...)
  87         diag.errors = append(diag.errors, fmt.Sprintf(f, args...))
  88 }
  89
  90 // Run the given func, logging appropriate messages before and after,
  91 // adding timing info, etc.
  92 //
  93 // The id argument should be unique among tests, and shouldn't change
  94 // when other tests are added/removed.
  95 func (diag *diagnoser) dotest(id int, title string, fn func() error) {
  96         if diag.done == nil {
  97                 diag.done = map[int]bool{}
  98         } else if diag.done[id] {
  99                 diag.errorf("(bug) reused test id %d", id)
 100         }
 101         diag.done[id] = true
 102
 103         diag.logger.Infof("%4d: %s", id, title)
 104         t0 := time.Now()
 105         err := fn()
 106         elapsed := fmt.Sprintf("%d ms", time.Now().Sub(t0)/time.Millisecond)
 107         if err != nil {
 108                 diag.errorf("%4d: %s (%s): %s", id, title, elapsed, err)
 109         } else {
 110                 diag.logger.Debugf("%4d: %s (%s): ok", id, title, elapsed)
 111         }
 112 }
 113
 114 func (diag *diagnoser) runtests() {
 115         client := arvados.NewClientFromEnv()
 116
 117         if client.APIHost == "" || client.AuthToken == "" {
 118                 diag.errorf("ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables are not set -- aborting without running any tests")
 119                 return
 120         }
 121
 122         var dd arvados.DiscoveryDocument
 123         ddpath := "discovery/v1/apis/arvados/v1/rest"
 124         diag.dotest(10, fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath), func() error {
 125                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 126                 defer cancel()
 127                 err := client.RequestAndDecodeContext(ctx, &dd, "GET", ddpath, nil, nil)
 128                 if err != nil {
 129                         return err
 130                 }
 131                 diag.debugf("BlobSignatureTTL = %d", dd.BlobSignatureTTL)
 132                 return nil
 133         })
 134
 135         var cluster arvados.Cluster
 136         cfgpath := "arvados/v1/config"
 137         cfgOK := false
 138         diag.dotest(20, fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath), func() error {
 139                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 140                 defer cancel()
 141                 err := client.RequestAndDecodeContext(ctx, &cluster, "GET", cfgpath, nil, nil)
 142                 if err != nil {
 143                         return err
 144                 }
 145                 diag.debugf("Collections.BlobSigning = %v", cluster.Collections.BlobSigning)
 146                 cfgOK = true
 147                 return nil
 148         })
 149
 150         var user arvados.User
 151         diag.dotest(30, "getting current user record", func() error {
 152                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 153                 defer cancel()
 154                 err := client.RequestAndDecodeContext(ctx, &user, "GET", "arvados/v1/users/current", nil, nil)
 155                 if err != nil {
 156                         return err
 157                 }
 158                 diag.debugf("user uuid = %s", user.UUID)
 159                 return nil
 160         })
 161
 162         if !cfgOK {
 163                 diag.errorf("cannot proceed without cluster config -- aborting without running any further tests")
 164                 return
 165         }
 166
 167         // uncomment to create some spurious errors
 168         // cluster.Services.WebDAVDownload.ExternalURL.Host = "0.0.0.0:9"
 169
 170         // TODO: detect routing errors here, like finding wb2 at the
 171         // wb1 address.
 172         for i, svc := range []*arvados.Service{
 173                 &cluster.Services.Keepproxy,
 174                 &cluster.Services.WebDAV,
 175                 &cluster.Services.WebDAVDownload,
 176                 &cluster.Services.Websocket,
 177                 &cluster.Services.Workbench1,
 178                 &cluster.Services.Workbench2,
 179         } {
 180                 diag.dotest(40+i, fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL), func() error {
 181                         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 182                         defer cancel()
 183                         u := svc.ExternalURL
 184                         if strings.HasPrefix(u.Scheme, "ws") {
 185                                 // We can do a real websocket test elsewhere,
 186                                 // but for now we'll just check the https
 187                                 // connection.
 188                                 u.Scheme = "http" + u.Scheme[2:]
 189                         }
 190                         if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
 191                                 u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
 192                         }
 193                         req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
 194                         if err != nil {
 195                                 return err
 196                         }
 197                         resp, err := http.DefaultClient.Do(req)
 198                         if err != nil {
 199                                 return err
 200                         }
 201                         resp.Body.Close()
 202                         return nil
 203                 })
 204         }
 205
 206         for i, url := range []string{
 207                 cluster.Services.Controller.ExternalURL.String(),
 208                 cluster.Services.Keepproxy.ExternalURL.String() + "d41d8cd98f00b204e9800998ecf8427e+0",
 209                 cluster.Services.WebDAVDownload.ExternalURL.String(),
 210         } {
 211                 diag.dotest(50+i, fmt.Sprintf("checking CORS headers at %s", url), func() error {
 212                         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 213                         defer cancel()
 214                         req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 215                         if err != nil {
 216                                 return err
 217                         }
 218                         req.Header.Set("Origin", "https://example.com")
 219                         resp, err := http.DefaultClient.Do(req)
 220                         if err != nil {
 221                                 return err
 222                         }
 223                         if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" {
 224                                 return fmt.Errorf("expected \"Access-Control-Allow-Origin: *\", got %q", hdr)
 225                         }
 226                         return nil
 227                 })
 228         }
 229
 230         var keeplist arvados.KeepServiceList
 231         diag.dotest(60, "checking internal/external client detection", func() error {
 232                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 233                 defer cancel()
 234                 err := client.RequestAndDecodeContext(ctx, &keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: 999999})
 235                 if err != nil {
 236                         return fmt.Errorf("error getting keep services list: %s", err)
 237                 } else if len(keeplist.Items) == 0 {
 238                         return fmt.Errorf("controller did not return any keep services")
 239                 }
 240                 found := map[string]int{}
 241                 for _, ks := range keeplist.Items {
 242                         found[ks.ServiceType]++
 243                 }
 244                 isInternal := found["proxy"] == 0 && len(keeplist.Items) > 0
 245                 isExternal := found["proxy"] > 0 && found["proxy"] == len(keeplist.Items)
 246                 if isExternal {
 247                         diag.debugf("controller returned only proxy services, this host is treated as \"external\"")
 248                 } else if isInternal {
 249                         diag.debugf("controller returned only non-proxy services, this host is treated as \"internal\"")
 250                 }
 251                 if (diag.checkInternal && !isInternal) || (diag.checkExternal && !isExternal) {
 252                         return fmt.Errorf("expecting internal=%v external=%v, but found internal=%v external=%v", diag.checkInternal, diag.checkExternal, isInternal, isExternal)
 253                 }
 254                 return nil
 255         })
 256
 257         for i, ks := range keeplist.Items {
 258                 u := url.URL{
 259                         Scheme: "http",
 260                         Host:   net.JoinHostPort(ks.ServiceHost, fmt.Sprintf("%d", ks.ServicePort)),
 261                         Path:   "/",
 262                 }
 263                 if ks.ServiceSSLFlag {
 264                         u.Scheme = "https"
 265                 }
 266                 diag.dotest(61+i, fmt.Sprintf("reading+writing via keep service at %s", u.String()), func() error {
 267                         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 268                         defer cancel()
 269                         req, err := http.NewRequestWithContext(ctx, "PUT", u.String()+"d41d8cd98f00b204e9800998ecf8427e", nil)
 270                         if err != nil {
 271                                 return err
 272                         }
 273                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 274                         resp, err := http.DefaultClient.Do(req)
 275                         if err != nil {
 276                                 return err
 277                         }
 278                         defer resp.Body.Close()
 279                         body, err := ioutil.ReadAll(resp.Body)
 280                         if err != nil {
 281                                 return fmt.Errorf("reading response body: %s", err)
 282                         }
 283                         loc := strings.TrimSpace(string(body))
 284                         if !strings.HasPrefix(loc, "d41d8") {
 285                                 return fmt.Errorf("unexpected response from write: %q", body)
 286                         }
 287
 288                         req, err = http.NewRequestWithContext(ctx, "GET", u.String()+loc, nil)
 289                         if err != nil {
 290                                 return err
 291                         }
 292                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 293                         resp, err = http.DefaultClient.Do(req)
 294                         if err != nil {
 295                                 return err
 296                         }
 297                         defer resp.Body.Close()
 298                         body, err = ioutil.ReadAll(resp.Body)
 299                         if err != nil {
 300                                 return fmt.Errorf("reading response body: %s", err)
 301                         }
 302                         if len(body) != 0 {
 303                                 return fmt.Errorf("unexpected response from read: %q", body)
 304                         }
 305
 306                         return nil
 307                 })
 308         }
 309
 310         var project arvados.Group
 311         diag.dotest(80, fmt.Sprintf("finding/creating %q project", diag.projectName), func() error {
 312                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 313                 defer cancel()
 314                 var grplist arvados.GroupList
 315                 err := client.RequestAndDecodeContext(ctx, &grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
 316                         Filters: []arvados.Filter{
 317                                 {"name", "=", diag.projectName},
 318                                 {"group_class", "=", "project"},
 319                                 {"owner_uuid", "=", user.UUID}},
 320                         Limit: 999999})
 321                 if err != nil {
 322                         return fmt.Errorf("list groups: %s", err)
 323                 }
 324                 if len(grplist.Items) > 0 {
 325                         project = grplist.Items[0]
 326                         diag.debugf("using existing project, uuid = %s", project.UUID)
 327                         return nil
 328                 }
 329                 diag.debugf("list groups: ok, no results")
 330                 err = client.RequestAndDecodeContext(ctx, &project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
 331                         "name":        diag.projectName,
 332                         "group_class": "project",
 333                 }})
 334                 if err != nil {
 335                         return fmt.Errorf("create project: %s", err)
 336                 }
 337                 diag.debugf("created project, uuid = %s", project.UUID)
 338                 return nil
 339         })
 340
 341         var collection arvados.Collection
 342         diag.dotest(90, "creating temporary collection", func() error {
 343                 if project.UUID == "" {
 344                         return fmt.Errorf("skipping, no project to work in")
 345                 }
 346                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 347                 defer cancel()
 348                 err := client.RequestAndDecodeContext(ctx, &collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 349                         "ensure_unique_name": true,
 350                         "collection": map[string]interface{}{
 351                                 "owner_uuid": project.UUID,
 352                                 "name":       "test collection",
 353                                 "trash_at":   time.Now().Add(time.Hour)}})
 354                 if err != nil {
 355                         return err
 356                 }
 357                 diag.debugf("ok, uuid = %s", collection.UUID)
 358                 return nil
 359         })
 360
 361         if collection.UUID != "" {
 362                 defer func() {
 363                         diag.dotest(9990, "deleting temporary collection", func() error {
 364                                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 365                                 defer cancel()
 366                                 return client.RequestAndDecodeContext(ctx, nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
 367                         })
 368                 }()
 369         }
 370
 371         diag.dotest(100, "uploading file via webdav", func() error {
 372                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 373                 defer cancel()
 374                 if collection.UUID == "" {
 375                         return fmt.Errorf("skipping, no test collection")
 376                 }
 377                 req, err := http.NewRequestWithContext(ctx, "PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata"))
 378                 if err != nil {
 379                         return fmt.Errorf("BUG? http.NewRequest: %s", err)
 380                 }
 381                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 382                 resp, err := http.DefaultClient.Do(req)
 383                 if err != nil {
 384                         return fmt.Errorf("error performing http request: %s", err)
 385                 }
 386                 resp.Body.Close()
 387                 if resp.StatusCode != http.StatusCreated {
 388                         return fmt.Errorf("status %s", resp.Status)
 389                 }
 390                 diag.debugf("ok, status %s", resp.Status)
 391                 err = client.RequestAndDecodeContext(ctx, &collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
 392                 if err != nil {
 393                         return fmt.Errorf("get updated collection: %s", err)
 394                 }
 395                 diag.debugf("ok, pdh %s", collection.PortableDataHash)
 396                 return nil
 397         })
 398
 399         davurl := cluster.Services.WebDAV.ExternalURL
 400         davWildcard := strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.")
 401         diag.dotest(110, fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl), func() error {
 402                 if davurl.Host == "" {
 403                         return fmt.Errorf("host missing - content previews will not work")
 404                 }
 405                 if !davWildcard && !cluster.Collections.TrustAllContent {
 406                         diag.warnf("WebDAV ExternalURL has no leading wildcard and TrustAllContent==false - content previews will not work")
 407                 }
 408                 return nil
 409         })
 410
 411         for i, trial := range []struct {
 412                 needcoll     bool
 413                 needWildcard bool
 414                 status       int
 415                 fileurl      string
 416         }{
 417                 {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
 418                 {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"},
 419                 {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
 420                 {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"},
 421                 {true, true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"},
 422                 {true, false, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
 423         } {
 424                 diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error {
 425                         if trial.needWildcard && !davWildcard {
 426                                 diag.warnf("skipping collection-id-in-vhost test because WebDAV ExternalURL has no leading wildcard")
 427                                 return nil
 428                         }
 429                         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 430                         defer cancel()
 431                         if trial.needcoll && collection.UUID == "" {
 432                                 return fmt.Errorf("skipping, no test collection")
 433                         }
 434                         req, err := http.NewRequestWithContext(ctx, "GET", trial.fileurl, nil)
 435                         if err != nil {
 436                                 return err
 437                         }
 438                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 439                         resp, err := http.DefaultClient.Do(req)
 440                         if err != nil {
 441                                 return err
 442                         }
 443                         defer resp.Body.Close()
 444                         body, err := ioutil.ReadAll(resp.Body)
 445                         if err != nil {
 446                                 return fmt.Errorf("reading response: %s", err)
 447                         }
 448                         if resp.StatusCode != trial.status {
 449                                 return fmt.Errorf("unexpected response status: %s", resp.Status)
 450                         }
 451                         if trial.status == http.StatusOK && string(body) != "testfiledata" {
 452                                 return fmt.Errorf("unexpected response content: %q", body)
 453                         }
 454                         return nil
 455                 })
 456         }
 457
 458         var vm arvados.VirtualMachine
 459         diag.dotest(130, "getting list of virtual machines", func() error {
 460                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 461                 defer cancel()
 462                 var vmlist arvados.VirtualMachineList
 463                 err := client.RequestAndDecodeContext(ctx, &vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
 464                 if err != nil {
 465                         return err
 466                 }
 467                 if len(vmlist.Items) < 1 {
 468                         diag.warnf("no VMs found")
 469                 } else {
 470                         vm = vmlist.Items[0]
 471                 }
 472                 return nil
 473         })
 474
 475         diag.dotest(140, "getting workbench1 webshell page", func() error {
 476                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 477                 defer cancel()
 478                 if vm.UUID == "" {
 479                         diag.warnf("skipping, no vm available")
 480                         return nil
 481                 }
 482                 webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername"
 483                 diag.debugf("url %s", webshelltermurl)
 484                 req, err := http.NewRequestWithContext(ctx, "GET", webshelltermurl, nil)
 485                 if err != nil {
 486                         return err
 487                 }
 488                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 489                 resp, err := http.DefaultClient.Do(req)
 490                 if err != nil {
 491                         return err
 492                 }
 493                 defer resp.Body.Close()
 494                 body, err := ioutil.ReadAll(resp.Body)
 495                 if err != nil {
 496                         return fmt.Errorf("reading response: %s", err)
 497                 }
 498                 if resp.StatusCode != http.StatusOK {
 499                         return fmt.Errorf("unexpected response status: %s %q", resp.Status, body)
 500                 }
 501                 return nil
 502         })
 503
 504         diag.dotest(150, "connecting to webshell service", func() error {
 505                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 506                 defer cancel()
 507                 if vm.UUID == "" {
 508                         diag.warnf("skipping, no vm available")
 509                         return nil
 510                 }
 511                 u := cluster.Services.WebShell.ExternalURL
 512                 webshellurl := u.String() + vm.Hostname + "?"
 513                 if strings.HasPrefix(u.Host, "*") {
 514                         u.Host = vm.Hostname + u.Host[1:]
 515                         webshellurl = u.String() + "?"
 516                 }
 517                 diag.debugf("url %s", webshellurl)
 518                 req, err := http.NewRequestWithContext(ctx, "POST", webshellurl, bytes.NewBufferString(url.Values{
 519                         "width":   {"80"},
 520                         "height":  {"25"},
 521                         "session": {"xyzzy"},
 522                         "rooturl": {webshellurl},
 523                 }.Encode()))
 524                 if err != nil {
 525                         return err
 526                 }
 527                 req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
 528                 resp, err := http.DefaultClient.Do(req)
 529                 if err != nil {
 530                         return err
 531                 }
 532                 defer resp.Body.Close()
 533                 diag.debugf("response status %s", resp.Status)
 534                 body, err := ioutil.ReadAll(resp.Body)
 535                 if err != nil {
 536                         return fmt.Errorf("reading response: %s", err)
 537                 }
 538                 diag.debugf("response body %q", body)
 539                 // We don't speak the protocol, so we get a 400 error
 540                 // from the webshell server even if everything is
 541                 // OK. Anything else (404, 502, ???) indicates a
 542                 // problem.
 543                 if resp.StatusCode != http.StatusBadRequest {
 544                         return fmt.Errorf("unexpected response status: %s, %q", resp.Status, body)
 545                 }
 546                 return nil
 547         })
 548
 549         diag.dotest(160, "running a container", func() error {
 550                 if diag.priority < 1 {
 551                         diag.infof("skipping (use priority > 0 if you want to run a container)")
 552                         return nil
 553                 }
 554                 if project.UUID == "" {
 555                         return fmt.Errorf("skipping, no project to work in")
 556                 }
 557
 558                 var cr arvados.ContainerRequest
 559                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 560                 defer cancel()
 561
 562                 timestamp := time.Now().Format(time.RFC3339)
 563                 err := client.RequestAndDecodeContext(ctx, &cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{
 564                         "owner_uuid":      project.UUID,
 565                         "name":            fmt.Sprintf("diagnostics container request %s", timestamp),
 566                         "container_image": diag.dockerImage,
 567                         "command":         []string{"echo", timestamp},
 568                         "use_existing":    false,
 569                         "output_path":     "/mnt/output",
 570                         "output_name":     fmt.Sprintf("diagnostics output %s", timestamp),
 571                         "priority":        diag.priority,
 572                         "state":           arvados.ContainerRequestStateCommitted,
 573                         "mounts": map[string]map[string]interface{}{
 574                                 "/mnt/output": {
 575                                         "kind":     "collection",
 576                                         "writable": true,
 577                                 },
 578                         },
 579                         "runtime_constraints": arvados.RuntimeConstraints{
 580                                 VCPUs:        1,
 581                                 RAM:          1 << 26,
 582                                 KeepCacheRAM: 1 << 26,
 583                         },
 584                 }})
 585                 if err != nil {
 586                         return err
 587                 }
 588                 diag.debugf("container request uuid = %s", cr.UUID)
 589                 diag.debugf("container uuid = %s", cr.ContainerUUID)
 590
 591                 timeout := 10 * time.Minute
 592                 diag.infof("container request submitted, waiting up to %v for container to run", arvados.Duration(timeout))
 593                 ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(timeout))
 594                 defer cancel()
 595
 596                 var c arvados.Container
 597                 for ; cr.State != arvados.ContainerRequestStateFinal; time.Sleep(2 * time.Second) {
 598                         ctx, cancel := context.WithDeadline(ctx, time.Now().Add(diag.timeout))
 599                         defer cancel()
 600
 601                         crStateWas := cr.State
 602                         err := client.RequestAndDecodeContext(ctx, &cr, "GET", "arvados/v1/container_requests/"+cr.UUID, nil, nil)
 603                         if err != nil {
 604                                 return err
 605                         }
 606                         if cr.State != crStateWas {
 607                                 diag.debugf("container request state = %s", cr.State)
 608                         }
 609
 610                         cStateWas := c.State
 611                         err = client.RequestAndDecodeContext(ctx, &c, "GET", "arvados/v1/containers/"+cr.ContainerUUID, nil, nil)
 612                         if err != nil {
 613                                 return err
 614                         }
 615                         if c.State != cStateWas {
 616                                 diag.debugf("container state = %s", c.State)
 617                         }
 618                 }
 619
 620                 if c.State != arvados.ContainerStateComplete {
 621                         return fmt.Errorf("container request %s is final but container %s did not complete: container state = %q", cr.UUID, cr.ContainerUUID, c.State)
 622                 } else if c.ExitCode != 0 {
 623                         return fmt.Errorf("container exited %d", c.ExitCode)
 624                 }
 625                 return nil
 626         })
 627 }