lib/diagnostics/cmd.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package diagnostics
   6
   7 import (
   8         "archive/tar"
   9         "bytes"
  10         "context"
  11         _ "embed"
  12         "flag"
  13         "fmt"
  14         "io"
  15         "io/ioutil"
  16         "net"
  17         "net/http"
  18         "net/url"
  19         "strings"
  20         "time"
  21
  22         "git.arvados.org/arvados.git/lib/cmd"
  23         "git.arvados.org/arvados.git/sdk/go/arvados"
  24         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  25         "github.com/sirupsen/logrus"
  26 )
  27
  28 type Command struct{}
  29
  30 func (Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
  31         var diag diagnoser
  32         f := flag.NewFlagSet(prog, flag.ContinueOnError)
  33         f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects")
  34         f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
  35         f.StringVar(&diag.dockerImage, "docker-image", "", "image to use when running a test container (default: use embedded hello-world image)")
  36         f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client")
  37         f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client")
  38         f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000, or 0 to skip)")
  39         f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests")
  40         if ok, code := cmd.ParseFlags(f, prog, args, "", stderr); !ok {
  41                 return code
  42         }
  43         diag.logger = ctxlog.New(stdout, "text", diag.logLevel)
  44         diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true, PadLevelText: true})
  45         diag.runtests()
  46         if len(diag.errors) == 0 {
  47                 diag.logger.Info("--- no errors ---")
  48                 return 0
  49         } else {
  50                 if diag.logger.Level > logrus.ErrorLevel {
  51                         fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n")
  52                         for _, e := range diag.errors {
  53                                 diag.logger.Error(e)
  54                         }
  55                 }
  56                 return 1
  57         }
  58 }
  59
  60 // docker save hello-world > hello-world.tar
  61 //go:embed hello-world.tar
  62 var HelloWorldDockerImage []byte
  63
  64 type diagnoser struct {
  65         stdout        io.Writer
  66         stderr        io.Writer
  67         logLevel      string
  68         priority      int
  69         projectName   string
  70         dockerImage   string
  71         checkInternal bool
  72         checkExternal bool
  73         timeout       time.Duration
  74         logger        *logrus.Logger
  75         errors        []string
  76         done          map[int]bool
  77 }
  78
  79 func (diag *diagnoser) debugf(f string, args ...interface{}) {
  80         diag.logger.Debugf("  ... "+f, args...)
  81 }
  82
  83 func (diag *diagnoser) infof(f string, args ...interface{}) {
  84         diag.logger.Infof("  ... "+f, args...)
  85 }
  86
  87 func (diag *diagnoser) warnf(f string, args ...interface{}) {
  88         diag.logger.Warnf("  ... "+f, args...)
  89 }
  90
  91 func (diag *diagnoser) errorf(f string, args ...interface{}) {
  92         diag.logger.Errorf(f, args...)
  93         diag.errors = append(diag.errors, fmt.Sprintf(f, args...))
  94 }
  95
  96 // Run the given func, logging appropriate messages before and after,
  97 // adding timing info, etc.
  98 //
  99 // The id argument should be unique among tests, and shouldn't change
 100 // when other tests are added/removed.
 101 func (diag *diagnoser) dotest(id int, title string, fn func() error) {
 102         if diag.done == nil {
 103                 diag.done = map[int]bool{}
 104         } else if diag.done[id] {
 105                 diag.errorf("(bug) reused test id %d", id)
 106         }
 107         diag.done[id] = true
 108
 109         diag.logger.Infof("%4d: %s", id, title)
 110         t0 := time.Now()
 111         err := fn()
 112         elapsed := fmt.Sprintf("%d ms", time.Now().Sub(t0)/time.Millisecond)
 113         if err != nil {
 114                 diag.errorf("%4d: %s (%s): %s", id, title, elapsed, err)
 115         } else {
 116                 diag.logger.Debugf("%4d: %s (%s): ok", id, title, elapsed)
 117         }
 118 }
 119
 120 func (diag *diagnoser) runtests() {
 121         client := arvados.NewClientFromEnv()
 122
 123         if client.APIHost == "" || client.AuthToken == "" {
 124                 diag.errorf("ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables are not set -- aborting without running any tests")
 125                 return
 126         }
 127
 128         var dd arvados.DiscoveryDocument
 129         ddpath := "discovery/v1/apis/arvados/v1/rest"
 130         diag.dotest(10, fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath), func() error {
 131                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 132                 defer cancel()
 133                 err := client.RequestAndDecodeContext(ctx, &dd, "GET", ddpath, nil, nil)
 134                 if err != nil {
 135                         return err
 136                 }
 137                 diag.debugf("BlobSignatureTTL = %d", dd.BlobSignatureTTL)
 138                 return nil
 139         })
 140
 141         var cluster arvados.Cluster
 142         cfgpath := "arvados/v1/config"
 143         cfgOK := false
 144         diag.dotest(20, fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath), func() error {
 145                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 146                 defer cancel()
 147                 err := client.RequestAndDecodeContext(ctx, &cluster, "GET", cfgpath, nil, nil)
 148                 if err != nil {
 149                         return err
 150                 }
 151                 diag.debugf("Collections.BlobSigning = %v", cluster.Collections.BlobSigning)
 152                 cfgOK = true
 153                 return nil
 154         })
 155
 156         var user arvados.User
 157         diag.dotest(30, "getting current user record", func() error {
 158                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 159                 defer cancel()
 160                 err := client.RequestAndDecodeContext(ctx, &user, "GET", "arvados/v1/users/current", nil, nil)
 161                 if err != nil {
 162                         return err
 163                 }
 164                 diag.debugf("user uuid = %s", user.UUID)
 165                 return nil
 166         })
 167
 168         if !cfgOK {
 169                 diag.errorf("cannot proceed without cluster config -- aborting without running any further tests")
 170                 return
 171         }
 172
 173         // uncomment to create some spurious errors
 174         // cluster.Services.WebDAVDownload.ExternalURL.Host = "0.0.0.0:9"
 175
 176         // TODO: detect routing errors here, like finding wb2 at the
 177         // wb1 address.
 178         for i, svc := range []*arvados.Service{
 179                 &cluster.Services.Keepproxy,
 180                 &cluster.Services.WebDAV,
 181                 &cluster.Services.WebDAVDownload,
 182                 &cluster.Services.Websocket,
 183                 &cluster.Services.Workbench1,
 184                 &cluster.Services.Workbench2,
 185         } {
 186                 diag.dotest(40+i, fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL), func() error {
 187                         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 188                         defer cancel()
 189                         u := svc.ExternalURL
 190                         if strings.HasPrefix(u.Scheme, "ws") {
 191                                 // We can do a real websocket test elsewhere,
 192                                 // but for now we'll just check the https
 193                                 // connection.
 194                                 u.Scheme = "http" + u.Scheme[2:]
 195                         }
 196                         if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
 197                                 u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
 198                         }
 199                         req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
 200                         if err != nil {
 201                                 return err
 202                         }
 203                         resp, err := http.DefaultClient.Do(req)
 204                         if err != nil {
 205                                 return err
 206                         }
 207                         resp.Body.Close()
 208                         return nil
 209                 })
 210         }
 211
 212         for i, url := range []string{
 213                 cluster.Services.Controller.ExternalURL.String(),
 214                 cluster.Services.Keepproxy.ExternalURL.String() + "d41d8cd98f00b204e9800998ecf8427e+0",
 215                 cluster.Services.WebDAVDownload.ExternalURL.String(),
 216         } {
 217                 diag.dotest(50+i, fmt.Sprintf("checking CORS headers at %s", url), func() error {
 218                         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 219                         defer cancel()
 220                         req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 221                         if err != nil {
 222                                 return err
 223                         }
 224                         req.Header.Set("Origin", "https://example.com")
 225                         resp, err := http.DefaultClient.Do(req)
 226                         if err != nil {
 227                                 return err
 228                         }
 229                         if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" {
 230                                 return fmt.Errorf("expected \"Access-Control-Allow-Origin: *\", got %q", hdr)
 231                         }
 232                         return nil
 233                 })
 234         }
 235
 236         var keeplist arvados.KeepServiceList
 237         diag.dotest(60, "checking internal/external client detection", func() error {
 238                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 239                 defer cancel()
 240                 err := client.RequestAndDecodeContext(ctx, &keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: 999999})
 241                 if err != nil {
 242                         return fmt.Errorf("error getting keep services list: %s", err)
 243                 } else if len(keeplist.Items) == 0 {
 244                         return fmt.Errorf("controller did not return any keep services")
 245                 }
 246                 found := map[string]int{}
 247                 for _, ks := range keeplist.Items {
 248                         found[ks.ServiceType]++
 249                 }
 250                 isInternal := found["proxy"] == 0 && len(keeplist.Items) > 0
 251                 isExternal := found["proxy"] > 0 && found["proxy"] == len(keeplist.Items)
 252                 if isExternal {
 253                         diag.debugf("controller returned only proxy services, this host is treated as \"external\"")
 254                 } else if isInternal {
 255                         diag.debugf("controller returned only non-proxy services, this host is treated as \"internal\"")
 256                 }
 257                 if (diag.checkInternal && !isInternal) || (diag.checkExternal && !isExternal) {
 258                         return fmt.Errorf("expecting internal=%v external=%v, but found internal=%v external=%v", diag.checkInternal, diag.checkExternal, isInternal, isExternal)
 259                 }
 260                 return nil
 261         })
 262
 263         for i, ks := range keeplist.Items {
 264                 u := url.URL{
 265                         Scheme: "http",
 266                         Host:   net.JoinHostPort(ks.ServiceHost, fmt.Sprintf("%d", ks.ServicePort)),
 267                         Path:   "/",
 268                 }
 269                 if ks.ServiceSSLFlag {
 270                         u.Scheme = "https"
 271                 }
 272                 diag.dotest(61+i, fmt.Sprintf("reading+writing via keep service at %s", u.String()), func() error {
 273                         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 274                         defer cancel()
 275                         req, err := http.NewRequestWithContext(ctx, "PUT", u.String()+"d41d8cd98f00b204e9800998ecf8427e", nil)
 276                         if err != nil {
 277                                 return err
 278                         }
 279                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 280                         resp, err := http.DefaultClient.Do(req)
 281                         if err != nil {
 282                                 return err
 283                         }
 284                         defer resp.Body.Close()
 285                         body, err := ioutil.ReadAll(resp.Body)
 286                         if err != nil {
 287                                 return fmt.Errorf("reading response body: %s", err)
 288                         }
 289                         loc := strings.TrimSpace(string(body))
 290                         if !strings.HasPrefix(loc, "d41d8") {
 291                                 return fmt.Errorf("unexpected response from write: %q", body)
 292                         }
 293
 294                         req, err = http.NewRequestWithContext(ctx, "GET", u.String()+loc, nil)
 295                         if err != nil {
 296                                 return err
 297                         }
 298                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 299                         resp, err = http.DefaultClient.Do(req)
 300                         if err != nil {
 301                                 return err
 302                         }
 303                         defer resp.Body.Close()
 304                         body, err = ioutil.ReadAll(resp.Body)
 305                         if err != nil {
 306                                 return fmt.Errorf("reading response body: %s", err)
 307                         }
 308                         if len(body) != 0 {
 309                                 return fmt.Errorf("unexpected response from read: %q", body)
 310                         }
 311
 312                         return nil
 313                 })
 314         }
 315
 316         var project arvados.Group
 317         diag.dotest(80, fmt.Sprintf("finding/creating %q project", diag.projectName), func() error {
 318                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 319                 defer cancel()
 320                 var grplist arvados.GroupList
 321                 err := client.RequestAndDecodeContext(ctx, &grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
 322                         Filters: []arvados.Filter{
 323                                 {"name", "=", diag.projectName},
 324                                 {"group_class", "=", "project"},
 325                                 {"owner_uuid", "=", user.UUID}},
 326                         Limit: 999999})
 327                 if err != nil {
 328                         return fmt.Errorf("list groups: %s", err)
 329                 }
 330                 if len(grplist.Items) > 0 {
 331                         project = grplist.Items[0]
 332                         diag.debugf("using existing project, uuid = %s", project.UUID)
 333                         return nil
 334                 }
 335                 diag.debugf("list groups: ok, no results")
 336                 err = client.RequestAndDecodeContext(ctx, &project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
 337                         "name":        diag.projectName,
 338                         "group_class": "project",
 339                 }})
 340                 if err != nil {
 341                         return fmt.Errorf("create project: %s", err)
 342                 }
 343                 diag.debugf("created project, uuid = %s", project.UUID)
 344                 return nil
 345         })
 346
 347         var collection arvados.Collection
 348         diag.dotest(90, "creating temporary collection", func() error {
 349                 if project.UUID == "" {
 350                         return fmt.Errorf("skipping, no project to work in")
 351                 }
 352                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 353                 defer cancel()
 354                 err := client.RequestAndDecodeContext(ctx, &collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 355                         "ensure_unique_name": true,
 356                         "collection": map[string]interface{}{
 357                                 "owner_uuid": project.UUID,
 358                                 "name":       "test collection",
 359                                 "trash_at":   time.Now().Add(time.Hour)}})
 360                 if err != nil {
 361                         return err
 362                 }
 363                 diag.debugf("ok, uuid = %s", collection.UUID)
 364                 return nil
 365         })
 366
 367         if collection.UUID != "" {
 368                 defer func() {
 369                         diag.dotest(9990, "deleting temporary collection", func() error {
 370                                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 371                                 defer cancel()
 372                                 return client.RequestAndDecodeContext(ctx, nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
 373                         })
 374                 }()
 375         }
 376
 377         // Read hello-world.tar to find image ID, so we can upload it
 378         // as "sha256:{...}.tar"
 379         var imageSHA2 string
 380         {
 381                 tr := tar.NewReader(bytes.NewReader(HelloWorldDockerImage))
 382                 for {
 383                         hdr, err := tr.Next()
 384                         if err == io.EOF {
 385                                 break
 386                         }
 387                         if err != nil {
 388                                 diag.errorf("internal error/bug: cannot read embedded docker image tar file: %s", err)
 389                                 return
 390                         }
 391                         if s := strings.TrimSuffix(hdr.Name, ".json"); len(s) == 64 && s != hdr.Name {
 392                                 imageSHA2 = s
 393                         }
 394                 }
 395                 if imageSHA2 == "" {
 396                         diag.errorf("internal error/bug: cannot find {sha256}.json file in embedded docker image tar file")
 397                         return
 398                 }
 399         }
 400
 401         diag.dotest(100, "uploading file via webdav", func() error {
 402                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 403                 defer cancel()
 404                 if collection.UUID == "" {
 405                         return fmt.Errorf("skipping, no test collection")
 406                 }
 407                 req, err := http.NewRequestWithContext(ctx, "PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/sha256:"+imageSHA2+".tar", bytes.NewReader(HelloWorldDockerImage))
 408                 if err != nil {
 409                         return fmt.Errorf("BUG? http.NewRequest: %s", err)
 410                 }
 411                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 412                 resp, err := http.DefaultClient.Do(req)
 413                 if err != nil {
 414                         return fmt.Errorf("error performing http request: %s", err)
 415                 }
 416                 resp.Body.Close()
 417                 if resp.StatusCode != http.StatusCreated {
 418                         return fmt.Errorf("status %s", resp.Status)
 419                 }
 420                 diag.debugf("ok, status %s", resp.Status)
 421                 err = client.RequestAndDecodeContext(ctx, &collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
 422                 if err != nil {
 423                         return fmt.Errorf("get updated collection: %s", err)
 424                 }
 425                 diag.debugf("ok, pdh %s", collection.PortableDataHash)
 426                 return nil
 427         })
 428
 429         davurl := cluster.Services.WebDAV.ExternalURL
 430         davWildcard := strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.")
 431         diag.dotest(110, fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl), func() error {
 432                 if davurl.Host == "" {
 433                         return fmt.Errorf("host missing - content previews will not work")
 434                 }
 435                 if !davWildcard && !cluster.Collections.TrustAllContent {
 436                         diag.warnf("WebDAV ExternalURL has no leading wildcard and TrustAllContent==false - content previews will not work")
 437                 }
 438                 return nil
 439         })
 440
 441         for i, trial := range []struct {
 442                 needcoll     bool
 443                 needWildcard bool
 444                 status       int
 445                 fileurl      string
 446         }{
 447                 {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
 448                 {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"},
 449                 {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
 450                 {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"},
 451                 {true, true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"},
 452                 {true, false, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/sha256:" + imageSHA2 + ".tar"},
 453         } {
 454                 diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error {
 455                         if trial.needWildcard && !davWildcard {
 456                                 diag.warnf("skipping collection-id-in-vhost test because WebDAV ExternalURL has no leading wildcard")
 457                                 return nil
 458                         }
 459                         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 460                         defer cancel()
 461                         if trial.needcoll && collection.UUID == "" {
 462                                 return fmt.Errorf("skipping, no test collection")
 463                         }
 464                         req, err := http.NewRequestWithContext(ctx, "GET", trial.fileurl, nil)
 465                         if err != nil {
 466                                 return err
 467                         }
 468                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 469                         resp, err := http.DefaultClient.Do(req)
 470                         if err != nil {
 471                                 return err
 472                         }
 473                         defer resp.Body.Close()
 474                         body, err := ioutil.ReadAll(resp.Body)
 475                         if err != nil {
 476                                 return fmt.Errorf("reading response: %s", err)
 477                         }
 478                         if resp.StatusCode != trial.status {
 479                                 return fmt.Errorf("unexpected response status: %s", resp.Status)
 480                         }
 481                         if trial.status == http.StatusOK && !bytes.Equal(body, HelloWorldDockerImage) {
 482                                 excerpt := body
 483                                 if len(excerpt) > 128 {
 484                                         excerpt = append([]byte(nil), body[:128]...)
 485                                         excerpt = append(excerpt, []byte("[...]")...)
 486                                 }
 487                                 return fmt.Errorf("unexpected response content: len %d, %q", len(body), excerpt)
 488                         }
 489                         return nil
 490                 })
 491         }
 492
 493         var vm arvados.VirtualMachine
 494         diag.dotest(130, "getting list of virtual machines", func() error {
 495                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 496                 defer cancel()
 497                 var vmlist arvados.VirtualMachineList
 498                 err := client.RequestAndDecodeContext(ctx, &vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
 499                 if err != nil {
 500                         return err
 501                 }
 502                 if len(vmlist.Items) < 1 {
 503                         diag.warnf("no VMs found")
 504                 } else {
 505                         vm = vmlist.Items[0]
 506                 }
 507                 return nil
 508         })
 509
 510         diag.dotest(140, "getting workbench1 webshell page", func() error {
 511                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 512                 defer cancel()
 513                 if vm.UUID == "" {
 514                         diag.warnf("skipping, no vm available")
 515                         return nil
 516                 }
 517                 webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername"
 518                 diag.debugf("url %s", webshelltermurl)
 519                 req, err := http.NewRequestWithContext(ctx, "GET", webshelltermurl, nil)
 520                 if err != nil {
 521                         return err
 522                 }
 523                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 524                 resp, err := http.DefaultClient.Do(req)
 525                 if err != nil {
 526                         return err
 527                 }
 528                 defer resp.Body.Close()
 529                 body, err := ioutil.ReadAll(resp.Body)
 530                 if err != nil {
 531                         return fmt.Errorf("reading response: %s", err)
 532                 }
 533                 if resp.StatusCode != http.StatusOK {
 534                         return fmt.Errorf("unexpected response status: %s %q", resp.Status, body)
 535                 }
 536                 return nil
 537         })
 538
 539         diag.dotest(150, "connecting to webshell service", func() error {
 540                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 541                 defer cancel()
 542                 if vm.UUID == "" {
 543                         diag.warnf("skipping, no vm available")
 544                         return nil
 545                 }
 546                 u := cluster.Services.WebShell.ExternalURL
 547                 webshellurl := u.String() + vm.Hostname + "?"
 548                 if strings.HasPrefix(u.Host, "*") {
 549                         u.Host = vm.Hostname + u.Host[1:]
 550                         webshellurl = u.String() + "?"
 551                 }
 552                 diag.debugf("url %s", webshellurl)
 553                 req, err := http.NewRequestWithContext(ctx, "POST", webshellurl, bytes.NewBufferString(url.Values{
 554                         "width":   {"80"},
 555                         "height":  {"25"},
 556                         "session": {"xyzzy"},
 557                         "rooturl": {webshellurl},
 558                 }.Encode()))
 559                 if err != nil {
 560                         return err
 561                 }
 562                 req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
 563                 resp, err := http.DefaultClient.Do(req)
 564                 if err != nil {
 565                         return err
 566                 }
 567                 defer resp.Body.Close()
 568                 diag.debugf("response status %s", resp.Status)
 569                 body, err := ioutil.ReadAll(resp.Body)
 570                 if err != nil {
 571                         return fmt.Errorf("reading response: %s", err)
 572                 }
 573                 diag.debugf("response body %q", body)
 574                 // We don't speak the protocol, so we get a 400 error
 575                 // from the webshell server even if everything is
 576                 // OK. Anything else (404, 502, ???) indicates a
 577                 // problem.
 578                 if resp.StatusCode != http.StatusBadRequest {
 579                         return fmt.Errorf("unexpected response status: %s, %q", resp.Status, body)
 580                 }
 581                 return nil
 582         })
 583
 584         diag.dotest(160, "running a container", func() error {
 585                 if diag.priority < 1 {
 586                         diag.infof("skipping (use priority > 0 if you want to run a container)")
 587                         return nil
 588                 }
 589                 if project.UUID == "" {
 590                         return fmt.Errorf("skipping, no project to work in")
 591                 }
 592
 593                 timestamp := time.Now().Format(time.RFC3339)
 594                 ctrCommand := []string{"echo", timestamp}
 595                 if diag.dockerImage == "" {
 596                         if collection.UUID == "" {
 597                                 return fmt.Errorf("skipping, no test collection to use as docker image")
 598                         }
 599                         diag.dockerImage = collection.PortableDataHash
 600                         ctrCommand = []string{"/hello"}
 601                 }
 602
 603                 var cr arvados.ContainerRequest
 604                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 605                 defer cancel()
 606
 607                 err := client.RequestAndDecodeContext(ctx, &cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{
 608                         "owner_uuid":      project.UUID,
 609                         "name":            fmt.Sprintf("diagnostics container request %s", timestamp),
 610                         "container_image": diag.dockerImage,
 611                         "command":         ctrCommand,
 612                         "use_existing":    false,
 613                         "output_path":     "/mnt/output",
 614                         "output_name":     fmt.Sprintf("diagnostics output %s", timestamp),
 615                         "priority":        diag.priority,
 616                         "state":           arvados.ContainerRequestStateCommitted,
 617                         "mounts": map[string]map[string]interface{}{
 618                                 "/mnt/output": {
 619                                         "kind":     "collection",
 620                                         "writable": true,
 621                                 },
 622                         },
 623                         "runtime_constraints": arvados.RuntimeConstraints{
 624                                 VCPUs:        1,
 625                                 RAM:          1 << 26,
 626                                 KeepCacheRAM: 1 << 26,
 627                         },
 628                 }})
 629                 if err != nil {
 630                         return err
 631                 }
 632                 diag.debugf("container request uuid = %s", cr.UUID)
 633                 diag.debugf("container uuid = %s", cr.ContainerUUID)
 634
 635                 timeout := 10 * time.Minute
 636                 diag.infof("container request submitted, waiting up to %v for container to run", arvados.Duration(timeout))
 637                 ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(timeout))
 638                 defer cancel()
 639
 640                 var c arvados.Container
 641                 for ; cr.State != arvados.ContainerRequestStateFinal; time.Sleep(2 * time.Second) {
 642                         ctx, cancel := context.WithDeadline(ctx, time.Now().Add(diag.timeout))
 643                         defer cancel()
 644
 645                         crStateWas := cr.State
 646                         err := client.RequestAndDecodeContext(ctx, &cr, "GET", "arvados/v1/container_requests/"+cr.UUID, nil, nil)
 647                         if err != nil {
 648                                 return err
 649                         }
 650                         if cr.State != crStateWas {
 651                                 diag.debugf("container request state = %s", cr.State)
 652                         }
 653
 654                         cStateWas := c.State
 655                         err = client.RequestAndDecodeContext(ctx, &c, "GET", "arvados/v1/containers/"+cr.ContainerUUID, nil, nil)
 656                         if err != nil {
 657                                 return err
 658                         }
 659                         if c.State != cStateWas {
 660                                 diag.debugf("container state = %s", c.State)
 661                         }
 662                 }
 663
 664                 if c.State != arvados.ContainerStateComplete {
 665                         return fmt.Errorf("container request %s is final but container %s did not complete: container state = %q", cr.UUID, cr.ContainerUUID, c.State)
 666                 } else if c.ExitCode != 0 {
 667                         return fmt.Errorf("container exited %d", c.ExitCode)
 668                 }
 669                 return nil
 670         })
 671 }