lib/diagnostics/cmd.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package diagnostics
   6
   7 import (
   8         "archive/tar"
   9         "bytes"
  10         "context"
  11         _ "embed"
  12         "flag"
  13         "fmt"
  14         "io"
  15         "io/ioutil"
  16         "net"
  17         "net/http"
  18         "net/url"
  19         "os"
  20         "strings"
  21         "time"
  22
  23         "git.arvados.org/arvados.git/lib/cmd"
  24         "git.arvados.org/arvados.git/lib/config"
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  27         "git.arvados.org/arvados.git/sdk/go/health"
  28         "github.com/sirupsen/logrus"
  29 )
  30
  31 type Command struct{}
  32
  33 func (Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
  34         var diag diagnoser
  35         f := flag.NewFlagSet(prog, flag.ContinueOnError)
  36         f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects")
  37         f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
  38         f.StringVar(&diag.dockerImage, "docker-image", "", "image to use when running a test container (default: use embedded hello-world image)")
  39         f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client")
  40         f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client")
  41         f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000, or 0 to skip)")
  42         f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests")
  43         if ok, code := cmd.ParseFlags(f, prog, args, "", stderr); !ok {
  44                 return code
  45         }
  46         diag.logger = ctxlog.New(stdout, "text", diag.logLevel)
  47         diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true, PadLevelText: true})
  48         diag.runtests()
  49         if len(diag.errors) == 0 {
  50                 diag.logger.Info("--- no errors ---")
  51                 return 0
  52         } else {
  53                 if diag.logger.Level > logrus.ErrorLevel {
  54                         fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n")
  55                         for _, e := range diag.errors {
  56                                 diag.logger.Error(e)
  57                         }
  58                 }
  59                 return 1
  60         }
  61 }
  62
  63 // docker save hello-world > hello-world.tar
  64 //go:embed hello-world.tar
  65 var HelloWorldDockerImage []byte
  66
  67 type diagnoser struct {
  68         stdout        io.Writer
  69         stderr        io.Writer
  70         logLevel      string
  71         priority      int
  72         projectName   string
  73         dockerImage   string
  74         checkInternal bool
  75         checkExternal bool
  76         timeout       time.Duration
  77         logger        *logrus.Logger
  78         errors        []string
  79         done          map[int]bool
  80 }
  81
  82 func (diag *diagnoser) debugf(f string, args ...interface{}) {
  83         diag.logger.Debugf("  ... "+f, args...)
  84 }
  85
  86 func (diag *diagnoser) infof(f string, args ...interface{}) {
  87         diag.logger.Infof("  ... "+f, args...)
  88 }
  89
  90 func (diag *diagnoser) warnf(f string, args ...interface{}) {
  91         diag.logger.Warnf("  ... "+f, args...)
  92 }
  93
  94 func (diag *diagnoser) errorf(f string, args ...interface{}) {
  95         diag.logger.Errorf(f, args...)
  96         diag.errors = append(diag.errors, fmt.Sprintf(f, args...))
  97 }
  98
  99 // Run the given func, logging appropriate messages before and after,
 100 // adding timing info, etc.
 101 //
 102 // The id argument should be unique among tests, and shouldn't change
 103 // when other tests are added/removed.
 104 func (diag *diagnoser) dotest(id int, title string, fn func() error) {
 105         if diag.done == nil {
 106                 diag.done = map[int]bool{}
 107         } else if diag.done[id] {
 108                 diag.errorf("(bug) reused test id %d", id)
 109         }
 110         diag.done[id] = true
 111
 112         diag.logger.Infof("%4d: %s", id, title)
 113         t0 := time.Now()
 114         err := fn()
 115         elapsed := fmt.Sprintf("%d ms", time.Now().Sub(t0)/time.Millisecond)
 116         if err != nil {
 117                 diag.errorf("%4d: %s (%s): %s", id, title, elapsed, err)
 118         } else {
 119                 diag.logger.Debugf("%4d: %s (%s): ok", id, title, elapsed)
 120         }
 121 }
 122
 123 func (diag *diagnoser) runtests() {
 124         client := arvados.NewClientFromEnv()
 125
 126         if client.APIHost == "" || client.AuthToken == "" {
 127                 diag.errorf("ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables are not set -- aborting without running any tests")
 128                 return
 129         }
 130
 131         diag.dotest(5, "running health check (same as `arvados-server check`)", func() error {
 132                 ldr := config.NewLoader(&bytes.Buffer{}, ctxlog.New(&bytes.Buffer{}, "text", "info"))
 133                 ldr.SetupFlags(flag.NewFlagSet("diagnostics", flag.ContinueOnError))
 134                 cfg, err := ldr.Load()
 135                 if err != nil {
 136                         diag.infof("skipping because config could not be loaded: %s", err)
 137                         return nil
 138                 }
 139                 cluster, err := cfg.GetCluster("")
 140                 if err != nil {
 141                         return err
 142                 }
 143                 if cluster.SystemRootToken != os.Getenv("ARVADOS_API_TOKEN") {
 144                         diag.infof("skipping because provided token is not SystemRootToken")
 145                 }
 146                 agg := &health.Aggregator{Cluster: cluster}
 147                 resp := agg.ClusterHealth()
 148                 for _, e := range resp.Errors {
 149                         diag.errorf("health check: %s", e)
 150                 }
 151                 diag.infof("health check: reported clock skew %v", resp.ClockSkew)
 152                 return nil
 153         })
 154
 155         var dd arvados.DiscoveryDocument
 156         ddpath := "discovery/v1/apis/arvados/v1/rest"
 157         diag.dotest(10, fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath), func() error {
 158                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 159                 defer cancel()
 160                 err := client.RequestAndDecodeContext(ctx, &dd, "GET", ddpath, nil, nil)
 161                 if err != nil {
 162                         return err
 163                 }
 164                 diag.debugf("BlobSignatureTTL = %d", dd.BlobSignatureTTL)
 165                 return nil
 166         })
 167
 168         var cluster arvados.Cluster
 169         cfgpath := "arvados/v1/config"
 170         cfgOK := false
 171         diag.dotest(20, fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath), func() error {
 172                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 173                 defer cancel()
 174                 err := client.RequestAndDecodeContext(ctx, &cluster, "GET", cfgpath, nil, nil)
 175                 if err != nil {
 176                         return err
 177                 }
 178                 diag.debugf("Collections.BlobSigning = %v", cluster.Collections.BlobSigning)
 179                 cfgOK = true
 180                 return nil
 181         })
 182
 183         var user arvados.User
 184         diag.dotest(30, "getting current user record", func() error {
 185                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 186                 defer cancel()
 187                 err := client.RequestAndDecodeContext(ctx, &user, "GET", "arvados/v1/users/current", nil, nil)
 188                 if err != nil {
 189                         return err
 190                 }
 191                 diag.debugf("user uuid = %s", user.UUID)
 192                 return nil
 193         })
 194
 195         if !cfgOK {
 196                 diag.errorf("cannot proceed without cluster config -- aborting without running any further tests")
 197                 return
 198         }
 199
 200         // uncomment to create some spurious errors
 201         // cluster.Services.WebDAVDownload.ExternalURL.Host = "0.0.0.0:9"
 202
 203         // TODO: detect routing errors here, like finding wb2 at the
 204         // wb1 address.
 205         for i, svc := range []*arvados.Service{
 206                 &cluster.Services.Keepproxy,
 207                 &cluster.Services.WebDAV,
 208                 &cluster.Services.WebDAVDownload,
 209                 &cluster.Services.Websocket,
 210                 &cluster.Services.Workbench1,
 211                 &cluster.Services.Workbench2,
 212         } {
 213                 diag.dotest(40+i, fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL), func() error {
 214                         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 215                         defer cancel()
 216                         u := svc.ExternalURL
 217                         if strings.HasPrefix(u.Scheme, "ws") {
 218                                 // We can do a real websocket test elsewhere,
 219                                 // but for now we'll just check the https
 220                                 // connection.
 221                                 u.Scheme = "http" + u.Scheme[2:]
 222                         }
 223                         if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
 224                                 u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
 225                         }
 226                         req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
 227                         if err != nil {
 228                                 return err
 229                         }
 230                         resp, err := http.DefaultClient.Do(req)
 231                         if err != nil {
 232                                 return err
 233                         }
 234                         resp.Body.Close()
 235                         return nil
 236                 })
 237         }
 238
 239         for i, url := range []string{
 240                 cluster.Services.Controller.ExternalURL.String(),
 241                 cluster.Services.Keepproxy.ExternalURL.String() + "d41d8cd98f00b204e9800998ecf8427e+0",
 242                 cluster.Services.WebDAVDownload.ExternalURL.String(),
 243         } {
 244                 diag.dotest(50+i, fmt.Sprintf("checking CORS headers at %s", url), func() error {
 245                         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 246                         defer cancel()
 247                         req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 248                         if err != nil {
 249                                 return err
 250                         }
 251                         req.Header.Set("Origin", "https://example.com")
 252                         resp, err := http.DefaultClient.Do(req)
 253                         if err != nil {
 254                                 return err
 255                         }
 256                         if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" {
 257                                 return fmt.Errorf("expected \"Access-Control-Allow-Origin: *\", got %q", hdr)
 258                         }
 259                         return nil
 260                 })
 261         }
 262
 263         var keeplist arvados.KeepServiceList
 264         diag.dotest(60, "checking internal/external client detection", func() error {
 265                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 266                 defer cancel()
 267                 err := client.RequestAndDecodeContext(ctx, &keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: 999999})
 268                 if err != nil {
 269                         return fmt.Errorf("error getting keep services list: %s", err)
 270                 } else if len(keeplist.Items) == 0 {
 271                         return fmt.Errorf("controller did not return any keep services")
 272                 }
 273                 found := map[string]int{}
 274                 for _, ks := range keeplist.Items {
 275                         found[ks.ServiceType]++
 276                 }
 277                 isInternal := found["proxy"] == 0 && len(keeplist.Items) > 0
 278                 isExternal := found["proxy"] > 0 && found["proxy"] == len(keeplist.Items)
 279                 if isExternal {
 280                         diag.debugf("controller returned only proxy services, this host is treated as \"external\"")
 281                 } else if isInternal {
 282                         diag.debugf("controller returned only non-proxy services, this host is treated as \"internal\"")
 283                 }
 284                 if (diag.checkInternal && !isInternal) || (diag.checkExternal && !isExternal) {
 285                         return fmt.Errorf("expecting internal=%v external=%v, but found internal=%v external=%v", diag.checkInternal, diag.checkExternal, isInternal, isExternal)
 286                 }
 287                 return nil
 288         })
 289
 290         for i, ks := range keeplist.Items {
 291                 u := url.URL{
 292                         Scheme: "http",
 293                         Host:   net.JoinHostPort(ks.ServiceHost, fmt.Sprintf("%d", ks.ServicePort)),
 294                         Path:   "/",
 295                 }
 296                 if ks.ServiceSSLFlag {
 297                         u.Scheme = "https"
 298                 }
 299                 diag.dotest(61+i, fmt.Sprintf("reading+writing via keep service at %s", u.String()), func() error {
 300                         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 301                         defer cancel()
 302                         req, err := http.NewRequestWithContext(ctx, "PUT", u.String()+"d41d8cd98f00b204e9800998ecf8427e", nil)
 303                         if err != nil {
 304                                 return err
 305                         }
 306                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 307                         resp, err := http.DefaultClient.Do(req)
 308                         if err != nil {
 309                                 return err
 310                         }
 311                         defer resp.Body.Close()
 312                         body, err := ioutil.ReadAll(resp.Body)
 313                         if err != nil {
 314                                 return fmt.Errorf("reading response body: %s", err)
 315                         }
 316                         loc := strings.TrimSpace(string(body))
 317                         if !strings.HasPrefix(loc, "d41d8") {
 318                                 return fmt.Errorf("unexpected response from write: %q", body)
 319                         }
 320
 321                         req, err = http.NewRequestWithContext(ctx, "GET", u.String()+loc, nil)
 322                         if err != nil {
 323                                 return err
 324                         }
 325                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 326                         resp, err = http.DefaultClient.Do(req)
 327                         if err != nil {
 328                                 return err
 329                         }
 330                         defer resp.Body.Close()
 331                         body, err = ioutil.ReadAll(resp.Body)
 332                         if err != nil {
 333                                 return fmt.Errorf("reading response body: %s", err)
 334                         }
 335                         if len(body) != 0 {
 336                                 return fmt.Errorf("unexpected response from read: %q", body)
 337                         }
 338
 339                         return nil
 340                 })
 341         }
 342
 343         var project arvados.Group
 344         diag.dotest(80, fmt.Sprintf("finding/creating %q project", diag.projectName), func() error {
 345                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 346                 defer cancel()
 347                 var grplist arvados.GroupList
 348                 err := client.RequestAndDecodeContext(ctx, &grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
 349                         Filters: []arvados.Filter{
 350                                 {"name", "=", diag.projectName},
 351                                 {"group_class", "=", "project"},
 352                                 {"owner_uuid", "=", user.UUID}},
 353                         Limit: 999999})
 354                 if err != nil {
 355                         return fmt.Errorf("list groups: %s", err)
 356                 }
 357                 if len(grplist.Items) > 0 {
 358                         project = grplist.Items[0]
 359                         diag.debugf("using existing project, uuid = %s", project.UUID)
 360                         return nil
 361                 }
 362                 diag.debugf("list groups: ok, no results")
 363                 err = client.RequestAndDecodeContext(ctx, &project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
 364                         "name":        diag.projectName,
 365                         "group_class": "project",
 366                 }})
 367                 if err != nil {
 368                         return fmt.Errorf("create project: %s", err)
 369                 }
 370                 diag.debugf("created project, uuid = %s", project.UUID)
 371                 return nil
 372         })
 373
 374         var collection arvados.Collection
 375         diag.dotest(90, "creating temporary collection", func() error {
 376                 if project.UUID == "" {
 377                         return fmt.Errorf("skipping, no project to work in")
 378                 }
 379                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 380                 defer cancel()
 381                 err := client.RequestAndDecodeContext(ctx, &collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 382                         "ensure_unique_name": true,
 383                         "collection": map[string]interface{}{
 384                                 "owner_uuid": project.UUID,
 385                                 "name":       "test collection",
 386                                 "trash_at":   time.Now().Add(time.Hour)}})
 387                 if err != nil {
 388                         return err
 389                 }
 390                 diag.debugf("ok, uuid = %s", collection.UUID)
 391                 return nil
 392         })
 393
 394         if collection.UUID != "" {
 395                 defer func() {
 396                         diag.dotest(9990, "deleting temporary collection", func() error {
 397                                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 398                                 defer cancel()
 399                                 return client.RequestAndDecodeContext(ctx, nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
 400                         })
 401                 }()
 402         }
 403
 404         // Read hello-world.tar to find image ID, so we can upload it
 405         // as "sha256:{...}.tar"
 406         var imageSHA2 string
 407         {
 408                 tr := tar.NewReader(bytes.NewReader(HelloWorldDockerImage))
 409                 for {
 410                         hdr, err := tr.Next()
 411                         if err == io.EOF {
 412                                 break
 413                         }
 414                         if err != nil {
 415                                 diag.errorf("internal error/bug: cannot read embedded docker image tar file: %s", err)
 416                                 return
 417                         }
 418                         if s := strings.TrimSuffix(hdr.Name, ".json"); len(s) == 64 && s != hdr.Name {
 419                                 imageSHA2 = s
 420                         }
 421                 }
 422                 if imageSHA2 == "" {
 423                         diag.errorf("internal error/bug: cannot find {sha256}.json file in embedded docker image tar file")
 424                         return
 425                 }
 426         }
 427         tarfilename := "sha256:" + imageSHA2 + ".tar"
 428
 429         diag.dotest(100, "uploading file via webdav", func() error {
 430                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 431                 defer cancel()
 432                 if collection.UUID == "" {
 433                         return fmt.Errorf("skipping, no test collection")
 434                 }
 435                 req, err := http.NewRequestWithContext(ctx, "PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/"+tarfilename, bytes.NewReader(HelloWorldDockerImage))
 436                 if err != nil {
 437                         return fmt.Errorf("BUG? http.NewRequest: %s", err)
 438                 }
 439                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 440                 resp, err := http.DefaultClient.Do(req)
 441                 if err != nil {
 442                         return fmt.Errorf("error performing http request: %s", err)
 443                 }
 444                 resp.Body.Close()
 445                 if resp.StatusCode != http.StatusCreated {
 446                         return fmt.Errorf("status %s", resp.Status)
 447                 }
 448                 diag.debugf("ok, status %s", resp.Status)
 449                 err = client.RequestAndDecodeContext(ctx, &collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
 450                 if err != nil {
 451                         return fmt.Errorf("get updated collection: %s", err)
 452                 }
 453                 diag.debugf("ok, pdh %s", collection.PortableDataHash)
 454                 return nil
 455         })
 456
 457         davurl := cluster.Services.WebDAV.ExternalURL
 458         davWildcard := strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.")
 459         diag.dotest(110, fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl), func() error {
 460                 if davurl.Host == "" {
 461                         return fmt.Errorf("host missing - content previews will not work")
 462                 }
 463                 if !davWildcard && !cluster.Collections.TrustAllContent {
 464                         diag.warnf("WebDAV ExternalURL has no leading wildcard and TrustAllContent==false - content previews will not work")
 465                 }
 466                 return nil
 467         })
 468
 469         for i, trial := range []struct {
 470                 needcoll     bool
 471                 needWildcard bool
 472                 status       int
 473                 fileurl      string
 474         }{
 475                 {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
 476                 {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + tarfilename},
 477                 {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
 478                 {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/" + tarfilename},
 479                 {true, true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + tarfilename},
 480                 {true, false, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/" + tarfilename},
 481         } {
 482                 diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error {
 483                         if trial.needWildcard && !davWildcard {
 484                                 diag.warnf("skipping collection-id-in-vhost test because WebDAV ExternalURL has no leading wildcard")
 485                                 return nil
 486                         }
 487                         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 488                         defer cancel()
 489                         if trial.needcoll && collection.UUID == "" {
 490                                 return fmt.Errorf("skipping, no test collection")
 491                         }
 492                         req, err := http.NewRequestWithContext(ctx, "GET", trial.fileurl, nil)
 493                         if err != nil {
 494                                 return err
 495                         }
 496                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 497                         resp, err := http.DefaultClient.Do(req)
 498                         if err != nil {
 499                                 return err
 500                         }
 501                         defer resp.Body.Close()
 502                         body, err := ioutil.ReadAll(resp.Body)
 503                         if err != nil {
 504                                 return fmt.Errorf("reading response: %s", err)
 505                         }
 506                         if resp.StatusCode != trial.status {
 507                                 return fmt.Errorf("unexpected response status: %s", resp.Status)
 508                         }
 509                         if trial.status == http.StatusOK && !bytes.Equal(body, HelloWorldDockerImage) {
 510                                 excerpt := body
 511                                 if len(excerpt) > 128 {
 512                                         excerpt = append([]byte(nil), body[:128]...)
 513                                         excerpt = append(excerpt, []byte("[...]")...)
 514                                 }
 515                                 return fmt.Errorf("unexpected response content: len %d, %q", len(body), excerpt)
 516                         }
 517                         return nil
 518                 })
 519         }
 520
 521         var vm arvados.VirtualMachine
 522         diag.dotest(130, "getting list of virtual machines", func() error {
 523                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 524                 defer cancel()
 525                 var vmlist arvados.VirtualMachineList
 526                 err := client.RequestAndDecodeContext(ctx, &vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
 527                 if err != nil {
 528                         return err
 529                 }
 530                 if len(vmlist.Items) < 1 {
 531                         diag.warnf("no VMs found")
 532                 } else {
 533                         vm = vmlist.Items[0]
 534                 }
 535                 return nil
 536         })
 537
 538         diag.dotest(140, "getting workbench1 webshell page", func() error {
 539                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 540                 defer cancel()
 541                 if vm.UUID == "" {
 542                         diag.warnf("skipping, no vm available")
 543                         return nil
 544                 }
 545                 webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername"
 546                 diag.debugf("url %s", webshelltermurl)
 547                 req, err := http.NewRequestWithContext(ctx, "GET", webshelltermurl, nil)
 548                 if err != nil {
 549                         return err
 550                 }
 551                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 552                 resp, err := http.DefaultClient.Do(req)
 553                 if err != nil {
 554                         return err
 555                 }
 556                 defer resp.Body.Close()
 557                 body, err := ioutil.ReadAll(resp.Body)
 558                 if err != nil {
 559                         return fmt.Errorf("reading response: %s", err)
 560                 }
 561                 if resp.StatusCode != http.StatusOK {
 562                         return fmt.Errorf("unexpected response status: %s %q", resp.Status, body)
 563                 }
 564                 return nil
 565         })
 566
 567         diag.dotest(150, "connecting to webshell service", func() error {
 568                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 569                 defer cancel()
 570                 if vm.UUID == "" {
 571                         diag.warnf("skipping, no vm available")
 572                         return nil
 573                 }
 574                 u := cluster.Services.WebShell.ExternalURL
 575                 webshellurl := u.String() + vm.Hostname + "?"
 576                 if strings.HasPrefix(u.Host, "*") {
 577                         u.Host = vm.Hostname + u.Host[1:]
 578                         webshellurl = u.String() + "?"
 579                 }
 580                 diag.debugf("url %s", webshellurl)
 581                 req, err := http.NewRequestWithContext(ctx, "POST", webshellurl, bytes.NewBufferString(url.Values{
 582                         "width":   {"80"},
 583                         "height":  {"25"},
 584                         "session": {"xyzzy"},
 585                         "rooturl": {webshellurl},
 586                 }.Encode()))
 587                 if err != nil {
 588                         return err
 589                 }
 590                 req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
 591                 resp, err := http.DefaultClient.Do(req)
 592                 if err != nil {
 593                         return err
 594                 }
 595                 defer resp.Body.Close()
 596                 diag.debugf("response status %s", resp.Status)
 597                 body, err := ioutil.ReadAll(resp.Body)
 598                 if err != nil {
 599                         return fmt.Errorf("reading response: %s", err)
 600                 }
 601                 diag.debugf("response body %q", body)
 602                 // We don't speak the protocol, so we get a 400 error
 603                 // from the webshell server even if everything is
 604                 // OK. Anything else (404, 502, ???) indicates a
 605                 // problem.
 606                 if resp.StatusCode != http.StatusBadRequest {
 607                         return fmt.Errorf("unexpected response status: %s, %q", resp.Status, body)
 608                 }
 609                 return nil
 610         })
 611
 612         diag.dotest(160, "running a container", func() error {
 613                 if diag.priority < 1 {
 614                         diag.infof("skipping (use priority > 0 if you want to run a container)")
 615                         return nil
 616                 }
 617                 if project.UUID == "" {
 618                         return fmt.Errorf("skipping, no project to work in")
 619                 }
 620
 621                 timestamp := time.Now().Format(time.RFC3339)
 622                 ctrCommand := []string{"echo", timestamp}
 623                 if diag.dockerImage == "" {
 624                         if collection.UUID == "" {
 625                                 return fmt.Errorf("skipping, no test collection to use as docker image")
 626                         }
 627                         diag.dockerImage = collection.PortableDataHash
 628                         ctrCommand = []string{"/hello"}
 629                 }
 630
 631                 var cr arvados.ContainerRequest
 632                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 633                 defer cancel()
 634
 635                 err := client.RequestAndDecodeContext(ctx, &cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{
 636                         "owner_uuid":      project.UUID,
 637                         "name":            fmt.Sprintf("diagnostics container request %s", timestamp),
 638                         "container_image": diag.dockerImage,
 639                         "command":         ctrCommand,
 640                         "use_existing":    false,
 641                         "output_path":     "/mnt/output",
 642                         "output_name":     fmt.Sprintf("diagnostics output %s", timestamp),
 643                         "priority":        diag.priority,
 644                         "state":           arvados.ContainerRequestStateCommitted,
 645                         "mounts": map[string]map[string]interface{}{
 646                                 "/mnt/output": {
 647                                         "kind":     "collection",
 648                                         "writable": true,
 649                                 },
 650                         },
 651                         "runtime_constraints": arvados.RuntimeConstraints{
 652                                 VCPUs:        1,
 653                                 RAM:          1 << 26,
 654                                 KeepCacheRAM: 1 << 26,
 655                         },
 656                 }})
 657                 if err != nil {
 658                         return err
 659                 }
 660                 diag.debugf("container request uuid = %s", cr.UUID)
 661                 diag.debugf("container uuid = %s", cr.ContainerUUID)
 662
 663                 timeout := 10 * time.Minute
 664                 diag.infof("container request submitted, waiting up to %v for container to run", arvados.Duration(timeout))
 665                 ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(timeout))
 666                 defer cancel()
 667
 668                 var c arvados.Container
 669                 for ; cr.State != arvados.ContainerRequestStateFinal; time.Sleep(2 * time.Second) {
 670                         ctx, cancel := context.WithDeadline(ctx, time.Now().Add(diag.timeout))
 671                         defer cancel()
 672
 673                         crStateWas := cr.State
 674                         err := client.RequestAndDecodeContext(ctx, &cr, "GET", "arvados/v1/container_requests/"+cr.UUID, nil, nil)
 675                         if err != nil {
 676                                 return err
 677                         }
 678                         if cr.State != crStateWas {
 679                                 diag.debugf("container request state = %s", cr.State)
 680                         }
 681
 682                         cStateWas := c.State
 683                         err = client.RequestAndDecodeContext(ctx, &c, "GET", "arvados/v1/containers/"+cr.ContainerUUID, nil, nil)
 684                         if err != nil {
 685                                 return err
 686                         }
 687                         if c.State != cStateWas {
 688                                 diag.debugf("container state = %s", c.State)
 689                         }
 690                 }
 691
 692                 if c.State != arvados.ContainerStateComplete {
 693                         return fmt.Errorf("container request %s is final but container %s did not complete: container state = %q", cr.UUID, cr.ContainerUUID, c.State)
 694                 } else if c.ExitCode != 0 {
 695                         return fmt.Errorf("container exited %d", c.ExitCode)
 696                 }
 697                 return nil
 698         })
 699 }