lib/diagnostics/cmd.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package diagnostics
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "flag"
  11         "fmt"
  12         "io"
  13         "io/ioutil"
  14         "net"
  15         "net/http"
  16         "net/url"
  17         "strings"
  18         "time"
  19
  20         "git.arvados.org/arvados.git/sdk/go/arvados"
  21         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  22         "github.com/sirupsen/logrus"
  23 )
  24
  25 type Command struct{}
  26
  27 func (cmd Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
  28         var diag diagnoser
  29         f := flag.NewFlagSet(prog, flag.ContinueOnError)
  30         f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects")
  31         f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
  32         f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client")
  33         f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client")
  34         f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests")
  35         err := f.Parse(args)
  36         if err == flag.ErrHelp {
  37                 return 0
  38         } else if err != nil {
  39                 fmt.Fprintln(stderr, err)
  40                 return 2
  41         }
  42         diag.logger = ctxlog.New(stdout, "text", diag.logLevel)
  43         diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true})
  44         diag.runtests()
  45         if len(diag.errors) == 0 {
  46                 diag.logger.Info("--- no errors ---")
  47                 return 0
  48         } else {
  49                 if diag.logger.Level > logrus.ErrorLevel {
  50                         fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n")
  51                         for _, e := range diag.errors {
  52                                 diag.logger.Error(e)
  53                         }
  54                 }
  55                 return 1
  56         }
  57 }
  58
  59 type diagnoser struct {
  60         stdout        io.Writer
  61         stderr        io.Writer
  62         logLevel      string
  63         projectName   string
  64         checkInternal bool
  65         checkExternal bool
  66         timeout       time.Duration
  67         logger        *logrus.Logger
  68         errors        []string
  69         done          map[int]bool
  70 }
  71
  72 func (diag *diagnoser) debugf(f string, args ...interface{}) {
  73         diag.logger.Debugf(f, args...)
  74 }
  75
  76 func (diag *diagnoser) infof(f string, args ...interface{}) {
  77         diag.logger.Infof(f, args...)
  78 }
  79
  80 func (diag *diagnoser) warnf(f string, args ...interface{}) {
  81         diag.logger.Warnf(f, args...)
  82 }
  83
  84 func (diag *diagnoser) errorf(f string, args ...interface{}) {
  85         diag.logger.Errorf(f, args...)
  86         diag.errors = append(diag.errors, fmt.Sprintf(f, args...))
  87 }
  88
  89 // Run the given func, logging appropriate messages before and after,
  90 // adding timing info, etc.
  91 //
  92 // The id argument should be unique among tests, and shouldn't change
  93 // when other tests are added/removed.
  94 func (diag *diagnoser) dotest(id int, title string, fn func() error) {
  95         if diag.done == nil {
  96                 diag.done = map[int]bool{}
  97         } else if diag.done[id] {
  98                 diag.errorf("(bug) reused test id %d", id)
  99         }
 100         diag.done[id] = true
 101
 102         diag.infof("%d %s", id, title)
 103         t0 := time.Now()
 104         err := fn()
 105         elapsed := fmt.Sprintf("%.0dms", time.Now().Sub(t0)/time.Millisecond)
 106         if err != nil {
 107                 diag.errorf("%s (%s): %s", title, elapsed, err)
 108         }
 109         diag.debugf("%d %s (%s): ok", id, title, elapsed)
 110 }
 111
 112 func (diag *diagnoser) runtests() {
 113         client := arvados.NewClientFromEnv()
 114
 115         if client.APIHost == "" || client.AuthToken == "" {
 116                 diag.errorf("ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables are not set -- aborting without running any tests")
 117                 return
 118         }
 119
 120         var dd arvados.DiscoveryDocument
 121         ddpath := "discovery/v1/apis/arvados/v1/rest"
 122         diag.dotest(10, fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath), func() error {
 123                 err := client.RequestAndDecode(&dd, "GET", ddpath, nil, nil)
 124                 if err != nil {
 125                         return err
 126                 }
 127                 diag.debugf("BlobSignatureTTL = %d", dd.BlobSignatureTTL)
 128                 return nil
 129         })
 130
 131         var cluster arvados.Cluster
 132         cfgpath := "arvados/v1/config"
 133         diag.dotest(20, fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath), func() error {
 134                 err := client.RequestAndDecode(&cluster, "GET", cfgpath, nil, nil)
 135                 if err != nil {
 136                         return err
 137                 }
 138                 diag.debugf("Collections.BlobSigning = %v", cluster.Collections.BlobSigning)
 139                 return nil
 140         })
 141
 142         var user arvados.User
 143         diag.dotest(30, "getting current user record", func() error {
 144                 err := client.RequestAndDecode(&user, "GET", "arvados/v1/users/current", nil, nil)
 145                 if err != nil {
 146                         return err
 147                 }
 148                 diag.debugf("user uuid = %s", user.UUID)
 149                 return nil
 150         })
 151
 152         // uncomment to create some spurious errors
 153         // cluster.Services.WebDAVDownload.ExternalURL.Host = "0.0.0.0:9"
 154
 155         // TODO: detect routing errors here, like finding wb2 at the
 156         // wb1 address.
 157         for i, svc := range []*arvados.Service{
 158                 &cluster.Services.Keepproxy,
 159                 &cluster.Services.WebDAV,
 160                 &cluster.Services.WebDAVDownload,
 161                 &cluster.Services.Websocket,
 162                 &cluster.Services.Workbench1,
 163                 &cluster.Services.Workbench2,
 164         } {
 165                 diag.dotest(40+i, fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL), func() error {
 166                         u := svc.ExternalURL
 167                         if strings.HasPrefix(u.Scheme, "ws") {
 168                                 // We can do a real websocket test elsewhere,
 169                                 // but for now we'll just check the https
 170                                 // connection.
 171                                 u.Scheme = "http" + u.Scheme[2:]
 172                         }
 173                         if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
 174                                 u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
 175                         }
 176                         req, err := http.NewRequest(http.MethodGet, u.String(), nil)
 177                         if err != nil {
 178                                 return err
 179                         }
 180                         resp, err := http.DefaultClient.Do(req)
 181                         if err != nil {
 182                                 return err
 183                         }
 184                         resp.Body.Close()
 185                         return nil
 186                 })
 187         }
 188
 189         for i, url := range []string{
 190                 cluster.Services.Controller.ExternalURL.String(),
 191                 cluster.Services.Keepproxy.ExternalURL.String() + "d41d8cd98f00b204e9800998ecf8427e+0",
 192                 cluster.Services.WebDAVDownload.ExternalURL.String(),
 193         } {
 194                 diag.dotest(50+i, fmt.Sprintf("checking CORS headers at %s", url), func() error {
 195                         req, err := http.NewRequest("GET", url, nil)
 196                         if err != nil {
 197                                 return err
 198                         }
 199                         req.Header.Set("Origin", "https://example.com")
 200                         resp, err := http.DefaultClient.Do(req)
 201                         if err != nil {
 202                                 return err
 203                         }
 204                         if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" {
 205                                 return fmt.Errorf("expected \"Access-Control-Allow-Origin: *\", got %q", hdr)
 206                         }
 207                         return nil
 208                 })
 209         }
 210
 211         var keeplist arvados.KeepServiceList
 212         diag.dotest(60, "checking internal/external client detection", func() error {
 213                 err := client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: -1})
 214                 if err != nil {
 215                         return fmt.Errorf("error getting keep services list: %s", err)
 216                 } else if len(keeplist.Items) == 0 {
 217                         return fmt.Errorf("controller did not return any keep services")
 218                 }
 219                 found := map[string]int{}
 220                 for _, ks := range keeplist.Items {
 221                         found[ks.ServiceType]++
 222                 }
 223                 isInternal := found["proxy"] == 0 && len(keeplist.Items) > 0
 224                 isExternal := found["proxy"] > 0 && found["proxy"] == len(keeplist.Items)
 225                 if isExternal {
 226                         diag.debugf("controller returned only proxy services, this host is treated as \"external\"")
 227                 } else if isInternal {
 228                         diag.debugf("controller returned only non-proxy services, this host is treated as \"internal\"")
 229                 }
 230                 if (diag.checkInternal && !isInternal) || (diag.checkExternal && !isExternal) {
 231                         return fmt.Errorf("expecting internal=%v external=%v, but found internal=%v external=%v", diag.checkInternal, diag.checkExternal, isInternal, isExternal)
 232                 }
 233                 return nil
 234         })
 235
 236         for i, ks := range keeplist.Items {
 237                 u := url.URL{
 238                         Scheme: "http",
 239                         Host:   net.JoinHostPort(ks.ServiceHost, fmt.Sprintf("%d", ks.ServicePort)),
 240                         Path:   "/",
 241                 }
 242                 if ks.ServiceSSLFlag {
 243                         u.Scheme = "https"
 244                 }
 245                 diag.dotest(61+i, fmt.Sprintf("reading+writing via keep service at %s", u.String()), func() error {
 246                         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 247                         defer cancel()
 248                         req, err := http.NewRequestWithContext(ctx, "PUT", u.String()+"d41d8cd98f00b204e9800998ecf8427e", nil)
 249                         if err != nil {
 250                                 return err
 251                         }
 252                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 253                         resp, err := http.DefaultClient.Do(req)
 254                         if err != nil {
 255                                 return err
 256                         }
 257                         defer resp.Body.Close()
 258                         body, err := ioutil.ReadAll(resp.Body)
 259                         if err != nil {
 260                                 return fmt.Errorf("reading response body: %s", err)
 261                         }
 262                         loc := strings.TrimSpace(string(body))
 263                         if !strings.HasPrefix(loc, "d41d8") {
 264                                 return fmt.Errorf("unexpected response from write: %q", body)
 265                         }
 266
 267                         req, err = http.NewRequestWithContext(ctx, "GET", u.String()+loc, nil)
 268                         if err != nil {
 269                                 return err
 270                         }
 271                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 272                         resp, err = http.DefaultClient.Do(req)
 273                         if err != nil {
 274                                 return err
 275                         }
 276                         defer resp.Body.Close()
 277                         body, err = ioutil.ReadAll(resp.Body)
 278                         if err != nil {
 279                                 return fmt.Errorf("reading response body: %s", err)
 280                         }
 281                         if len(body) != 0 {
 282                                 return fmt.Errorf("unexpected response from read: %q", body)
 283                         }
 284
 285                         return nil
 286                 })
 287         }
 288
 289         var project arvados.Group
 290         diag.dotest(80, fmt.Sprintf("finding/creating %q project", diag.projectName), func() error {
 291                 var grplist arvados.GroupList
 292                 err := client.RequestAndDecode(&grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
 293                         Filters: []arvados.Filter{
 294                                 {"name", "=", diag.projectName},
 295                                 {"group_class", "=", "project"},
 296                                 {"owner_uuid", "=", user.UUID}},
 297                         Limit: -1})
 298                 if err != nil {
 299                         return fmt.Errorf("list groups: %s", err)
 300                 }
 301                 if len(grplist.Items) > 0 {
 302                         project = grplist.Items[0]
 303                         diag.debugf("using existing project, uuid = %s", project.UUID)
 304                         return nil
 305                 }
 306                 diag.debugf("list groups: ok, no results")
 307                 err = client.RequestAndDecode(&project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
 308                         "name":        diag.projectName,
 309                         "group_class": "project",
 310                 }})
 311                 if err != nil {
 312                         return fmt.Errorf("create project: %s", err)
 313                 }
 314                 diag.debugf("created project, uuid = %s", project.UUID)
 315                 return nil
 316         })
 317
 318         var collection arvados.Collection
 319         diag.dotest(90, "creating temporary collection", func() error {
 320                 err := client.RequestAndDecode(&collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 321                         "ensure_unique_name": true,
 322                         "collection": map[string]interface{}{
 323                                 "name":     "test collection",
 324                                 "trash_at": time.Now().Add(time.Hour)}})
 325                 if err != nil {
 326                         return err
 327                 }
 328                 diag.debugf("ok, uuid = %s", collection.UUID)
 329                 return nil
 330         })
 331
 332         if collection.UUID != "" {
 333                 defer func() {
 334                         diag.dotest(9990, "deleting temporary collection", func() error {
 335                                 return client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
 336                         })
 337                 }()
 338         }
 339
 340         diag.dotest(100, "uploading file via webdav", func() error {
 341                 if collection.UUID == "" {
 342                         return fmt.Errorf("skipping, no test collection")
 343                 }
 344                 req, err := http.NewRequest("PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata"))
 345                 if err != nil {
 346                         return fmt.Errorf("BUG? http.NewRequest: %s", err)
 347                 }
 348                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 349                 resp, err := http.DefaultClient.Do(req)
 350                 if err != nil {
 351                         return fmt.Errorf("error performing http request: %s", err)
 352                 }
 353                 resp.Body.Close()
 354                 if resp.StatusCode != http.StatusCreated {
 355                         return fmt.Errorf("status %s", resp.Status)
 356                 }
 357                 diag.debugf("ok, status %s", resp.Status)
 358                 err = client.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
 359                 if err != nil {
 360                         return fmt.Errorf("get updated collection: %s", err)
 361                 }
 362                 diag.debugf("ok, pdh %s", collection.PortableDataHash)
 363                 return nil
 364         })
 365
 366         davurl := cluster.Services.WebDAV.ExternalURL
 367         diag.dotest(110, fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl), func() error {
 368                 if davurl.Host == "" {
 369                         return fmt.Errorf("host missing - content previews will not work")
 370                 }
 371                 if !strings.HasPrefix(davurl.Host, "*--") && !strings.HasPrefix(davurl.Host, "*.") && !cluster.Collections.TrustAllContent {
 372                         diag.warnf("WebDAV ExternalURL has no leading wildcard and TrustAllContent==false - content previews will not work")
 373                 }
 374                 return nil
 375         })
 376
 377         for i, trial := range []struct {
 378                 needcoll bool
 379                 status   int
 380                 fileurl  string
 381         }{
 382                 {false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
 383                 {false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"},
 384                 {false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
 385                 {false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"},
 386                 {true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"},
 387                 {true, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
 388         } {
 389                 diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error {
 390                         if trial.needcoll && collection.UUID == "" {
 391                                 return fmt.Errorf("skipping, no test collection")
 392                         }
 393                         req, err := http.NewRequest("GET", trial.fileurl, nil)
 394                         if err != nil {
 395                                 return err
 396                         }
 397                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 398                         resp, err := http.DefaultClient.Do(req)
 399                         if err != nil {
 400                                 return err
 401                         }
 402                         defer resp.Body.Close()
 403                         body, err := ioutil.ReadAll(resp.Body)
 404                         if err != nil {
 405                                 return fmt.Errorf("reading response: %s", err)
 406                         }
 407                         if resp.StatusCode != trial.status {
 408                                 return fmt.Errorf("unexpected response status: %s", resp.Status)
 409                         }
 410                         if trial.status == http.StatusOK && string(body) != "testfiledata" {
 411                                 return fmt.Errorf("unexpected response content: %q", body)
 412                         }
 413                         return nil
 414                 })
 415         }
 416
 417         var vm arvados.VirtualMachine
 418         diag.dotest(130, "getting list of virtual machines", func() error {
 419                 var vmlist arvados.VirtualMachineList
 420                 err := client.RequestAndDecode(&vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
 421                 if err != nil {
 422                         return err
 423                 }
 424                 if len(vmlist.Items) < 1 {
 425                         return fmt.Errorf("no VMs found")
 426                 }
 427                 vm = vmlist.Items[0]
 428                 return nil
 429         })
 430
 431         diag.dotest(140, "getting workbench1 webshell page", func() error {
 432                 if vm.UUID == "" {
 433                         return fmt.Errorf("skipping, no vm available")
 434                 }
 435                 webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername"
 436                 diag.debugf("url %s", webshelltermurl)
 437                 req, err := http.NewRequest("GET", webshelltermurl, nil)
 438                 if err != nil {
 439                         return err
 440                 }
 441                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 442                 resp, err := http.DefaultClient.Do(req)
 443                 if err != nil {
 444                         return err
 445                 }
 446                 defer resp.Body.Close()
 447                 body, err := ioutil.ReadAll(resp.Body)
 448                 if err != nil {
 449                         return fmt.Errorf("reading response: %s", err)
 450                 }
 451                 if resp.StatusCode != http.StatusOK {
 452                         return fmt.Errorf("unexpected response status: %s %q", resp.Status, body)
 453                 }
 454                 return nil
 455         })
 456
 457         diag.dotest(150, "connecting to webshell service", func() error {
 458                 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 459                 defer cancel()
 460                 if vm.UUID == "" {
 461                         return fmt.Errorf("skipping, no vm available")
 462                 }
 463                 u := cluster.Services.WebShell.ExternalURL
 464                 webshellurl := u.String() + vm.Hostname + "?"
 465                 if strings.HasPrefix(u.Host, "*") {
 466                         u.Host = vm.Hostname + u.Host[1:]
 467                         webshellurl = u.String() + "?"
 468                 }
 469                 diag.debugf("url %s", webshellurl)
 470                 req, err := http.NewRequestWithContext(ctx, "POST", webshellurl, bytes.NewBufferString(url.Values{
 471                         "width":   {"80"},
 472                         "height":  {"25"},
 473                         "session": {"xyzzy"},
 474                         "rooturl": {webshellurl},
 475                 }.Encode()))
 476                 if err != nil {
 477                         return err
 478                 }
 479                 req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
 480                 resp, err := http.DefaultClient.Do(req)
 481                 if err != nil {
 482                         return err
 483                 }
 484                 defer resp.Body.Close()
 485                 diag.debugf("response status %s", resp.Status)
 486                 body, err := ioutil.ReadAll(resp.Body)
 487                 if err != nil {
 488                         return fmt.Errorf("reading response: %s", err)
 489                 }
 490                 diag.debugf("response body %q", body)
 491                 // We don't speak the protocol, so we get a 400 error
 492                 // from the webshell server even if everything is
 493                 // OK. Anything else (404, 502, ???) indicates a
 494                 // problem.
 495                 if resp.StatusCode != http.StatusBadRequest {
 496                         return fmt.Errorf("unexpected response status: %s, %q", resp.Status, body)
 497                 }
 498                 return nil
 499         })
 500 }