lib/diagnostics/cmd.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package diagnostics
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "flag"
  11         "fmt"
  12         "io"
  13         "io/ioutil"
  14         "net/http"
  15         "net/url"
  16         "strings"
  17         "time"
  18
  19         "git.arvados.org/arvados.git/sdk/go/arvados"
  20         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  21         "github.com/sirupsen/logrus"
  22 )
  23
  24 type Command struct {
  25         projectName string
  26 }
  27
  28 func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
  29         f := flag.NewFlagSet(prog, flag.ContinueOnError)
  30         f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects")
  31         loglevel := f.String("log-level", "info", "logging level (debug, info, warning, error)")
  32         checkInternal := f.Bool("internal-client", false, "check that this host is considered an \"internal\" client")
  33         checkExternal := f.Bool("external-client", false, "check that this host is considered an \"external\" client")
  34         timeout := f.Duration("timeout", 10*time.Second, "timeout for http requests")
  35         err := f.Parse(args)
  36         if err == flag.ErrHelp {
  37                 return 0
  38         } else if err != nil {
  39                 fmt.Fprintln(stderr, err)
  40                 return 2
  41         }
  42
  43         ctx := context.Background()
  44
  45         logger := ctxlog.New(stdout, "text", *loglevel)
  46         logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true})
  47
  48         infof := logger.Infof
  49         warnf := logger.Warnf
  50         debugf := logger.Debugf
  51         var errors []string
  52         errorf := func(f string, args ...interface{}) {
  53                 logger.Errorf(f, args...)
  54                 errors = append(errors, fmt.Sprintf(f, args...))
  55         }
  56         defer func() {
  57                 if len(errors) == 0 {
  58                         logger.Info("--- no errors ---")
  59                 } else {
  60                         fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n")
  61                         for _, e := range errors {
  62                                 logger.Error(e)
  63                         }
  64                 }
  65         }()
  66
  67         client := arvados.NewClientFromEnv()
  68
  69         var dd arvados.DiscoveryDocument
  70         ddpath := "discovery/v1/apis/arvados/v1/rest"
  71         testname := fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath)
  72         logger.Info(testname)
  73         err = client.RequestAndDecode(&dd, "GET", ddpath, nil, nil)
  74         if err != nil {
  75                 errorf("%s: %s", testname, err)
  76         } else {
  77                 infof("%s: ok, BlobSignatureTTL is %d", testname, dd.BlobSignatureTTL)
  78         }
  79
  80         var cluster arvados.Cluster
  81         cfgpath := "arvados/v1/config"
  82         testname = fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath)
  83         logger.Info(testname)
  84         err = client.RequestAndDecode(&cluster, "GET", cfgpath, nil, nil)
  85         if err != nil {
  86                 errorf("%s: %s", testname, err)
  87         } else {
  88                 infof("%s: ok, Collections.BlobSigning = %v", testname, cluster.Collections.BlobSigning)
  89         }
  90
  91         var user arvados.User
  92         testname = "getting current user record"
  93         logger.Info(testname)
  94         err = client.RequestAndDecode(&user, "GET", "arvados/v1/users/current", nil, nil)
  95         if err != nil {
  96                 errorf("%s: %s", testname, err)
  97                 return 2
  98         } else {
  99                 infof("%s: ok, uuid = %s", testname, user.UUID)
 100         }
 101
 102         // uncomment to create some spurious errors
 103         // cluster.Services.WebDAVDownload.ExternalURL.Host = "0.0.0.0:9"
 104
 105         // TODO: detect routing errors here, like finding wb2 at the
 106         // wb1 address.
 107         for _, svc := range []*arvados.Service{
 108                 &cluster.Services.Keepproxy,
 109                 &cluster.Services.WebDAV,
 110                 &cluster.Services.WebDAVDownload,
 111                 &cluster.Services.Websocket,
 112                 &cluster.Services.Workbench1,
 113                 &cluster.Services.Workbench2,
 114         } {
 115                 testname = fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL)
 116                 logger.Info(testname)
 117                 u := svc.ExternalURL
 118                 if strings.HasPrefix(u.Scheme, "ws") {
 119                         // We can do a real websocket test elsewhere,
 120                         // but for now we'll just check the https
 121                         // connection.
 122                         u.Scheme = "http" + u.Scheme[2:]
 123                 }
 124                 if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
 125                         u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
 126                 }
 127                 req, err := http.NewRequest(http.MethodGet, u.String(), nil)
 128                 if err != nil {
 129                         errorf("%s: %s", testname, err)
 130                         continue
 131                 }
 132                 resp, err := http.DefaultClient.Do(req)
 133                 if err != nil {
 134                         errorf("%s: %s", testname, err)
 135                         continue
 136                 }
 137                 resp.Body.Close()
 138                 infof("%s: ok", testname)
 139         }
 140
 141         for _, url := range []string{
 142                 cluster.Services.Controller.ExternalURL.String(),
 143                 cluster.Services.Keepproxy.ExternalURL.String() + "d41d8cd98f00b204e9800998ecf8427e+0",
 144                 cluster.Services.WebDAVDownload.ExternalURL.String(),
 145         } {
 146                 testname = fmt.Sprintf("checking CORS headers at %s", url)
 147                 logger.Info(testname)
 148                 req, err := http.NewRequest("GET", url, nil)
 149                 if err != nil {
 150                         errorf("%s: %s", testname, err)
 151                         continue
 152                 }
 153                 req.Header.Set("Origin", "https://example.com")
 154                 resp, err := http.DefaultClient.Do(req)
 155                 if err != nil {
 156                         errorf("%s: %s", testname, err)
 157                         continue
 158                 }
 159                 if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" {
 160                         warnf("%s: expected \"Access-Control-Allow-Origin: *\", got %q", testname, hdr)
 161                 } else {
 162                         infof("%s: ok", testname)
 163                 }
 164         }
 165
 166         var keeplist arvados.KeepServiceList
 167         testname = "checking internal/external client detection"
 168         logger.Info(testname)
 169         err = client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: -1})
 170         if err != nil {
 171                 errorf("%s: error getting keep services list: %s", testname, err)
 172         } else if len(keeplist.Items) == 0 {
 173                 errorf("%s: controller did not return any keep services", testname)
 174         } else {
 175                 found := map[string]int{}
 176                 for _, ks := range keeplist.Items {
 177                         found[ks.ServiceType]++
 178                 }
 179                 infof := infof
 180                 isInternal := found["proxy"] == 0 && len(keeplist.Items) > 0
 181                 isExternal := found["proxy"] > 0 && found["proxy"] == len(keeplist.Items)
 182                 if (*checkInternal && !isInternal) || (*checkExternal && !isExternal) {
 183                         infof = errorf
 184                 }
 185                 if isExternal {
 186                         infof("%s: controller returned only proxy services, this host is considered \"external\"", testname)
 187                 } else if isInternal {
 188                         infof("%s: controller returned only non-proxy services, this host is considered \"internal\"", testname)
 189                 } else {
 190                         errorf("%s: controller returned both proxy and non-proxy services: %v", testname, found)
 191                 }
 192         }
 193
 194         var project arvados.Group
 195         var grplist arvados.GroupList
 196         testname = fmt.Sprintf("finding/creating %q project", diag.projectName)
 197         logger.Info(testname)
 198         err = client.RequestAndDecode(&grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
 199                 Filters: []arvados.Filter{
 200                         {"name", "=", diag.projectName},
 201                         {"group_class", "=", "project"},
 202                         {"owner_uuid", "=", user.UUID}},
 203                 Limit: -1})
 204         if err != nil {
 205                 errorf("%s: list groups: %s", testname, err)
 206         } else if len(grplist.Items) < 1 {
 207                 infof("%s: list groups: ok, no results", testname)
 208                 err = client.RequestAndDecode(&project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
 209                         "name":        diag.projectName,
 210                         "group_class": "project",
 211                 }})
 212                 if err != nil {
 213                         errorf("%s: create project: %s", testname, err)
 214                 } else {
 215                         infof("%s: created project, uuid = %s", testname, project.UUID)
 216                 }
 217         } else {
 218                 project = grplist.Items[0]
 219                 infof("%s: ok, using existing project, uuid = %s", testname, project.UUID)
 220         }
 221
 222         testname = "creating temporary collection"
 223         logger.Info(testname)
 224         var collection arvados.Collection
 225         err = client.RequestAndDecode(&collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 226                 "ensure_unique_name": true,
 227                 "collection": map[string]interface{}{
 228                         "name":     "test collection",
 229                         "trash_at": time.Now().Add(time.Hour)}})
 230         if err != nil {
 231                 errorf("%s: %s", testname, err)
 232         } else {
 233                 infof("%s: ok, uuid = %s", testname, collection.UUID)
 234                 defer func() {
 235                         testname := "deleting temporary collection"
 236                         logger.Info(testname)
 237                         err := client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
 238                         if err != nil {
 239                                 errorf("%s: %s", testname, err)
 240                         } else {
 241                                 infof("%s: ok", testname)
 242                         }
 243                 }()
 244         }
 245
 246         testname = "uploading file via webdav"
 247         logger.Info(testname)
 248         func() {
 249                 if collection.UUID == "" {
 250                         infof("%s: skipping, no test collection", testname)
 251                         return
 252                 }
 253                 req, err := http.NewRequest("PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata"))
 254                 if err != nil {
 255                         errorf("%s: BUG? http.NewRequest: %s", testname, err)
 256                         return
 257                 }
 258                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 259                 resp, err := http.DefaultClient.Do(req)
 260                 if err != nil {
 261                         errorf("%s: error performing http request: %s", testname, err)
 262                         return
 263                 }
 264                 resp.Body.Close()
 265                 if resp.StatusCode != http.StatusCreated {
 266                         errorf("%s: status %s", testname, resp.Status)
 267                         return
 268                 }
 269                 infof("%s: ok, status %s", testname, resp.Status)
 270                 err = client.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
 271                 if err != nil {
 272                         errorf("%s: get updated collection: %s", testname, err)
 273                         return
 274                 }
 275                 infof("%s: get updated collection: ok, pdh %s", testname, collection.PortableDataHash)
 276         }()
 277
 278         davurl := cluster.Services.WebDAV.ExternalURL
 279         testname = fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl)
 280         logger.Info(testname)
 281         if strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.") {
 282                 infof("%s: looks ok", testname)
 283         } else if davurl.Host == "" {
 284                 warnf("%s: host missing - content previews will not work", testname)
 285         } else {
 286                 warnf("%s: host has no leading wildcard - content previews will not work unless TrustAllContent==true", testname)
 287         }
 288
 289         for _, trial := range []struct {
 290                 status  int
 291                 fileurl string
 292         }{
 293                 {http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
 294                 {http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"},
 295                 {http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
 296                 {http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"},
 297                 {http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"},
 298                 {http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
 299         } {
 300                 func() {
 301                         testname := fmt.Sprintf("downloading from webdav (%s)", trial.fileurl)
 302                         logger.Info(testname)
 303                         if collection.UUID == "" {
 304                                 errorf("%s: skipping, no test collection", testname)
 305                                 return
 306                         }
 307                         req, err := http.NewRequest("GET", trial.fileurl, nil)
 308                         if err != nil {
 309                                 errorf("%s: %s", testname, err)
 310                                 return
 311                         }
 312                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 313                         resp, err := http.DefaultClient.Do(req)
 314                         if err != nil {
 315                                 errorf("%s: %s", testname, err)
 316                                 return
 317                         }
 318                         defer resp.Body.Close()
 319                         body, err := ioutil.ReadAll(resp.Body)
 320                         if err != nil {
 321                                 errorf("%s: error reading response: %s", testname, err)
 322                         }
 323                         if resp.StatusCode != trial.status {
 324                                 errorf("%s: unexpected response status: %s", testname, resp.Status)
 325                         } else if trial.status == http.StatusOK && string(body) != "testfiledata" {
 326                                 errorf("%s: unexpected response content: %q", testname, body)
 327                         } else {
 328                                 infof("%s: ok", testname)
 329                         }
 330                 }()
 331         }
 332
 333         var vm arvados.VirtualMachine
 334         var vmlist arvados.VirtualMachineList
 335         testname = "getting list of virtual machines"
 336         logger.Info(testname)
 337         err = client.RequestAndDecode(&vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
 338         if err != nil {
 339                 errorf("%s: %s", testname, err)
 340         } else if len(vmlist.Items) < 1 {
 341                 errorf("%s: none found", testname)
 342         } else {
 343                 vm = vmlist.Items[0]
 344                 infof("%s: ok", testname)
 345         }
 346
 347         testname = "getting workbench1 webshell page"
 348         logger.Info(testname)
 349         func() {
 350                 if vm.UUID == "" {
 351                         errorf("%s: skipping, no vm available", testname)
 352                         return
 353                 }
 354                 webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername"
 355                 debugf("%s: url %s", testname, webshelltermurl)
 356                 req, err := http.NewRequest("GET", webshelltermurl, nil)
 357                 if err != nil {
 358                         errorf("%s: %s", testname, err)
 359                         return
 360                 }
 361                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 362                 resp, err := http.DefaultClient.Do(req)
 363                 if err != nil {
 364                         errorf("%s: %s", testname, err)
 365                         return
 366                 }
 367                 defer resp.Body.Close()
 368                 body, err := ioutil.ReadAll(resp.Body)
 369                 if err != nil {
 370                         errorf("%s: error reading response: %s", testname, err)
 371                 }
 372                 if resp.StatusCode != http.StatusOK {
 373                         errorf("%s: unexpected response status: %s %q", testname, resp.Status, body)
 374                         return
 375                 }
 376                 infof("%s: ok", testname)
 377         }()
 378
 379         testname = "connecting to webshell service"
 380         logger.Info(testname)
 381         func() {
 382                 ctx, cancel := context.WithDeadline(ctx, time.Now().Add(*timeout))
 383                 defer cancel()
 384                 if vm.UUID == "" {
 385                         errorf("%s: skipping, no vm available", testname)
 386                         return
 387                 }
 388                 u := cluster.Services.WebShell.ExternalURL
 389                 webshellurl := u.String() + vm.Hostname + "?"
 390                 if strings.HasPrefix(u.Host, "*") {
 391                         u.Host = vm.Hostname + u.Host[1:]
 392                         webshellurl = u.String() + "?"
 393                 }
 394                 debugf("%s: url %s", testname, webshellurl)
 395                 req, err := http.NewRequestWithContext(ctx, "POST", webshellurl, bytes.NewBufferString(url.Values{
 396                         "width":   {"80"},
 397                         "height":  {"25"},
 398                         "session": {"xyzzy"},
 399                         "rooturl": {webshellurl},
 400                 }.Encode()))
 401                 if err != nil {
 402                         errorf("%s: %s", testname, err)
 403                         return
 404                 }
 405                 req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
 406                 resp, err := http.DefaultClient.Do(req)
 407                 if err != nil {
 408                         errorf("%s: %s", testname, err)
 409                         return
 410                 }
 411                 defer resp.Body.Close()
 412                 debugf("%s: response status %s", testname, resp.Status)
 413                 body, err := ioutil.ReadAll(resp.Body)
 414                 if err != nil {
 415                         errorf("%s: error reading response: %s", testname, err)
 416                 }
 417                 debugf("%s: response body %q", testname, body)
 418                 // We don't speak the protocol, so we get a 400 error
 419                 // from the webshell server even if everything is
 420                 // OK. Anything else (404, 502, ???) indicates a
 421                 // problem.
 422                 if resp.StatusCode != http.StatusBadRequest {
 423                         errorf("%s: unexpected response status: %s, %q", testname, resp.Status, body)
 424                         return
 425                 }
 426                 infof("%s: ok", testname)
 427         }()
 428         return 0
 429 }