lib/diagnostics/cmd.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package diagnostics
   6
   7 import (
   8         "bytes"
   9         "flag"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net/http"
  14         "strings"
  15         "time"
  16
  17         "git.arvados.org/arvados.git/sdk/go/arvados"
  18         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  19         "github.com/sirupsen/logrus"
  20 )
  21
  22 type Command struct {
  23         projectName string
  24 }
  25
  26 func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
  27         f := flag.NewFlagSet(prog, flag.ContinueOnError)
  28         f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects")
  29         loglevel := f.String("log-level", "info", "logging level (debug, info, warning, error)")
  30         checkInternal := f.Bool("internal-client", false, "check that this host is considered an \"internal\" client")
  31         checkExternal := f.Bool("external-client", false, "check that this host is considered an \"external\" client")
  32         err := f.Parse(args)
  33         if err == flag.ErrHelp {
  34                 return 0
  35         } else if err != nil {
  36                 fmt.Fprintln(stderr, err)
  37                 return 2
  38         }
  39
  40         logger := ctxlog.New(stdout, "text", *loglevel)
  41         logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true})
  42
  43         infof := logger.Infof
  44         warnf := logger.Warnf
  45         var errors []string
  46         errorf := func(f string, args ...interface{}) {
  47                 logger.Errorf(f, args...)
  48                 errors = append(errors, fmt.Sprintf(f, args...))
  49         }
  50         defer func() {
  51                 if len(errors) == 0 {
  52                         logger.Info("--- no errors ---")
  53                 } else {
  54                         fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n")
  55                         for _, e := range errors {
  56                                 logger.Error(e)
  57                         }
  58                 }
  59         }()
  60
  61         client := arvados.NewClientFromEnv()
  62
  63         var dd arvados.DiscoveryDocument
  64         ddpath := "discovery/v1/apis/arvados/v1/rest"
  65         testname := fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath)
  66         logger.Info(testname)
  67         err = client.RequestAndDecode(&dd, "GET", ddpath, nil, nil)
  68         if err != nil {
  69                 errorf("%s: %s", testname, err)
  70         } else {
  71                 infof("%s: ok, BlobSignatureTTL is %d", testname, dd.BlobSignatureTTL)
  72         }
  73
  74         var cluster arvados.Cluster
  75         cfgpath := "arvados/v1/config"
  76         testname = fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath)
  77         logger.Info(testname)
  78         err = client.RequestAndDecode(&cluster, "GET", cfgpath, nil, nil)
  79         if err != nil {
  80                 errorf("%s: %s", testname, err)
  81         } else {
  82                 infof("%s: ok, Collections.BlobSigning = %v", testname, cluster.Collections.BlobSigning)
  83         }
  84
  85         var user arvados.User
  86         testname = "getting current user record"
  87         logger.Info(testname)
  88         err = client.RequestAndDecode(&user, "GET", "arvados/v1/users/current", nil, nil)
  89         if err != nil {
  90                 errorf("%s: %s", testname, err)
  91                 return 2
  92         } else {
  93                 infof("%s: ok, uuid = %s", testname, user.UUID)
  94         }
  95
  96         // uncomment to create some spurious errors
  97         // cluster.Services.WebDAVDownload.ExternalURL.Host = "0.0.0.0:9"
  98
  99         // TODO: detect routing errors here, like finding wb2 at the
 100         // wb1 address.
 101         for _, svc := range []*arvados.Service{
 102                 &cluster.Services.Keepproxy,
 103                 &cluster.Services.WebDAV,
 104                 &cluster.Services.WebDAVDownload,
 105                 &cluster.Services.Websocket,
 106                 &cluster.Services.Workbench1,
 107                 &cluster.Services.Workbench2,
 108         } {
 109                 testname = fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL)
 110                 logger.Info(testname)
 111                 u := svc.ExternalURL
 112                 if strings.HasPrefix(u.Scheme, "ws") {
 113                         // We can do a real websocket test elsewhere,
 114                         // but for now we'll just check the https
 115                         // connection.
 116                         u.Scheme = "http" + u.Scheme[2:]
 117                 }
 118                 if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
 119                         u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
 120                 }
 121                 req, err := http.NewRequest(http.MethodGet, u.String(), nil)
 122                 if err != nil {
 123                         errorf("%s: %s", testname, err)
 124                         continue
 125                 }
 126                 resp, err := http.DefaultClient.Do(req)
 127                 if err != nil {
 128                         errorf("%s: %s", testname, err)
 129                         continue
 130                 }
 131                 resp.Body.Close()
 132                 infof("%s: ok", testname)
 133         }
 134
 135         for _, url := range []string{
 136                 cluster.Services.Controller.ExternalURL.String(),
 137                 cluster.Services.Keepproxy.ExternalURL.String() + "d41d8cd98f00b204e9800998ecf8427e+0",
 138                 cluster.Services.WebDAVDownload.ExternalURL.String(),
 139         } {
 140                 testname = fmt.Sprintf("checking CORS headers at %s", url)
 141                 logger.Info(testname)
 142                 req, err := http.NewRequest("GET", url, nil)
 143                 if err != nil {
 144                         errorf("%s: %s", testname, err)
 145                         continue
 146                 }
 147                 req.Header.Set("Origin", "https://example.com")
 148                 resp, err := http.DefaultClient.Do(req)
 149                 if err != nil {
 150                         errorf("%s: %s", testname, err)
 151                         continue
 152                 }
 153                 if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" {
 154                         warnf("%s: expected \"Access-Control-Allow-Origin: *\", got %q", testname, hdr)
 155                 } else {
 156                         infof("%s: ok", testname)
 157                 }
 158         }
 159
 160         var keeplist arvados.KeepServiceList
 161         testname = "checking internal/external client detection"
 162         logger.Info(testname)
 163         err = client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: -1})
 164         if err != nil {
 165                 errorf("%s: error getting keep services list: %s", testname, err)
 166         } else if len(keeplist.Items) == 0 {
 167                 errorf("%s: controller did not return any keep services", testname)
 168         } else {
 169                 found := map[string]int{}
 170                 for _, ks := range keeplist.Items {
 171                         found[ks.ServiceType]++
 172                 }
 173                 infof := infof
 174                 isInternal := found["proxy"] == 0 && len(keeplist.Items) > 0
 175                 isExternal := found["proxy"] > 0 && found["proxy"] == len(keeplist.Items)
 176                 if (*checkInternal && !isInternal) || (*checkExternal && !isExternal) {
 177                         infof = errorf
 178                 }
 179                 if isExternal {
 180                         infof("%s: controller returned only proxy services, this host is considered \"external\"", testname)
 181                 } else if isInternal {
 182                         infof("%s: controller returned only non-proxy services, this host is considered \"internal\"", testname)
 183                 } else {
 184                         errorf("%s: controller returned both proxy and non-proxy services: %v", testname, found)
 185                 }
 186         }
 187
 188         var project arvados.Group
 189         var grplist arvados.GroupList
 190         testname = fmt.Sprintf("finding/creating %q project", diag.projectName)
 191         logger.Info(testname)
 192         err = client.RequestAndDecode(&grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
 193                 Filters: []arvados.Filter{
 194                         {"name", "=", diag.projectName},
 195                         {"group_class", "=", "project"},
 196                         {"owner_uuid", "=", user.UUID}},
 197                 Limit: -1})
 198         if err != nil {
 199                 errorf("%s: list groups: %s", testname, err)
 200         } else if len(grplist.Items) < 1 {
 201                 infof("%s: list groups: ok, no results", testname)
 202                 err = client.RequestAndDecode(&project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
 203                         "name":        diag.projectName,
 204                         "group_class": "project",
 205                 }})
 206                 if err != nil {
 207                         errorf("%s: create project: %s", testname, err)
 208                 } else {
 209                         infof("%s: created project, uuid = %s", testname, project.UUID)
 210                 }
 211         } else {
 212                 project = grplist.Items[0]
 213                 infof("%s: ok, using existing project, uuid = %s", testname, project.UUID)
 214         }
 215
 216         testname = "creating temporary collection"
 217         logger.Info(testname)
 218         var collection arvados.Collection
 219         err = client.RequestAndDecode(&collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 220                 "ensure_unique_name": true,
 221                 "collection": map[string]interface{}{
 222                         "name":     "test collection",
 223                         "trash_at": time.Now().Add(time.Hour)}})
 224         if err != nil {
 225                 errorf("%s: %s", testname, err)
 226         } else {
 227                 infof("%s: ok, uuid = %s", testname, collection.UUID)
 228                 defer func() {
 229                         testname := "deleting temporary collection"
 230                         logger.Info(testname)
 231                         err := client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
 232                         if err != nil {
 233                                 errorf("%s: %s", testname, err)
 234                         } else {
 235                                 infof("%s: ok", testname)
 236                         }
 237                 }()
 238         }
 239
 240         testname = "uploading file via webdav"
 241         logger.Info(testname)
 242         func() {
 243                 if collection.UUID == "" {
 244                         infof("%s: skipping, no test collection", testname)
 245                         return
 246                 }
 247                 req, err := http.NewRequest("PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata"))
 248                 if err != nil {
 249                         errorf("%s: BUG? http.NewRequest: %s", testname, err)
 250                         return
 251                 }
 252                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 253                 resp, err := http.DefaultClient.Do(req)
 254                 if err != nil {
 255                         errorf("%s: error performing http request: %s", testname, err)
 256                         return
 257                 }
 258                 resp.Body.Close()
 259                 if resp.StatusCode != http.StatusCreated {
 260                         errorf("%s: status %s", testname, resp.Status)
 261                         return
 262                 }
 263                 infof("%s: ok, status %s", testname, resp.Status)
 264                 err = client.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
 265                 if err != nil {
 266                         errorf("%s: get updated collection: %s", testname, err)
 267                         return
 268                 }
 269                 infof("%s: get updated collection: ok, pdh %s", testname, collection.PortableDataHash)
 270         }()
 271
 272         davurl := cluster.Services.WebDAV.ExternalURL
 273         testname = fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl)
 274         logger.Info(testname)
 275         if strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.") {
 276                 infof("%s: looks ok", testname)
 277         } else if davurl.Host == "" {
 278                 warnf("%s: host missing - content previews will not work", testname)
 279         } else {
 280                 warnf("%s: host has no leading wildcard - content previews will not work unless TrustAllContent==true", testname)
 281         }
 282
 283         for _, trial := range []struct {
 284                 status  int
 285                 fileurl string
 286         }{
 287                 {http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
 288                 {http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"},
 289                 {http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
 290                 {http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"},
 291                 {http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"},
 292                 {http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
 293         } {
 294                 func() {
 295                         testname := fmt.Sprintf("downloading from webdav (%s)", trial.fileurl)
 296                         logger.Info(testname)
 297                         if collection.UUID == "" {
 298                                 errorf("%s: skipping, no test collection", testname)
 299                                 return
 300                         }
 301                         req, err := http.NewRequest("GET", trial.fileurl, nil)
 302                         if err != nil {
 303                                 errorf("%s: %s", testname, err)
 304                                 return
 305                         }
 306                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 307                         resp, err := http.DefaultClient.Do(req)
 308                         if err != nil {
 309                                 errorf("%s: %s", testname, err)
 310                                 return
 311                         }
 312                         defer resp.Body.Close()
 313                         body, err := ioutil.ReadAll(resp.Body)
 314                         if err != nil {
 315                                 errorf("%s: error reading response: %s", testname, err)
 316                         }
 317                         if resp.StatusCode != trial.status {
 318                                 errorf("%s: unexpected response status: %s", testname, resp.Status)
 319                         } else if trial.status == http.StatusOK && string(body) != "testfiledata" {
 320                                 errorf("%s: unexpected response content: %q", testname, body)
 321                         } else {
 322                                 infof("%s: ok", testname)
 323                         }
 324                 }()
 325         }
 326
 327         return 0
 328 }