1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
20 "git.arvados.org/arvados.git/sdk/go/arvados"
21 "git.arvados.org/arvados.git/sdk/go/ctxlog"
22 "github.com/sirupsen/logrus"
27 func (cmd Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
29 f := flag.NewFlagSet(prog, flag.ContinueOnError)
30 f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects")
31 f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
32 f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client")
33 f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client")
34 f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests")
36 if err == flag.ErrHelp {
38 } else if err != nil {
39 fmt.Fprintln(stderr, err)
42 diag.logger = ctxlog.New(stdout, "text", diag.logLevel)
43 diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true})
45 if len(diag.errors) == 0 {
46 diag.logger.Info("--- no errors ---")
49 if diag.logger.Level > logrus.ErrorLevel {
50 fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n")
51 for _, e := range diag.errors {
59 type diagnoser struct {
72 func (diag *diagnoser) debugf(f string, args ...interface{}) {
73 diag.logger.Debugf(f, args...)
76 func (diag *diagnoser) infof(f string, args ...interface{}) {
77 diag.logger.Infof(f, args...)
80 func (diag *diagnoser) warnf(f string, args ...interface{}) {
81 diag.logger.Warnf(f, args...)
84 func (diag *diagnoser) errorf(f string, args ...interface{}) {
85 diag.logger.Errorf(f, args...)
86 diag.errors = append(diag.errors, fmt.Sprintf(f, args...))
89 // Run the given func, logging appropriate messages before and after,
90 // adding timing info, etc.
92 // The id argument should be unique among tests, and shouldn't change
93 // when other tests are added/removed.
94 func (diag *diagnoser) dotest(id int, title string, fn func() error) {
96 diag.done = map[int]bool{}
97 } else if diag.done[id] {
98 diag.errorf("(bug) reused test id %d", id)
102 diag.infof("%d %s", id, title)
105 elapsed := fmt.Sprintf("%.0dms", time.Now().Sub(t0)/time.Millisecond)
107 diag.errorf("%s (%s): %s", title, elapsed, err)
109 diag.debugf("%d %s (%s): ok", id, title, elapsed)
112 func (diag *diagnoser) runtests() {
113 client := arvados.NewClientFromEnv()
115 if client.APIHost == "" || client.AuthToken == "" {
116 diag.errorf("ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables are not set -- aborting without running any tests")
120 var dd arvados.DiscoveryDocument
121 ddpath := "discovery/v1/apis/arvados/v1/rest"
122 diag.dotest(10, fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath), func() error {
123 err := client.RequestAndDecode(&dd, "GET", ddpath, nil, nil)
127 diag.debugf("BlobSignatureTTL = %d", dd.BlobSignatureTTL)
131 var cluster arvados.Cluster
132 cfgpath := "arvados/v1/config"
133 diag.dotest(20, fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath), func() error {
134 err := client.RequestAndDecode(&cluster, "GET", cfgpath, nil, nil)
138 diag.debugf("Collections.BlobSigning = %v", cluster.Collections.BlobSigning)
142 var user arvados.User
143 diag.dotest(30, "getting current user record", func() error {
144 err := client.RequestAndDecode(&user, "GET", "arvados/v1/users/current", nil, nil)
148 diag.debugf("user uuid = %s", user.UUID)
152 // uncomment to create some spurious errors
153 // cluster.Services.WebDAVDownload.ExternalURL.Host = "0.0.0.0:9"
155 // TODO: detect routing errors here, like finding wb2 at the
157 for i, svc := range []*arvados.Service{
158 &cluster.Services.Keepproxy,
159 &cluster.Services.WebDAV,
160 &cluster.Services.WebDAVDownload,
161 &cluster.Services.Websocket,
162 &cluster.Services.Workbench1,
163 &cluster.Services.Workbench2,
165 diag.dotest(40+i, fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL), func() error {
167 if strings.HasPrefix(u.Scheme, "ws") {
168 // We can do a real websocket test elsewhere,
169 // but for now we'll just check the https
171 u.Scheme = "http" + u.Scheme[2:]
173 if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
174 u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
176 req, err := http.NewRequest(http.MethodGet, u.String(), nil)
180 resp, err := http.DefaultClient.Do(req)
189 for i, url := range []string{
190 cluster.Services.Controller.ExternalURL.String(),
191 cluster.Services.Keepproxy.ExternalURL.String() + "d41d8cd98f00b204e9800998ecf8427e+0",
192 cluster.Services.WebDAVDownload.ExternalURL.String(),
194 diag.dotest(50+i, fmt.Sprintf("checking CORS headers at %s", url), func() error {
195 req, err := http.NewRequest("GET", url, nil)
199 req.Header.Set("Origin", "https://example.com")
200 resp, err := http.DefaultClient.Do(req)
204 if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" {
205 return fmt.Errorf("expected \"Access-Control-Allow-Origin: *\", got %q", hdr)
211 var keeplist arvados.KeepServiceList
212 diag.dotest(60, "checking internal/external client detection", func() error {
213 err := client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: -1})
215 return fmt.Errorf("error getting keep services list: %s", err)
216 } else if len(keeplist.Items) == 0 {
217 return fmt.Errorf("controller did not return any keep services")
219 found := map[string]int{}
220 for _, ks := range keeplist.Items {
221 found[ks.ServiceType]++
223 isInternal := found["proxy"] == 0 && len(keeplist.Items) > 0
224 isExternal := found["proxy"] > 0 && found["proxy"] == len(keeplist.Items)
226 diag.debugf("controller returned only proxy services, this host is treated as \"external\"")
227 } else if isInternal {
228 diag.debugf("controller returned only non-proxy services, this host is treated as \"internal\"")
230 if (diag.checkInternal && !isInternal) || (diag.checkExternal && !isExternal) {
231 return fmt.Errorf("expecting internal=%v external=%v, but found internal=%v external=%v", diag.checkInternal, diag.checkExternal, isInternal, isExternal)
236 for i, ks := range keeplist.Items {
239 Host: net.JoinHostPort(ks.ServiceHost, fmt.Sprintf("%d", ks.ServicePort)),
242 if ks.ServiceSSLFlag {
245 diag.dotest(61+i, fmt.Sprintf("reading+writing via keep service at %s", u.String()), func() error {
246 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
248 req, err := http.NewRequestWithContext(ctx, "PUT", u.String()+"d41d8cd98f00b204e9800998ecf8427e", nil)
252 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
253 resp, err := http.DefaultClient.Do(req)
257 defer resp.Body.Close()
258 body, err := ioutil.ReadAll(resp.Body)
260 return fmt.Errorf("reading response body: %s", err)
262 loc := strings.TrimSpace(string(body))
263 if !strings.HasPrefix(loc, "d41d8") {
264 return fmt.Errorf("unexpected response from write: %q", body)
267 req, err = http.NewRequestWithContext(ctx, "GET", u.String()+loc, nil)
271 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
272 resp, err = http.DefaultClient.Do(req)
276 defer resp.Body.Close()
277 body, err = ioutil.ReadAll(resp.Body)
279 return fmt.Errorf("reading response body: %s", err)
282 return fmt.Errorf("unexpected response from read: %q", body)
289 var project arvados.Group
290 diag.dotest(80, fmt.Sprintf("finding/creating %q project", diag.projectName), func() error {
291 var grplist arvados.GroupList
292 err := client.RequestAndDecode(&grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
293 Filters: []arvados.Filter{
294 {"name", "=", diag.projectName},
295 {"group_class", "=", "project"},
296 {"owner_uuid", "=", user.UUID}},
299 return fmt.Errorf("list groups: %s", err)
301 if len(grplist.Items) > 0 {
302 project = grplist.Items[0]
303 diag.debugf("using existing project, uuid = %s", project.UUID)
306 diag.debugf("list groups: ok, no results")
307 err = client.RequestAndDecode(&project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
308 "name": diag.projectName,
309 "group_class": "project",
312 return fmt.Errorf("create project: %s", err)
314 diag.debugf("created project, uuid = %s", project.UUID)
318 var collection arvados.Collection
319 diag.dotest(90, "creating temporary collection", func() error {
320 err := client.RequestAndDecode(&collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
321 "ensure_unique_name": true,
322 "collection": map[string]interface{}{
323 "name": "test collection",
324 "trash_at": time.Now().Add(time.Hour)}})
328 diag.debugf("ok, uuid = %s", collection.UUID)
332 if collection.UUID != "" {
334 diag.dotest(9990, "deleting temporary collection", func() error {
335 return client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
340 diag.dotest(100, "uploading file via webdav", func() error {
341 if collection.UUID == "" {
342 return fmt.Errorf("skipping, no test collection")
344 req, err := http.NewRequest("PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata"))
346 return fmt.Errorf("BUG? http.NewRequest: %s", err)
348 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
349 resp, err := http.DefaultClient.Do(req)
351 return fmt.Errorf("error performing http request: %s", err)
354 if resp.StatusCode != http.StatusCreated {
355 return fmt.Errorf("status %s", resp.Status)
357 diag.debugf("ok, status %s", resp.Status)
358 err = client.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
360 return fmt.Errorf("get updated collection: %s", err)
362 diag.debugf("ok, pdh %s", collection.PortableDataHash)
366 davurl := cluster.Services.WebDAV.ExternalURL
367 diag.dotest(110, fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl), func() error {
368 if davurl.Host == "" {
369 return fmt.Errorf("host missing - content previews will not work")
371 if !strings.HasPrefix(davurl.Host, "*--") && !strings.HasPrefix(davurl.Host, "*.") && !cluster.Collections.TrustAllContent {
372 diag.warnf("WebDAV ExternalURL has no leading wildcard and TrustAllContent==false - content previews will not work")
377 for i, trial := range []struct {
382 {false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
383 {false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"},
384 {false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
385 {false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"},
386 {true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"},
387 {true, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
389 diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error {
390 if trial.needcoll && collection.UUID == "" {
391 return fmt.Errorf("skipping, no test collection")
393 req, err := http.NewRequest("GET", trial.fileurl, nil)
397 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
398 resp, err := http.DefaultClient.Do(req)
402 defer resp.Body.Close()
403 body, err := ioutil.ReadAll(resp.Body)
405 return fmt.Errorf("reading response: %s", err)
407 if resp.StatusCode != trial.status {
408 return fmt.Errorf("unexpected response status: %s", resp.Status)
410 if trial.status == http.StatusOK && string(body) != "testfiledata" {
411 return fmt.Errorf("unexpected response content: %q", body)
417 var vm arvados.VirtualMachine
418 diag.dotest(130, "getting list of virtual machines", func() error {
419 var vmlist arvados.VirtualMachineList
420 err := client.RequestAndDecode(&vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
424 if len(vmlist.Items) < 1 {
425 return fmt.Errorf("no VMs found")
431 diag.dotest(140, "getting workbench1 webshell page", func() error {
433 return fmt.Errorf("skipping, no vm available")
435 webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername"
436 diag.debugf("url %s", webshelltermurl)
437 req, err := http.NewRequest("GET", webshelltermurl, nil)
441 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
442 resp, err := http.DefaultClient.Do(req)
446 defer resp.Body.Close()
447 body, err := ioutil.ReadAll(resp.Body)
449 return fmt.Errorf("reading response: %s", err)
451 if resp.StatusCode != http.StatusOK {
452 return fmt.Errorf("unexpected response status: %s %q", resp.Status, body)
457 diag.dotest(150, "connecting to webshell service", func() error {
458 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
461 return fmt.Errorf("skipping, no vm available")
463 u := cluster.Services.WebShell.ExternalURL
464 webshellurl := u.String() + vm.Hostname + "?"
465 if strings.HasPrefix(u.Host, "*") {
466 u.Host = vm.Hostname + u.Host[1:]
467 webshellurl = u.String() + "?"
469 diag.debugf("url %s", webshellurl)
470 req, err := http.NewRequestWithContext(ctx, "POST", webshellurl, bytes.NewBufferString(url.Values{
473 "session": {"xyzzy"},
474 "rooturl": {webshellurl},
479 req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
480 resp, err := http.DefaultClient.Do(req)
484 defer resp.Body.Close()
485 diag.debugf("response status %s", resp.Status)
486 body, err := ioutil.ReadAll(resp.Body)
488 return fmt.Errorf("reading response: %s", err)
490 diag.debugf("response body %q", body)
491 // We don't speak the protocol, so we get a 400 error
492 // from the webshell server even if everything is
493 // OK. Anything else (404, 502, ???) indicates a
495 if resp.StatusCode != http.StatusBadRequest {
496 return fmt.Errorf("unexpected response status: %s, %q", resp.Status, body)