X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/c1fa7d1c6840ce03e763191c92d3548da8494388..a88aa282a8da52589053244f4ef41283d51f629a:/lib/diagnostics/cmd.go diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go index b0241b3ae4..799abf9da4 100644 --- a/lib/diagnostics/cmd.go +++ b/lib/diagnostics/cmd.go @@ -5,8 +5,10 @@ package diagnostics import ( + "archive/tar" "bytes" "context" + _ "embed" "flag" "fmt" "io" @@ -17,6 +19,7 @@ import ( "strings" "time" + "git.arvados.org/arvados.git/lib/cmd" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/ctxlog" "github.com/sirupsen/logrus" @@ -24,21 +27,18 @@ import ( type Command struct{} -func (cmd Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { +func (Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { var diag diagnoser f := flag.NewFlagSet(prog, flag.ContinueOnError) f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects") f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)") + f.StringVar(&diag.dockerImage, "docker-image", "", "image to use when running a test container (default: use embedded hello-world image)") f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client") f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client") f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000, or 0 to skip)") f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests") - err := f.Parse(args) - if err == flag.ErrHelp { - return 0 - } else if err != nil { - fmt.Fprintln(stderr, err) - return 2 + if ok, code := cmd.ParseFlags(f, prog, args, "", stderr); !ok { + return code } diag.logger = ctxlog.New(stdout, "text", diag.logLevel) diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true, PadLevelText: true}) @@ -57,12 +57,17 @@ func (cmd Command) RunCommand(prog string, args []string, stdin io.Reader, stdou } } +// docker save hello-world > hello-world.tar +//go:embed hello-world.tar +var HelloWorldDockerImage []byte + type diagnoser struct { stdout io.Writer stderr io.Writer logLevel string priority int projectName string + dockerImage string checkInternal bool checkExternal bool timeout time.Duration @@ -135,6 +140,7 @@ func (diag *diagnoser) runtests() { var cluster arvados.Cluster cfgpath := "arvados/v1/config" + cfgOK := false diag.dotest(20, fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath), func() error { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) defer cancel() @@ -143,6 +149,7 @@ func (diag *diagnoser) runtests() { return err } diag.debugf("Collections.BlobSigning = %v", cluster.Collections.BlobSigning) + cfgOK = true return nil }) @@ -158,6 +165,11 @@ func (diag *diagnoser) runtests() { return nil }) + if !cfgOK { + diag.errorf("cannot proceed without cluster config -- aborting without running any further tests") + return + } + // uncomment to create some spurious errors // cluster.Services.WebDAVDownload.ExternalURL.Host = "0.0.0.0:9" @@ -362,13 +374,38 @@ func (diag *diagnoser) runtests() { }() } + // Read hello-world.tar to find image ID, so we can upload it + // as "sha256:{...}.tar" + var imageSHA2 string + { + tr := tar.NewReader(bytes.NewReader(HelloWorldDockerImage)) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + diag.errorf("internal error/bug: cannot read embedded docker image tar file: %s", err) + return + } + if s := strings.TrimSuffix(hdr.Name, ".json"); len(s) == 64 && s != hdr.Name { + imageSHA2 = s + } + } + if imageSHA2 == "" { + diag.errorf("internal error/bug: cannot find {sha256}.json file in embedded docker image tar file") + return + } + } + tarfilename := "sha256:" + imageSHA2 + ".tar" + diag.dotest(100, "uploading file via webdav", func() error { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) defer cancel() if collection.UUID == "" { return fmt.Errorf("skipping, no test collection") } - req, err := http.NewRequestWithContext(ctx, "PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata")) + req, err := http.NewRequestWithContext(ctx, "PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/"+tarfilename, bytes.NewReader(HelloWorldDockerImage)) if err != nil { return fmt.Errorf("BUG? http.NewRequest: %s", err) } @@ -391,29 +428,35 @@ func (diag *diagnoser) runtests() { }) davurl := cluster.Services.WebDAV.ExternalURL + davWildcard := strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.") diag.dotest(110, fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl), func() error { if davurl.Host == "" { return fmt.Errorf("host missing - content previews will not work") } - if !strings.HasPrefix(davurl.Host, "*--") && !strings.HasPrefix(davurl.Host, "*.") && !cluster.Collections.TrustAllContent { + if !davWildcard && !cluster.Collections.TrustAllContent { diag.warnf("WebDAV ExternalURL has no leading wildcard and TrustAllContent==false - content previews will not work") } return nil }) for i, trial := range []struct { - needcoll bool - status int - fileurl string + needcoll bool + needWildcard bool + status int + fileurl string }{ - {false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"}, - {false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"}, - {false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"}, - {false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"}, - {true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"}, - {true, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"}, + {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"}, + {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + tarfilename}, + {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"}, + {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/" + tarfilename}, + {true, true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + tarfilename}, + {true, false, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/" + tarfilename}, } { diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error { + if trial.needWildcard && !davWildcard { + diag.warnf("skipping collection-id-in-vhost test because WebDAV ExternalURL has no leading wildcard") + return nil + } ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) defer cancel() if trial.needcoll && collection.UUID == "" { @@ -436,8 +479,13 @@ func (diag *diagnoser) runtests() { if resp.StatusCode != trial.status { return fmt.Errorf("unexpected response status: %s", resp.Status) } - if trial.status == http.StatusOK && string(body) != "testfiledata" { - return fmt.Errorf("unexpected response content: %q", body) + if trial.status == http.StatusOK && !bytes.Equal(body, HelloWorldDockerImage) { + excerpt := body + if len(excerpt) > 128 { + excerpt = append([]byte(nil), body[:128]...) + excerpt = append(excerpt, []byte("[...]")...) + } + return fmt.Errorf("unexpected response content: len %d, %q", len(body), excerpt) } return nil }) @@ -453,9 +501,10 @@ func (diag *diagnoser) runtests() { return err } if len(vmlist.Items) < 1 { - return fmt.Errorf("no VMs found") + diag.warnf("no VMs found") + } else { + vm = vmlist.Items[0] } - vm = vmlist.Items[0] return nil }) @@ -463,7 +512,8 @@ func (diag *diagnoser) runtests() { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) defer cancel() if vm.UUID == "" { - return fmt.Errorf("skipping, no vm available") + diag.warnf("skipping, no vm available") + return nil } webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername" diag.debugf("url %s", webshelltermurl) @@ -491,7 +541,8 @@ func (diag *diagnoser) runtests() { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) defer cancel() if vm.UUID == "" { - return fmt.Errorf("skipping, no vm available") + diag.warnf("skipping, no vm available") + return nil } u := cluster.Services.WebShell.ExternalURL webshellurl := u.String() + vm.Hostname + "?" @@ -540,16 +591,25 @@ func (diag *diagnoser) runtests() { return fmt.Errorf("skipping, no project to work in") } + timestamp := time.Now().Format(time.RFC3339) + ctrCommand := []string{"echo", timestamp} + if diag.dockerImage == "" { + if collection.UUID == "" { + return fmt.Errorf("skipping, no test collection to use as docker image") + } + diag.dockerImage = collection.PortableDataHash + ctrCommand = []string{"/hello"} + } + var cr arvados.ContainerRequest ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) defer cancel() - timestamp := time.Now().Format(time.RFC3339) err := client.RequestAndDecodeContext(ctx, &cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{ "owner_uuid": project.UUID, "name": fmt.Sprintf("diagnostics container request %s", timestamp), - "container_image": "arvados/jobs", - "command": []string{"echo", timestamp}, + "container_image": diag.dockerImage, + "command": ctrCommand, "use_existing": false, "output_path": "/mnt/output", "output_name": fmt.Sprintf("diagnostics output %s", timestamp),