From 32b402114038bc6761c04c370afad786dbbb3125 Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Fri, 22 Jul 2022 14:54:32 -0400 Subject: [PATCH] 19281: Use embedded hello-world.tar as default diagnostics image. Ensures diagnostics work even if no docker images have been uploaded yet and docker.io wasn't installed at "init" time. Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- lib/diagnostics/cmd.go | 58 ++++++++++++++++++++++++++++---- lib/diagnostics/hello-world.tar | Bin 0 -> 24064 bytes lib/install/init.go | 23 ------------- 3 files changed, 51 insertions(+), 30 deletions(-) create mode 100644 lib/diagnostics/hello-world.tar diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go index 3455d3307e..b381a3c8c1 100644 --- a/lib/diagnostics/cmd.go +++ b/lib/diagnostics/cmd.go @@ -5,8 +5,10 @@ package diagnostics import ( + "archive/tar" "bytes" "context" + _ "embed" "flag" "fmt" "io" @@ -30,7 +32,7 @@ func (Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s f := flag.NewFlagSet(prog, flag.ContinueOnError) f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects") f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)") - f.StringVar(&diag.dockerImage, "docker-image", "alpine:latest", "image to use when running a test container") + f.StringVar(&diag.dockerImage, "docker-image", "", "image to use when running a test container (default: use embedded hello-world image)") f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client") f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client") f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000, or 0 to skip)") @@ -55,6 +57,10 @@ func (Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s } } +// docker save hello-world > hello-world.tar +//go:embed hello-world.tar +var helloWorldDockerImage []byte + type diagnoser struct { stdout io.Writer stderr io.Writer @@ -368,13 +374,37 @@ func (diag *diagnoser) runtests() { }() } + // Read hello-world.tar to find image ID, so we can upload it + // as "sha256:{...}.tar" + var imageSHA2 string + { + tr := tar.NewReader(bytes.NewReader(helloWorldDockerImage)) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + diag.errorf("internal error/bug: cannot read embedded docker image tar file: %s", err) + return + } + if s := strings.TrimSuffix(hdr.Name, ".json"); len(s) == 64 && s != hdr.Name { + imageSHA2 = s + } + } + if imageSHA2 == "" { + diag.errorf("internal error/bug: cannot find {sha256}.json file in embedded docker image tar file") + return + } + } + diag.dotest(100, "uploading file via webdav", func() error { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) defer cancel() if collection.UUID == "" { return fmt.Errorf("skipping, no test collection") } - req, err := http.NewRequestWithContext(ctx, "PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata")) + req, err := http.NewRequestWithContext(ctx, "PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/sha256:"+imageSHA2+".tar", bytes.NewReader(helloWorldDockerImage)) if err != nil { return fmt.Errorf("BUG? http.NewRequest: %s", err) } @@ -419,7 +449,7 @@ func (diag *diagnoser) runtests() { {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"}, {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"}, {true, true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"}, - {true, false, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"}, + {true, false, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/sha256:" + imageSHA2 + ".tar"}, } { diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error { if trial.needWildcard && !davWildcard { @@ -448,8 +478,13 @@ func (diag *diagnoser) runtests() { if resp.StatusCode != trial.status { return fmt.Errorf("unexpected response status: %s", resp.Status) } - if trial.status == http.StatusOK && string(body) != "testfiledata" { - return fmt.Errorf("unexpected response content: %q", body) + if trial.status == http.StatusOK && !bytes.Equal(body, helloWorldDockerImage) { + excerpt := body + if len(excerpt) > 128 { + excerpt = append([]byte(nil), body[:128]...) + excerpt = append(excerpt, []byte("[...]")...) + } + return fmt.Errorf("unexpected response content: len %d, %q", len(body), excerpt) } return nil }) @@ -555,16 +590,25 @@ func (diag *diagnoser) runtests() { return fmt.Errorf("skipping, no project to work in") } + timestamp := time.Now().Format(time.RFC3339) + ctrCommand := []string{"echo", timestamp} + if diag.dockerImage == "" { + if collection.UUID == "" { + return fmt.Errorf("skipping, no test collection to use as docker image") + } + diag.dockerImage = collection.PortableDataHash + ctrCommand = []string{"/hello"} + } + var cr arvados.ContainerRequest ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) defer cancel() - timestamp := time.Now().Format(time.RFC3339) err := client.RequestAndDecodeContext(ctx, &cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{ "owner_uuid": project.UUID, "name": fmt.Sprintf("diagnostics container request %s", timestamp), "container_image": diag.dockerImage, - "command": []string{"echo", timestamp}, + "command": ctrCommand, "use_existing": false, "output_path": "/mnt/output", "output_name": fmt.Sprintf("diagnostics output %s", timestamp), diff --git a/lib/diagnostics/hello-world.tar b/lib/diagnostics/hello-world.tar new file mode 100644 index 0000000000000000000000000000000000000000..60b50ea252021af8f463aaf0c70357d31c3e2191 GIT binary patch literal 24064 zcmeHPe{37o9lx}=F`*@?Xdx0~9L8t?QrmaupBaUiI`BNXYiJYbnnvmQ?p^I+`@;F0 zG%L|sxt%ypmb!m8Y2yzjZQ`E|@kiSs>Ncfm(KHR%A01;`#ZPyE(y0gvg?Qh0KTO;v%e+=0cGe zAB>1%5C_F*L}WP3#;_kl?2lPJsZZ4uzie4Hg~j^+;hxP~lAC@!pSb}h5C{i@v;A(D0Xc4#{~3-8 zQ}m*JGM`a)+jrdmjPGkf*+$!;-!KFgz5ah%lnV=tX@T|6GU3qd_0L2?&iY>f)+K4) z?a7LsF3)18fv3|WD416P82x!Q4)n;Cqjg6 zR5t{QrR+{m*KQAzi)w+-<5@ibcS6oX-&Pxx`-_Ghkc+D-FJ!i;Vz%JvlK5N^H`K`` z^^28QfgxC&ed^due$y~6T&c$>Z7K+0kj39q81ZxsczWY~XuZEwRQx$v;B)?BI$P*! zD9;oxHyBlB5wf1aPS1us;6C8-L(yN%z);o+aJjvsAP?V3LvF*y1U<0bKG85@?f1pw0I*i^W` zl+A(ZW--O5aSp_HkB@hHL|J$gD?=k#DQ0Ex2h1uT6Yxbq(VSj}`fsym4bOeE-A0@U zj^h;}lU1>xmJ~9*_`Dbnf^%eWtS6T(ly-T>m*mdnC|L>w7JK9O=lC(K_{bjWOVj+{ z|6J&c`#;zSMx5{eF9mc78;twP66%{v74d%re$dU{BAaE7i7*ZO|6rH`el`+x{680# zOI`Q#p40=&nwnIHnE$EQY)x#Y8AE8;JwD+v@f=VNe@`6 zZA!J7e5Be%vR0Y0`m^bpW|PmRHXo5Q`lHQuitaWu7R@a;i#BbPTLDcq>kq5ieWsjE z?=i)wKQ6N}mR%3Mt%^kMYLh*j%^*kY{IdSqErm95)wO7{YRK=es@K-fwBEXhlqM_+{Jg;* z+D6Eo(?~myj-I;@9lf|5HJv~&PpfMh6u9eWq}E##m*w}YkAWOj^7o_i`geSgMcUiy zN~Ev<6biL}ojX#mW$N|%3D;`iYZ|oswdvX2&^WOLX`eFHLn$J@_q|>~163aP(ifhA zu4~=hptFssB7NmCRIj4_r%E59$~DK3=2}5)*4xqkf5I@!m+lz77U{iir1iZ{qjI%v z>P@8Y^FC!5<$4zEAw`;K2P~RF{uf4F!rKe^H%NZ}N2ME){+zenFu}EHwNZy!6X(H_ zUSi4Qv8{$d0i>}{u0Zt@`>N`Sd;FyfQ|AnuYwv>}kp8@vIIZS^(UDH~5F26%@2=7< zqkq)6C@(DALs9XM5~o+t>UBu_TdNDk3_Pel0YEJ; zy>-HMY#lfR?LAK59+=otV?GC4Cnv~1I%w~u^FWI3`VFW&;hy?nU?Sch?@MZb?*~_1 z>P5l{q`jD`jRc*5N$!GO?ydPMS+UNZ+l4IbI7sagP+Xv%=^)oOVCMPalwqm3Om8aZHwSL3Y`%HD2Riy8~&ew*%>8Zlpif zwozYATH6q)0Lva~Pk@*^UcTrbUDKl-?a`|7@=0|;J+*8-5bijXO1-VI|Z3!JJ zH|kWX=~TS@iGRd}prS3MUkf+`1pCMwC?DTSCiW$y-B&va6>+UvK3VVCU){C2)O^Af z?*#ZKH$;ik4!%Z4P+5O3%)wTdwjluMY=d^Bm49!vl*vqil0kIWU+*TR0Wf}`9cdeA zfCU_ss>tqmUq8}5-|`UBU7r9`O26Y%!mHo%CLxu5%~1e7`SI(?+m4e}(~;87pu0B! zDn%IByGgAFc(=4A%P+do-Mc$#|0e4|zj^9a(09n_vmmeYhh5lP7j=eoH{UQwG9noEa1(f(J|9;Dw-qvbyWj8+X9mb-QsNZj9nCS7NN zi!3Hs_nXaHSkJCaBwKNHnCyQdn)Fqy(eAq0AZt4bX*!8MK8-3$9o=@G6g1Z=Q{+ZM z%AYomUI%@A4)Si&hraeBqN=jKl{8nLYx|GU^V%Jxw*1aTq=hz;?A~dz6i2=vKkNa9 zB$+JCDW-n#S|}YRHv@BW-9T0kOtPn7y&7wJ>%iay66y!vHCPUlzhaFQKlKS&vY(;7 zb?SV*KGIs-3@!1GUr%Usviz4i_%XhH0#*@N>{ix%JV|aW>dL24{UtLGw^{9FboA`< z3+Q*{H=0$~)Ag51mq23r$TGOa9Q=*Z--L+tYwmz7jHMc!nEi!F8l2=H;2_{2;2_{2 z;2_{2@GU~11(73xw4}&+I$<2SyRD@!lP%J4kgo^_1nFW%9xc*phF9q^xunq90=zJh z%ke6l%A@(=VazMgEYoRBkD3SmXhBvKIRBL!^R=|}$@ESPCq3b4C@8tCx6|qkMCC`J z1=ErYhK$02LAt2o;bK<{&G_g(XfVy9gv7s36!a_;q(>a0(Fb*rSSQ40CKU;sS=F!z_zdr=SjlG#e9rV7 zoY$m_avp=@z%6+h%;EFtEV#Z@>?8uTz!%8SXdwORQbC0p`)n^gnpHF4MK3KH^UB~` zmsRPK$(ZJgnU7itohgI`V1|%rsC% z`9(b9Cnp@OwZF!qO+VXAHSKC^x_M=@`vB~INd5QV^V)m$I@twL@$1`mTu-)9PI3@% z5O5H15O5H15O5H15O5H15O5IqdJyPd-EDSc(*?S1@3#0Uc)sT=W?**-8&=AfEo4>7 zr{Z03m7qZRVDHMSJmtfgAxVJ^sL3T$0`vhi@Jbr3SgtsxlgYqln+bsDu>p*{#^*(|vV^!Nl>ZV!pn(kZo;{Cy(DrI!_csviZ!?<+5G>AT$vB?f%L0bCNDDAa5|ig&e(T zl&9?Z8!$QS_bhCFyI-63&yu(2f4e2W(iXI+EoWB`K!()W_IF$TdoB5`GmbSSXp_ub zudsclmZEI=9SwCPOZc7SAmAY2Ag~w&B%BV3F$wcwK7?c8K$wTu{6jbziHXrLn~t)f zSU8mCgHaI&LhzzLpN9AUxlmXXASeLDZ1OEZ{Esla8+v8@Cx#2K&ikK>@$1#f?*6j( z^_?IK5U=2y7aPM0vM}?M1&nkes5lW+=8B;56$0{@kIm)`wgS&=Rat@1HUqmC2=PPW z#l!;97;A+1F{1$ur9sk;OGB>{Xe%bohE2Ucp`~mNcZC9>FvrIt@Y25|Mg+_V7^g)> z;)7B!8e@2Y6R{vlK{m>9oRnt6Q7#+{OF=Fa<)yH|!UJjAPkY8Y7c%e42t@OnM}Hw; zPNxUwht;tmBix0jOI1pE9D?XHL^bL3D6*_dBz}!Y9UCUGPYmWF1V0h8k~EYRp>4pv zr{F*YV