lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/md5"
  11         "encoding/json"
  12         "errors"
  13         "fmt"
  14         "io"
  15         "io/fs"
  16         "io/ioutil"
  17         "log"
  18         "math/rand"
  19         "net/http"
  20         "net/http/httptest"
  21         "net/http/httputil"
  22         "net/url"
  23         "os"
  24         "os/exec"
  25         "path"
  26         "regexp"
  27         "runtime/pprof"
  28         "strings"
  29         "sync"
  30         "sync/atomic"
  31         "syscall"
  32         "testing"
  33         "time"
  34
  35         "git.arvados.org/arvados.git/lib/cloud"
  36         "git.arvados.org/arvados.git/lib/cmd"
  37         "git.arvados.org/arvados.git/sdk/go/arvados"
  38         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  39         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  40         "git.arvados.org/arvados.git/sdk/go/manifest"
  41
  42         . "gopkg.in/check.v1"
  43         git_client "gopkg.in/src-d/go-git.v4/plumbing/transport/client"
  44         git_http "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
  45 )
  46
  47 // Gocheck boilerplate
  48 func TestCrunchExec(t *testing.T) {
  49         TestingT(t)
  50 }
  51
  52 const logLineStart = `(?m)(.*\n)*\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d+Z `
  53
  54 var _ = Suite(&TestSuite{})
  55
  56 type TestSuite struct {
  57         client                   *arvados.Client
  58         api                      *ArvTestClient
  59         runner                   *ContainerRunner
  60         executor                 *stubExecutor
  61         keepmount                string
  62         keepmountTmp             []string
  63         testDispatcherKeepClient KeepTestClient
  64         testContainerKeepClient  KeepTestClient
  65         debian12MemoryCurrent    int64
  66         debian12SwapCurrent      int64
  67 }
  68
  69 func (s *TestSuite) SetUpSuite(c *C) {
  70         buf, err := os.ReadFile("../crunchstat/testdata/debian12/sys/fs/cgroup/user.slice/user-1000.slice/session-4.scope/memory.current")
  71         c.Assert(err, IsNil)
  72         _, err = fmt.Sscanf(string(buf), "%d", &s.debian12MemoryCurrent)
  73         c.Assert(err, IsNil)
  74
  75         buf, err = os.ReadFile("../crunchstat/testdata/debian12/sys/fs/cgroup/user.slice/user-1000.slice/session-4.scope/memory.swap.current")
  76         c.Assert(err, IsNil)
  77         _, err = fmt.Sscanf(string(buf), "%d", &s.debian12SwapCurrent)
  78         c.Assert(err, IsNil)
  79 }
  80
  81 func (s *TestSuite) SetUpTest(c *C) {
  82         s.client = arvados.NewClientFromEnv()
  83         s.executor = &stubExecutor{}
  84         var err error
  85         s.api = &ArvTestClient{}
  86         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  87         c.Assert(err, IsNil)
  88         s.runner.executor = s.executor
  89         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  90                 return s.api, &s.testContainerKeepClient, s.client, nil
  91         }
  92         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  93                 s.runner.ArvMountPoint = s.keepmount
  94                 for i, opt := range cmd {
  95                         if opt == "--mount-tmp" {
  96                                 err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700)
  97                                 if err != nil {
  98                                         return nil, err
  99                                 }
 100                                 s.keepmountTmp = append(s.keepmountTmp, cmd[i+1])
 101                         }
 102                 }
 103                 return nil, nil
 104         }
 105         s.keepmount = c.MkDir()
 106         err = os.Mkdir(s.keepmount+"/by_id", 0755)
 107         s.keepmountTmp = nil
 108         c.Assert(err, IsNil)
 109         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
 110         c.Assert(err, IsNil)
 111         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
 112         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
 113         c.Assert(err, IsNil)
 114         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
 115         c.Assert(err, IsNil)
 116         s.runner.ArvMountPoint = s.keepmount
 117         os.Setenv("InstanceType", `{"ProviderType":"a1.2xlarge","Price":1.2}`)
 118 }
 119
 120 type ArvTestClient struct {
 121         Total   int64
 122         Calls   int
 123         Content []arvadosclient.Dict
 124         arvados.Container
 125         secretMounts []byte
 126         Logs         map[string]*bytes.Buffer
 127         sync.Mutex
 128         WasSetRunning bool
 129         callraw       bool
 130 }
 131
 132 type KeepTestClient struct {
 133         Called         bool
 134         StorageClasses []string
 135         blocks         sync.Map
 136 }
 137
 138 type stubExecutor struct {
 139         imageLoaded bool
 140         loaded      string
 141         loadErr     error
 142         exitCode    int
 143         createErr   error
 144         created     containerSpec
 145         startErr    error
 146         waitSleep   time.Duration
 147         waitErr     error
 148         stopErr     error
 149         stopped     bool
 150         closed      bool
 151         runFunc     func() int
 152         exit        chan int
 153 }
 154
 155 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 156         containerClient *arvados.Client) error {
 157         e.loaded = tarball
 158         return e.loadErr
 159 }
 160 func (e *stubExecutor) Runtime() string                 { return "stub" }
 161 func (e *stubExecutor) Version() string                 { return "stub " + cmd.Version.String() }
 162 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 163 func (e *stubExecutor) Start() error {
 164         e.exit = make(chan int, 1)
 165         go func() { e.exit <- e.runFunc() }()
 166         return e.startErr
 167 }
 168 func (e *stubExecutor) Pid() int    { return 1115883 } // matches pid in ../crunchstat/testdata/debian12/proc/
 169 func (e *stubExecutor) Stop() error { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 170 func (e *stubExecutor) Close()      { e.closed = true }
 171 func (e *stubExecutor) Wait(context.Context) (int, error) {
 172         return <-e.exit, e.waitErr
 173 }
 174 func (e *stubExecutor) InjectCommand(ctx context.Context, _, _ string, _ bool, _ []string) (*exec.Cmd, error) {
 175         return nil, errors.New("unimplemented")
 176 }
 177 func (e *stubExecutor) IPAddress() (string, error) { return "", errors.New("unimplemented") }
 178
 179 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 180
 181 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 182 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 183 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 184
 185 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 186 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 187
 188 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 189 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 190 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 191 `
 192
 193 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 194
 195 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 196 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 197
 198 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 199 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 200
 201 func (client *ArvTestClient) Create(resourceType string,
 202         parameters arvadosclient.Dict,
 203         output interface{}) error {
 204
 205         client.Mutex.Lock()
 206         defer client.Mutex.Unlock()
 207
 208         client.Calls++
 209         client.Content = append(client.Content, parameters)
 210
 211         if resourceType == "logs" {
 212                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 213                 if client.Logs == nil {
 214                         client.Logs = make(map[string]*bytes.Buffer)
 215                 }
 216                 if client.Logs[et] == nil {
 217                         client.Logs[et] = &bytes.Buffer{}
 218                 }
 219                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 220         }
 221
 222         if resourceType == "collections" && output != nil {
 223                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 224                 md5sum := md5.Sum([]byte(mt))
 225                 outmap := output.(*arvados.Collection)
 226                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5sum, len(mt))
 227                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%015x", md5sum[:7])
 228         }
 229
 230         return nil
 231 }
 232
 233 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 234         switch {
 235         case method == "GET" && resourceType == "containers" && action == "auth":
 236                 return json.Unmarshal([]byte(`{
 237                         "kind": "arvados#api_client_authorization",
 238                         "uuid": "`+fakeAuthUUID+`",
 239                         "api_token": "`+fakeAuthToken+`"
 240                         }`), output)
 241         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 242                 if client.secretMounts != nil {
 243                         return json.Unmarshal(client.secretMounts, output)
 244                 }
 245                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 246         default:
 247                 return fmt.Errorf("Not found")
 248         }
 249 }
 250
 251 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 252         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 253         var j []byte
 254         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 255                 j = []byte(`{
 256                         "kind": "arvados#nodeList",
 257                         "items": [{
 258                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 259                                 "hostname": "compute2",
 260                                 "properties": {"total_cpu_cores": 16}
 261                         }]}`)
 262         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 263                 if uuid == "" {
 264                         j, err = json.Marshal(map[string]interface{}{
 265                                 "items": []interface{}{client.Container},
 266                                 "kind":  "arvados#nodeList",
 267                         })
 268                 } else {
 269                         j, err = json.Marshal(client.Container)
 270                 }
 271         } else {
 272                 j = []byte(`{
 273                         "command": ["sleep", "1"],
 274                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 275                         "cwd": ".",
 276                         "environment": {},
 277                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 278                         "output_path": "/tmp",
 279                         "priority": 1,
 280                         "runtime_constraints": {}
 281                 }`)
 282         }
 283         return ioutil.NopCloser(bytes.NewReader(j)), err
 284 }
 285
 286 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 287         if resourceType == "collections" {
 288                 if uuid == hwPDH {
 289                         output.(*arvados.Collection).ManifestText = hwManifest
 290                 } else if uuid == otherPDH {
 291                         output.(*arvados.Collection).ManifestText = otherManifest
 292                 } else if uuid == normalizedWithSubdirsPDH {
 293                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 294                 } else if uuid == denormalizedWithSubdirsPDH {
 295                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 296                 }
 297         }
 298         if resourceType == "containers" {
 299                 (*output.(*arvados.Container)) = client.Container
 300         }
 301         return nil
 302 }
 303
 304 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 305         client.Mutex.Lock()
 306         defer client.Mutex.Unlock()
 307         client.Calls++
 308         client.Content = append(client.Content, parameters)
 309         if resourceType == "containers" {
 310                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 311                         client.WasSetRunning = true
 312                 }
 313         } else if resourceType == "collections" && output != nil {
 314                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 315                 output.(*arvados.Collection).UUID = uuid
 316                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 317         }
 318         return nil
 319 }
 320
 321 var discoveryMap = map[string]interface{}{
 322         "defaultTrashLifetime":               float64(1209600),
 323         "crunchLimitLogBytesPerJob":          float64(67108864),
 324         "crunchLogThrottleBytes":             float64(65536),
 325         "crunchLogThrottlePeriod":            float64(60),
 326         "crunchLogThrottleLines":             float64(1024),
 327         "crunchLogPartialLineThrottlePeriod": float64(5),
 328         "crunchLogBytesPerEvent":             float64(4096),
 329         "crunchLogSecondsBetweenEvents":      float64(1),
 330 }
 331
 332 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 333         return discoveryMap[key], nil
 334 }
 335
 336 // CalledWith returns the parameters from the first API call whose
 337 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 338 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 339 // no call matches, it returns nil.
 340 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 341 call:
 342         for _, content := range client.Content {
 343                 var v interface{} = content
 344                 for _, k := range strings.Split(jpath, ".") {
 345                         if dict, ok := v.(arvadosclient.Dict); !ok {
 346                                 continue call
 347                         } else {
 348                                 v = dict[k]
 349                         }
 350                 }
 351                 if v == expect {
 352                         return content
 353                 }
 354         }
 355         return nil
 356 }
 357
 358 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 359         return locator, nil
 360 }
 361
 362 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 363         locator := fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data))
 364         client.blocks.Store(locator, append([]byte(nil), opts.Data...))
 365         return arvados.BlockWriteResponse{
 366                 Locator: locator,
 367         }, nil
 368 }
 369
 370 func (client *KeepTestClient) ReadAt(locator string, dst []byte, offset int) (int, error) {
 371         loaded, ok := client.blocks.Load(locator)
 372         if !ok {
 373                 return 0, os.ErrNotExist
 374         }
 375         data := loaded.([]byte)
 376         if offset >= len(data) {
 377                 return 0, io.EOF
 378         }
 379         return copy(dst, data[offset:]), nil
 380 }
 381
 382 func (client *KeepTestClient) Close() {
 383         client.blocks.Range(func(locator, value interface{}) bool {
 384                 client.blocks.Delete(locator)
 385                 return true
 386         })
 387 }
 388
 389 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 390         client.StorageClasses = sc
 391 }
 392
 393 type FileWrapper struct {
 394         io.ReadCloser
 395         len int64
 396 }
 397
 398 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 399         return nil, errors.New("not implemented")
 400 }
 401
 402 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 403         return 0, errors.New("not implemented")
 404 }
 405
 406 func (fw FileWrapper) Size() int64 {
 407         return fw.len
 408 }
 409
 410 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 411         return nil, errors.New("not implemented")
 412 }
 413
 414 func (fw FileWrapper) Truncate(int64) error {
 415         return errors.New("not implemented")
 416 }
 417
 418 func (fw FileWrapper) Write([]byte) (int, error) {
 419         return 0, errors.New("not implemented")
 420 }
 421
 422 func (fw FileWrapper) Sync() error {
 423         return errors.New("not implemented")
 424 }
 425
 426 func (fw FileWrapper) Snapshot() (*arvados.Subtree, error) {
 427         return nil, errors.New("not implemented")
 428 }
 429
 430 func (fw FileWrapper) Splice(*arvados.Subtree) error {
 431         return errors.New("not implemented")
 432 }
 433
 434 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 435         if filename == hwImageID+".tar" {
 436                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 437                 client.Called = true
 438                 return FileWrapper{rdr, 1321984}, nil
 439         } else if filename == "/file1_in_main.txt" {
 440                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 441                 client.Called = true
 442                 return FileWrapper{rdr, 3}, nil
 443         }
 444         return nil, nil
 445 }
 446
 447 type apiStubServer struct {
 448         server    *httptest.Server
 449         proxy     *httputil.ReverseProxy
 450         intercept func(http.ResponseWriter, *http.Request) bool
 451
 452         container arvados.Container
 453         logs      map[string]string
 454 }
 455
 456 func apiStub() (*arvados.Client, *apiStubServer) {
 457         client := arvados.NewClientFromEnv()
 458         apistub := &apiStubServer{}
 459         apistub.server = httptest.NewTLSServer(apistub)
 460         apistub.proxy = httputil.NewSingleHostReverseProxy(&url.URL{Scheme: "https", Host: client.APIHost})
 461         if client.Insecure {
 462                 apistub.proxy.Transport = arvados.InsecureHTTPClient.Transport
 463         }
 464         client.APIHost = apistub.server.Listener.Addr().String()
 465         return client, apistub
 466 }
 467
 468 func (apistub *apiStubServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 469         if apistub.intercept != nil && apistub.intercept(w, r) {
 470                 return
 471         }
 472         if r.Method == "POST" && r.URL.Path == "/arvados/v1/logs" {
 473                 var body struct {
 474                         Log struct {
 475                                 EventType  string `json:"event_type"`
 476                                 Properties struct {
 477                                         Text string
 478                                 }
 479                         }
 480                 }
 481                 json.NewDecoder(r.Body).Decode(&body)
 482                 apistub.logs[body.Log.EventType] += body.Log.Properties.Text
 483                 return
 484         }
 485         if r.Method == "GET" && r.URL.Path == "/arvados/v1/collections/"+hwPDH {
 486                 json.NewEncoder(w).Encode(arvados.Collection{ManifestText: hwManifest})
 487                 return
 488         }
 489         if r.Method == "GET" && r.URL.Path == "/arvados/v1/collections/"+otherPDH {
 490                 json.NewEncoder(w).Encode(arvados.Collection{ManifestText: otherManifest})
 491                 return
 492         }
 493         if r.Method == "GET" && r.URL.Path == "/arvados/v1/collections/"+normalizedWithSubdirsPDH {
 494                 json.NewEncoder(w).Encode(arvados.Collection{ManifestText: normalizedManifestWithSubdirs})
 495                 return
 496         }
 497         if r.Method == "GET" && r.URL.Path == "/arvados/v1/collections/"+denormalizedWithSubdirsPDH {
 498                 json.NewEncoder(w).Encode(arvados.Collection{ManifestText: denormalizedManifestWithSubdirs})
 499                 return
 500         }
 501         if r.Method == "GET" && r.URL.Path == "/arvados/v1/containers/"+apistub.container.UUID {
 502                 json.NewEncoder(w).Encode(apistub.container)
 503                 return
 504         }
 505         apistub.proxy.ServeHTTP(w, r)
 506 }
 507
 508 func (s *TestSuite) TestLoadImage(c *C) {
 509         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 510         s.runner.Container.Mounts = map[string]arvados.Mount{
 511                 "/out": {Kind: "tmp", Writable: true},
 512         }
 513         s.runner.Container.OutputPath = "/out"
 514
 515         _, err := s.runner.SetupMounts()
 516         c.Assert(err, IsNil)
 517
 518         imageID, err := s.runner.LoadImage()
 519         c.Check(err, IsNil)
 520         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 521         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 522
 523         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 524         s.executor.imageLoaded = false
 525         s.executor.loaded = ""
 526         s.executor.loadErr = errors.New("bork")
 527         imageID, err = s.runner.LoadImage()
 528         c.Check(err, ErrorMatches, ".*bork")
 529         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 530
 531         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 532         s.executor.imageLoaded = false
 533         s.executor.loaded = ""
 534         s.executor.loadErr = nil
 535         imageID, err = s.runner.LoadImage()
 536         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 537         c.Check(s.executor.loaded, Equals, "")
 538 }
 539
 540 type ArvErrorTestClient struct{}
 541
 542 func (ArvErrorTestClient) Create(resourceType string,
 543         parameters arvadosclient.Dict,
 544         output interface{}) error {
 545         return nil
 546 }
 547
 548 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 549         if method == "GET" && resourceType == "containers" && action == "auth" {
 550                 return nil
 551         }
 552         return errors.New("ArvError")
 553 }
 554
 555 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 556         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 557         return nil, errors.New("ArvError")
 558 }
 559
 560 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 561         return errors.New("ArvError")
 562 }
 563
 564 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 565         return nil
 566 }
 567
 568 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 569         return discoveryMap[key], nil
 570 }
 571
 572 type KeepErrorTestClient struct {
 573         KeepTestClient
 574 }
 575
 576 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 577         return nil, errors.New("KeepError")
 578 }
 579
 580 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 581         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 582 }
 583
 584 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 585         return "", errors.New("KeepError")
 586 }
 587
 588 type KeepReadErrorTestClient struct {
 589         KeepTestClient
 590 }
 591
 592 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 593         return 0, errors.New("KeepError")
 594 }
 595
 596 type ErrorReader struct {
 597         FileWrapper
 598 }
 599
 600 func (ErrorReader) Read(p []byte) (n int, err error) {
 601         return 0, errors.New("ErrorReader")
 602 }
 603
 604 func (ErrorReader) Seek(int64, int) (int64, error) {
 605         return 0, errors.New("ErrorReader")
 606 }
 607
 608 func (*KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 609         return ErrorReader{}, nil
 610 }
 611
 612 type ClosableBuffer struct {
 613         bytes.Buffer
 614 }
 615
 616 func (*ClosableBuffer) Close() error {
 617         return nil
 618 }
 619
 620 type TestLogs struct {
 621         Stdout ClosableBuffer
 622         Stderr ClosableBuffer
 623 }
 624
 625 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 626         if logstr == "stdout" {
 627                 return &tl.Stdout, nil
 628         }
 629         if logstr == "stderr" {
 630                 return &tl.Stderr, nil
 631         }
 632         return nil, errors.New("???")
 633 }
 634
 635 func dockerLog(fd byte, msg string) []byte {
 636         by := []byte(msg)
 637         header := make([]byte, 8+len(by))
 638         header[0] = fd
 639         header[7] = byte(len(by))
 640         copy(header[8:], by)
 641         return header
 642 }
 643
 644 func (s *TestSuite) TestRunContainer(c *C) {
 645         s.executor.runFunc = func() int {
 646                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 647                 return 0
 648         }
 649
 650         var logs TestLogs
 651         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 652         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 653         s.runner.Container.Command = []string{"./hw"}
 654         s.runner.Container.OutputStorageClasses = []string{"default"}
 655
 656         imageID, err := s.runner.LoadImage()
 657         c.Assert(err, IsNil)
 658
 659         err = s.runner.CreateContainer(imageID, nil)
 660         c.Assert(err, IsNil)
 661
 662         err = s.runner.StartContainer()
 663         c.Assert(err, IsNil)
 664
 665         err = s.runner.WaitFinish()
 666         c.Assert(err, IsNil)
 667
 668         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 669         c.Check(logs.Stderr.String(), Equals, "")
 670 }
 671
 672 func (s *TestSuite) TestCommitLogs(c *C) {
 673         api := &ArvTestClient{}
 674         kc := &KeepTestClient{}
 675         defer kc.Close()
 676         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 677         c.Assert(err, IsNil)
 678         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 679
 680         cr.CrunchLog.Print("Hello world!")
 681         cr.CrunchLog.Print("Goodbye")
 682         cr.finalState = "Complete"
 683
 684         err = cr.CommitLogs()
 685         c.Check(err, IsNil)
 686
 687         c.Check(api.Calls, Equals, 2)
 688         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 689         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 690         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 691         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 692 }
 693
 694 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 695         api := &ArvTestClient{}
 696         kc := &KeepTestClient{}
 697         defer kc.Close()
 698         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 699         c.Assert(err, IsNil)
 700
 701         err = cr.UpdateContainerRunning("")
 702         c.Check(err, IsNil)
 703
 704         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 705 }
 706
 707 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 708         api := &ArvTestClient{}
 709         kc := &KeepTestClient{}
 710         defer kc.Close()
 711         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 712         c.Assert(err, IsNil)
 713
 714         cr.LogsPDH = new(string)
 715         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 716
 717         cr.ExitCode = new(int)
 718         *cr.ExitCode = 42
 719         cr.finalState = "Complete"
 720
 721         err = cr.UpdateContainerFinal()
 722         c.Check(err, IsNil)
 723
 724         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 725         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 726         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 727 }
 728
 729 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 730         api := &ArvTestClient{}
 731         kc := &KeepTestClient{}
 732         defer kc.Close()
 733         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 734         c.Assert(err, IsNil)
 735         cr.cCancelled = true
 736         cr.finalState = "Cancelled"
 737
 738         err = cr.UpdateContainerFinal()
 739         c.Check(err, IsNil)
 740
 741         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 742         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 743         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 744 }
 745
 746 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 747 // dress rehearsal of the Run() function, starting from a JSON container record.
 748 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, fn func() int) (*ArvTestClient, *ContainerRunner, string) {
 749         err := json.Unmarshal([]byte(record), &s.api.Container)
 750         c.Assert(err, IsNil)
 751         initialState := s.api.Container.State
 752
 753         var sm struct {
 754                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 755         }
 756         err = json.Unmarshal([]byte(record), &sm)
 757         c.Check(err, IsNil)
 758         secretMounts, err := json.Marshal(sm)
 759         c.Assert(err, IsNil)
 760         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 761
 762         s.executor.runFunc = fn
 763
 764         s.runner.statInterval = 100 * time.Millisecond
 765         s.runner.containerWatchdogInterval = time.Second
 766
 767         realTemp := c.MkDir()
 768         tempcount := 0
 769         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 770                 tempcount++
 771                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 772                 err := os.Mkdir(d, os.ModePerm)
 773                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 774                         // Test case must have pre-populated the tempdir
 775                         err = nil
 776                 }
 777                 return d, err
 778         }
 779         client, _ := apiStub()
 780         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 781                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, client, nil
 782         }
 783
 784         if extraMounts != nil && len(extraMounts) > 0 {
 785                 err := s.runner.SetupArvMountPoint("keep")
 786                 c.Check(err, IsNil)
 787
 788                 for _, m := range extraMounts {
 789                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 790                 }
 791         }
 792
 793         err = s.runner.Run()
 794         if s.api.CalledWith("container.state", "Complete") != nil {
 795                 c.Check(err, IsNil)
 796         }
 797         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 798                 c.Check(s.api.WasSetRunning, Equals, true)
 799                 var lastupdate arvadosclient.Dict
 800                 for _, content := range s.api.Content {
 801                         if content["container"] != nil {
 802                                 lastupdate = content["container"].(arvadosclient.Dict)
 803                         }
 804                 }
 805                 if lastupdate["log"] == nil {
 806                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 807                 }
 808         }
 809
 810         if err != nil {
 811                 for k, v := range s.api.Logs {
 812                         c.Log(k)
 813                         c.Log(v.String())
 814                 }
 815         }
 816
 817         return s.api, s.runner, realTemp
 818 }
 819
 820 func (s *TestSuite) TestFullRunHello(c *C) {
 821         s.runner.enableMemoryLimit = true
 822         s.runner.networkMode = "default"
 823         s.fullRunHelper(c, `{
 824     "command": ["echo", "hello world"],
 825     "container_image": "`+arvadostest.DockerImage112PDH+`",
 826     "cwd": ".",
 827     "environment": {"foo":"bar","baz":"waz"},
 828     "mounts": {"/tmp": {"kind": "tmp"} },
 829     "output_path": "/tmp",
 830     "priority": 1,
 831     "runtime_constraints": {"vcpus":1,"ram":1000000},
 832     "state": "Locked",
 833     "output_storage_classes": ["default"]
 834 }`, nil, func() int {
 835                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 836                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 837                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 838                 c.Check(s.executor.created.VCPUs, Equals, 1)
 839                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 840                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 841                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 842                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
 843                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 844                 return 0
 845         })
 846
 847         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 848         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 849         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 850         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 851         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 852 }
 853
 854 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 855         var ran bool
 856         s.fullRunHelper(c, `{
 857     "command": ["sleep", "3"],
 858     "container_image": "`+arvadostest.DockerImage112PDH+`",
 859     "cwd": ".",
 860     "environment": {},
 861     "mounts": {"/tmp": {"kind": "tmp"} },
 862     "output_path": "/tmp",
 863     "priority": 1,
 864     "runtime_constraints": {},
 865     "scheduling_parameters":{"max_run_time": 1},
 866     "state": "Running"
 867 }`, nil, func() int {
 868                 ran = true
 869                 return 2
 870         })
 871         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 872         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 873         c.Check(ran, Equals, false)
 874 }
 875
 876 func ec2MetadataServerStub(c *C, token *string, failureRate float64, stoptime *atomic.Value) *httptest.Server {
 877         failedOnce := false
 878         return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 879                 if !failedOnce || rand.Float64() < failureRate {
 880                         w.WriteHeader(http.StatusServiceUnavailable)
 881                         failedOnce = true
 882                         return
 883                 }
 884                 switch r.URL.Path {
 885                 case "/latest/api/token":
 886                         fmt.Fprintln(w, *token)
 887                 case "/latest/meta-data/spot/instance-action":
 888                         if r.Header.Get("X-aws-ec2-metadata-token") != *token {
 889                                 w.WriteHeader(http.StatusUnauthorized)
 890                         } else if t, _ := stoptime.Load().(time.Time); t.IsZero() {
 891                                 w.WriteHeader(http.StatusNotFound)
 892                         } else {
 893                                 fmt.Fprintf(w, `{"action":"stop","time":"%s"}`, t.Format(time.RFC3339))
 894                         }
 895                 default:
 896                         w.WriteHeader(http.StatusNotFound)
 897                 }
 898         }))
 899 }
 900
 901 func (s *TestSuite) TestSpotInterruptionNotice(c *C) {
 902         s.testSpotInterruptionNotice(c, 0.1)
 903 }
 904
 905 func (s *TestSuite) TestSpotInterruptionNoticeNotAvailable(c *C) {
 906         s.testSpotInterruptionNotice(c, 1)
 907 }
 908
 909 func (s *TestSuite) testSpotInterruptionNotice(c *C, failureRate float64) {
 910         var stoptime atomic.Value
 911         token := "fake-ec2-metadata-token"
 912         stub := ec2MetadataServerStub(c, &token, failureRate, &stoptime)
 913         defer stub.Close()
 914
 915         defer func(i time.Duration, u string) {
 916                 spotInterruptionCheckInterval = i
 917                 ec2MetadataBaseURL = u
 918         }(spotInterruptionCheckInterval, ec2MetadataBaseURL)
 919         spotInterruptionCheckInterval = time.Second / 8
 920         ec2MetadataBaseURL = stub.URL
 921
 922         go s.runner.checkSpotInterruptionNotices()
 923         s.fullRunHelper(c, `{
 924     "command": ["sleep", "3"],
 925     "container_image": "`+arvadostest.DockerImage112PDH+`",
 926     "cwd": ".",
 927     "environment": {},
 928     "mounts": {"/tmp": {"kind": "tmp"} },
 929     "output_path": "/tmp",
 930     "priority": 1,
 931     "runtime_constraints": {},
 932     "state": "Locked"
 933 }`, nil, func() int {
 934                 time.Sleep(time.Second)
 935                 stoptime.Store(time.Now().Add(time.Minute).UTC())
 936                 token = "different-fake-ec2-metadata-token"
 937                 time.Sleep(time.Second)
 938                 return 0
 939         })
 940         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Checking for spot interruptions every 125ms using instance metadata at http://.*`)
 941         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Error checking spot interruptions: 503 Service Unavailable.*`)
 942         if failureRate == 1 {
 943                 c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Giving up on checking spot interruptions after too many consecutive failures.*`)
 944         } else {
 945                 text := `Cloud provider scheduled instance stop at ` + stoptime.Load().(time.Time).Format(time.RFC3339)
 946                 c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*`+text+`.*`)
 947                 c.Check(s.api.CalledWith("container.runtime_status.warning", "preemption notice"), NotNil)
 948                 c.Check(s.api.CalledWith("container.runtime_status.warningDetail", text), NotNil)
 949                 c.Check(s.api.CalledWith("container.runtime_status.preemptionNotice", text), NotNil)
 950         }
 951 }
 952
 953 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 954         s.fullRunHelper(c, `{
 955     "command": ["sleep", "3"],
 956     "container_image": "`+arvadostest.DockerImage112PDH+`",
 957     "cwd": ".",
 958     "environment": {},
 959     "mounts": {"/tmp": {"kind": "tmp"} },
 960     "output_path": "/tmp",
 961     "priority": 1,
 962     "runtime_constraints": {},
 963     "scheduling_parameters":{"max_run_time": 1},
 964     "state": "Locked"
 965 }`, nil, func() int {
 966                 time.Sleep(3 * time.Second)
 967                 return 0
 968         })
 969
 970         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 971         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 972 }
 973
 974 func (s *TestSuite) TestContainerWaitFails(c *C) {
 975         s.fullRunHelper(c, `{
 976     "command": ["sleep", "3"],
 977     "container_image": "`+arvadostest.DockerImage112PDH+`",
 978     "cwd": ".",
 979     "mounts": {"/tmp": {"kind": "tmp"} },
 980     "output_path": "/tmp",
 981     "priority": 1,
 982     "state": "Locked"
 983 }`, nil, func() int {
 984                 s.executor.waitErr = errors.New("Container is not running")
 985                 return 0
 986         })
 987
 988         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 989         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 990 }
 991
 992 func (s *TestSuite) TestCrunchstat(c *C) {
 993         s.runner.crunchstatFakeFS = os.DirFS("../crunchstat/testdata/debian12")
 994         s.fullRunHelper(c, `{
 995                 "command": ["sleep", "1"],
 996                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 997                 "cwd": ".",
 998                 "environment": {},
 999                 "mounts": {"/tmp": {"kind": "tmp"} },
1000                 "output_path": "/tmp",
1001                 "priority": 1,
1002                 "runtime_constraints": {},
1003                 "state": "Locked"
1004         }`, nil, func() int {
1005                 time.Sleep(time.Second)
1006                 return 0
1007         })
1008
1009         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1010         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1011
1012         c.Assert(s.api.Logs["crunchstat"], NotNil)
1013         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*mem \d+ swap \d+ pgmajfault \d+ rss.*`)
1014
1015         // Check that we called (*crunchstat.Reporter)Stop().
1016         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Maximum crunch-run memory rss usage was \d+ bytes\n.*`)
1017 }
1018
1019 func (s *TestSuite) TestNodeInfoLog(c *C) {
1020         s.fullRunHelper(c, `{
1021                 "command": ["true"],
1022                 "container_image": "`+arvadostest.DockerImage112PDH+`",
1023                 "cwd": ".",
1024                 "environment": {},
1025                 "mounts": {"/tmp": {"kind": "tmp"} },
1026                 "output_path": "/tmp",
1027                 "priority": 1,
1028                 "runtime_constraints": {},
1029                 "state": "Locked"
1030         }`, nil, func() int {
1031                 return 0
1032         })
1033
1034         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1035         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1036
1037         buf, err := fs.ReadFile(arvados.FS(s.runner.LogCollection), "/node.json")
1038         c.Assert(err, IsNil)
1039         json := string(buf)
1040         c.Check(json, Matches, `(?ms).*"ProviderType": *"a1\.2xlarge".*`)
1041         c.Check(json, Matches, `(?ms).*"Price": *1\.2.*`)
1042
1043         c.Assert(s.api.Logs["node-info"], NotNil)
1044         json = s.api.Logs["node-info"].String()
1045         c.Check(json, Matches, `(?ms).*Host Information.*`)
1046         c.Check(json, Matches, `(?ms).*CPU Information.*`)
1047         c.Check(json, Matches, `(?ms).*Memory Information.*`)
1048         c.Check(json, Matches, `(?ms).*Disk Space.*`)
1049         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
1050 }
1051
1052 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
1053         s.fullRunHelper(c, `{
1054                 "command": ["sleep", "1"],
1055                 "container_image": "`+arvadostest.DockerImage112PDH+`",
1056                 "cwd": ".",
1057                 "environment": {},
1058                 "mounts": {"/tmp": {"kind": "tmp"} },
1059                 "output_path": "/tmp",
1060                 "priority": 1,
1061                 "runtime_constraints": {},
1062                 "state": "Locked"
1063         }`, nil, func() int {
1064                 return 0
1065         })
1066
1067         c.Assert(s.api.Logs["crunch-run"], NotNil)
1068         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
1069         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`)
1070         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container: zzzzz-zzzzz-zzzzzzzzzzzzzzz.*`)
1071         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Using container runtime: stub.*`)
1072 }
1073
1074 func (s *TestSuite) testLogRSSThresholds(c *C, ram int64, expected []int, notExpected int) {
1075         s.runner.crunchstatFakeFS = os.DirFS("../crunchstat/testdata/debian12")
1076         s.fullRunHelper(c, `{
1077                 "command": ["true"],
1078                 "container_image": "`+arvadostest.DockerImage112PDH+`",
1079                 "cwd": ".",
1080                 "environment": {},
1081                 "mounts": {"/tmp": {"kind": "tmp"} },
1082                 "output_path": "/tmp",
1083                 "priority": 1,
1084                 "runtime_constraints": {"ram": `+fmt.Sprintf("%d", ram)+`},
1085                 "state": "Locked"
1086         }`, nil, func() int { return 0 })
1087         c.Logf("=== crunchstat logs\n%s\n", s.api.Logs["crunchstat"].String())
1088         logs := s.api.Logs["crunch-run"].String()
1089         pattern := logLineStart + `Container using over %d%% of memory \(rss %d/%d bytes\)`
1090         var threshold int
1091         for _, threshold = range expected {
1092                 c.Check(logs, Matches, fmt.Sprintf(pattern, threshold, s.debian12MemoryCurrent, ram))
1093         }
1094         if notExpected > threshold {
1095                 c.Check(logs, Not(Matches), fmt.Sprintf(pattern, notExpected, s.debian12MemoryCurrent, ram))
1096         }
1097 }
1098
1099 func (s *TestSuite) TestLogNoRSSThresholds(c *C) {
1100         s.testLogRSSThresholds(c, s.debian12MemoryCurrent*10, []int{}, 90)
1101 }
1102
1103 func (s *TestSuite) TestLogSomeRSSThresholds(c *C) {
1104         onePercentRSS := s.debian12MemoryCurrent / 100
1105         s.testLogRSSThresholds(c, 102*onePercentRSS, []int{90, 95}, 99)
1106 }
1107
1108 func (s *TestSuite) TestLogAllRSSThresholds(c *C) {
1109         s.testLogRSSThresholds(c, s.debian12MemoryCurrent, []int{90, 95, 99}, 0)
1110 }
1111
1112 func (s *TestSuite) TestLogMaximaAfterRun(c *C) {
1113         s.runner.crunchstatFakeFS = os.DirFS("../crunchstat/testdata/debian12")
1114         s.runner.parentTemp = c.MkDir()
1115         s.fullRunHelper(c, `{
1116         "command": ["true"],
1117         "container_image": "`+arvadostest.DockerImage112PDH+`",
1118         "cwd": ".",
1119         "environment": {},
1120         "mounts": {"/tmp": {"kind": "tmp"} },
1121         "output_path": "/tmp",
1122         "priority": 1,
1123         "runtime_constraints": {"ram": `+fmt.Sprintf("%d", s.debian12MemoryCurrent*10)+`},
1124         "state": "Locked"
1125     }`, nil, func() int { return 0 })
1126         logs := s.api.Logs["crunch-run"].String()
1127         for _, expected := range []string{
1128                 `Maximum disk usage was \d+%, \d+/\d+ bytes`,
1129                 fmt.Sprintf(`Maximum container memory swap usage was %d bytes`, s.debian12SwapCurrent),
1130                 `Maximum container memory pgmajfault usage was \d+ faults`,
1131                 fmt.Sprintf(`Maximum container memory rss usage was 10%%, %d/%d bytes`, s.debian12MemoryCurrent, s.debian12MemoryCurrent*10),
1132                 `Maximum crunch-run memory rss usage was \d+ bytes`,
1133         } {
1134                 c.Check(logs, Matches, logLineStart+expected)
1135         }
1136 }
1137
1138 func (s *TestSuite) TestCommitNodeInfoBeforeStart(c *C) {
1139         var collection_create, container_update arvadosclient.Dict
1140         s.fullRunHelper(c, `{
1141                 "command": ["true"],
1142                 "container_image": "`+arvadostest.DockerImage112PDH+`",
1143                 "cwd": ".",
1144                 "environment": {},
1145                 "mounts": {"/tmp": {"kind": "tmp"} },
1146                 "output_path": "/tmp",
1147                 "priority": 1,
1148                 "runtime_constraints": {},
1149                 "state": "Locked",
1150                 "uuid": "zzzzz-dz642-202301121543210"
1151         }`, nil, func() int {
1152                 collection_create = s.api.CalledWith("ensure_unique_name", true)
1153                 container_update = s.api.CalledWith("container.state", "Running")
1154                 return 0
1155         })
1156
1157         c.Assert(collection_create, NotNil)
1158         log_collection := collection_create["collection"].(arvadosclient.Dict)
1159         c.Check(log_collection["name"], Equals, "logs for zzzzz-dz642-202301121543210")
1160         manifest_text := log_collection["manifest_text"].(string)
1161         // We check that the file size is at least two digits as an easy way to
1162         // check the file isn't empty.
1163         c.Check(manifest_text, Matches, `\. .+ \d+:\d{2,}:node-info\.txt( .+)?\n`)
1164         c.Check(manifest_text, Matches, `\. .+ \d+:\d{2,}:node\.json( .+)?\n`)
1165
1166         c.Assert(container_update, NotNil)
1167         // As of Arvados 2.5.0, the container update must specify its log in PDH
1168         // format for the API server to propagate it to container requests, which
1169         // is what we care about for this test.
1170         expect_pdh := fmt.Sprintf("%x+%d", md5.Sum([]byte(manifest_text)), len(manifest_text))
1171         c.Check(container_update["container"].(arvadosclient.Dict)["log"], Equals, expect_pdh)
1172 }
1173
1174 func (s *TestSuite) TestContainerRecordLog(c *C) {
1175         s.fullRunHelper(c, `{
1176                 "command": ["sleep", "1"],
1177                 "container_image": "`+arvadostest.DockerImage112PDH+`",
1178                 "cwd": ".",
1179                 "environment": {},
1180                 "mounts": {"/tmp": {"kind": "tmp"} },
1181                 "output_path": "/tmp",
1182                 "priority": 1,
1183                 "runtime_constraints": {},
1184                 "state": "Locked"
1185         }`, nil,
1186                 func() int {
1187                         time.Sleep(time.Second)
1188                         return 0
1189                 })
1190
1191         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1192         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1193
1194         c.Assert(s.api.Logs["container"], NotNil)
1195         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
1196 }
1197
1198 func (s *TestSuite) TestFullRunStderr(c *C) {
1199         s.fullRunHelper(c, `{
1200     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
1201     "container_image": "`+arvadostest.DockerImage112PDH+`",
1202     "cwd": ".",
1203     "environment": {},
1204     "mounts": {"/tmp": {"kind": "tmp"} },
1205     "output_path": "/tmp",
1206     "priority": 1,
1207     "runtime_constraints": {},
1208     "state": "Locked"
1209 }`, nil, func() int {
1210                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1211                 fmt.Fprintln(s.executor.created.Stderr, "world")
1212                 return 1
1213         })
1214
1215         final := s.api.CalledWith("container.state", "Complete")
1216         c.Assert(final, NotNil)
1217         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1218         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1219
1220         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
1221         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
1222 }
1223
1224 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
1225         s.fullRunHelper(c, `{
1226     "command": ["pwd"],
1227     "container_image": "`+arvadostest.DockerImage112PDH+`",
1228     "cwd": ".",
1229     "environment": {},
1230     "mounts": {"/tmp": {"kind": "tmp"} },
1231     "output_path": "/tmp",
1232     "priority": 1,
1233     "runtime_constraints": {},
1234     "state": "Locked"
1235 }`, nil, func() int {
1236                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
1237                 return 0
1238         })
1239
1240         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1241         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1242         c.Log(s.api.Logs["stdout"])
1243         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
1244 }
1245
1246 func (s *TestSuite) TestFullRunSetCwd(c *C) {
1247         s.fullRunHelper(c, `{
1248     "command": ["pwd"],
1249     "container_image": "`+arvadostest.DockerImage112PDH+`",
1250     "cwd": "/bin",
1251     "environment": {},
1252     "mounts": {"/tmp": {"kind": "tmp"} },
1253     "output_path": "/tmp",
1254     "priority": 1,
1255     "runtime_constraints": {},
1256     "state": "Locked"
1257 }`, nil, func() int {
1258                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1259                 return 0
1260         })
1261
1262         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1263         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1264         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1265 }
1266
1267 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
1268         s.fullRunHelper(c, `{
1269     "command": ["pwd"],
1270     "container_image": "`+arvadostest.DockerImage112PDH+`",
1271     "cwd": "/bin",
1272     "environment": {},
1273     "mounts": {"/tmp": {"kind": "tmp"} },
1274     "output_path": "/tmp",
1275     "priority": 1,
1276     "runtime_constraints": {},
1277     "state": "Locked",
1278     "output_storage_classes": ["foo", "bar"]
1279 }`, nil, func() int {
1280                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1281                 return 0
1282         })
1283
1284         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1285         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1286         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1287         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1288         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1289 }
1290
1291 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
1292         s.fullRunHelper(c, `{
1293     "command": ["pwd"],
1294     "container_image": "`+arvadostest.DockerImage112PDH+`",
1295     "cwd": "/bin",
1296     "environment": {},
1297     "mounts": {"/tmp": {"kind": "tmp"} },
1298     "output_path": "/tmp",
1299     "priority": 1,
1300     "runtime_constraints": {"cuda": {"device_count": 2}},
1301     "state": "Locked",
1302     "output_storage_classes": ["foo", "bar"]
1303 }`, nil, func() int {
1304                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1305                 return 0
1306         })
1307         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1308 }
1309
1310 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1311         s.fullRunHelper(c, `{
1312     "command": ["pwd"],
1313     "container_image": "`+arvadostest.DockerImage112PDH+`",
1314     "cwd": "/bin",
1315     "environment": {},
1316     "mounts": {"/tmp": {"kind": "tmp"} },
1317     "output_path": "/tmp",
1318     "priority": 1,
1319     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1320     "state": "Locked",
1321     "output_storage_classes": ["foo", "bar"]
1322 }`, nil, func() int {
1323                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1324                 return 0
1325         })
1326         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1327 }
1328
1329 func (s *TestSuite) TestStopOnSignal(c *C) {
1330         s.executor.runFunc = func() int {
1331                 s.executor.created.Stdout.Write([]byte("foo\n"))
1332                 s.runner.SigChan <- syscall.SIGINT
1333                 time.Sleep(10 * time.Second)
1334                 return 0
1335         }
1336         s.testStopContainer(c)
1337 }
1338
1339 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1340         s.executor.runFunc = func() int {
1341                 s.executor.created.Stdout.Write([]byte("foo\n"))
1342                 s.runner.ArvMountExit <- nil
1343                 close(s.runner.ArvMountExit)
1344                 time.Sleep(10 * time.Second)
1345                 return 0
1346         }
1347         s.runner.ArvMountExit = make(chan error)
1348         s.testStopContainer(c)
1349 }
1350
1351 func (s *TestSuite) testStopContainer(c *C) {
1352         record := `{
1353     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1354     "container_image": "` + arvadostest.DockerImage112PDH + `",
1355     "cwd": ".",
1356     "environment": {},
1357     "mounts": {"/tmp": {"kind": "tmp"} },
1358     "output_path": "/tmp",
1359     "priority": 1,
1360     "runtime_constraints": {},
1361     "state": "Locked"
1362 }`
1363
1364         err := json.Unmarshal([]byte(record), &s.api.Container)
1365         c.Assert(err, IsNil)
1366
1367         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1368         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1369                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1370         }
1371
1372         done := make(chan error)
1373         go func() {
1374                 done <- s.runner.Run()
1375         }()
1376         select {
1377         case <-time.After(20 * time.Second):
1378                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1379                 c.Fatal("timed out")
1380         case err = <-done:
1381                 c.Check(err, IsNil)
1382         }
1383         for k, v := range s.api.Logs {
1384                 c.Log(k)
1385                 c.Log(v.String(), "\n")
1386         }
1387
1388         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1389         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1390         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1391 }
1392
1393 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1394         s.fullRunHelper(c, `{
1395     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1396     "container_image": "`+arvadostest.DockerImage112PDH+`",
1397     "cwd": "/bin",
1398     "environment": {"FROBIZ": "bilbo"},
1399     "mounts": {"/tmp": {"kind": "tmp"} },
1400     "output_path": "/tmp",
1401     "priority": 1,
1402     "runtime_constraints": {},
1403     "state": "Locked"
1404 }`, nil, func() int {
1405                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1406                 return 0
1407         })
1408
1409         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1410         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1411         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1412 }
1413
1414 type ArvMountCmdLine struct {
1415         Cmd   []string
1416         token string
1417 }
1418
1419 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1420         am.Cmd = c
1421         am.token = token
1422         return nil, nil
1423 }
1424
1425 func stubCert(c *C, temp string) string {
1426         path := temp + "/ca-certificates.crt"
1427         err := os.WriteFile(path, []byte{}, 0666)
1428         c.Assert(err, IsNil)
1429         os.Setenv("SSL_CERT_FILE", path)
1430         return path
1431 }
1432
1433 func (s *TestSuite) TestSetupMounts(c *C) {
1434         cr := s.runner
1435         am := &ArvMountCmdLine{}
1436         cr.RunArvMount = am.ArvMountTest
1437         cr.containerClient, _ = apiStub()
1438         cr.ContainerArvClient = &ArvTestClient{}
1439         cr.ContainerKeepClient = &KeepTestClient{}
1440         cr.Container.OutputStorageClasses = []string{"default"}
1441
1442         realTemp := c.MkDir()
1443         certTemp := c.MkDir()
1444         stubCertPath := stubCert(c, certTemp)
1445         cr.parentTemp = realTemp
1446
1447         i := 0
1448         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1449                 i++
1450                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1451                 err := os.Mkdir(d, os.ModePerm)
1452                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1453                         // Test case must have pre-populated the tempdir
1454                         err = nil
1455                 }
1456                 return d, err
1457         }
1458
1459         checkEmpty := func() {
1460                 // Should be deleted.
1461                 _, err := os.Stat(realTemp)
1462                 c.Assert(os.IsNotExist(err), Equals, true)
1463
1464                 // Now recreate it for the next test.
1465                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1466         }
1467
1468         {
1469                 i = 0
1470                 cr.ArvMountPoint = ""
1471                 cr.Container.Mounts = make(map[string]arvados.Mount)
1472                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1473                 cr.Container.OutputPath = "/tmp"
1474                 cr.statInterval = 5 * time.Second
1475                 bindmounts, err := cr.SetupMounts()
1476                 c.Check(err, IsNil)
1477                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1478                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1479                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1480                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1481                 os.RemoveAll(cr.ArvMountPoint)
1482                 cr.CleanupDirs()
1483                 checkEmpty()
1484         }
1485
1486         {
1487                 i = 0
1488                 cr.ArvMountPoint = ""
1489                 cr.Container.Mounts = make(map[string]arvados.Mount)
1490                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1491                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1492                 cr.Container.OutputPath = "/out"
1493                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1494
1495                 bindmounts, err := cr.SetupMounts()
1496                 c.Check(err, IsNil)
1497                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1498                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1499                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1500                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1501                 os.RemoveAll(cr.ArvMountPoint)
1502                 cr.CleanupDirs()
1503                 checkEmpty()
1504         }
1505
1506         {
1507                 i = 0
1508                 cr.ArvMountPoint = ""
1509                 cr.Container.Mounts = make(map[string]arvados.Mount)
1510                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1511                 cr.Container.OutputPath = "/tmp"
1512                 cr.Container.RuntimeConstraints.API = true
1513                 cr.Container.OutputStorageClasses = []string{"default"}
1514
1515                 bindmounts, err := cr.SetupMounts()
1516                 c.Check(err, IsNil)
1517                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1518                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1519                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1520                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1521                 os.RemoveAll(cr.ArvMountPoint)
1522                 cr.CleanupDirs()
1523                 checkEmpty()
1524
1525                 cr.Container.RuntimeConstraints.API = false
1526         }
1527
1528         {
1529                 i = 0
1530                 cr.ArvMountPoint = ""
1531                 cr.Container.Mounts = map[string]arvados.Mount{
1532                         "/keeptmp": {Kind: "collection", Writable: true},
1533                 }
1534                 cr.Container.OutputPath = "/keeptmp"
1535
1536                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1537
1538                 bindmounts, err := cr.SetupMounts()
1539                 c.Check(err, IsNil)
1540                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1541                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1542                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1543                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1544                 os.RemoveAll(cr.ArvMountPoint)
1545                 cr.CleanupDirs()
1546                 checkEmpty()
1547         }
1548
1549         {
1550                 i = 0
1551                 cr.ArvMountPoint = ""
1552                 cr.Container.Mounts = map[string]arvados.Mount{
1553                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1554                         "/keepout": {Kind: "collection", Writable: true},
1555                 }
1556                 cr.Container.OutputPath = "/keepout"
1557
1558                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1559                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1560
1561                 bindmounts, err := cr.SetupMounts()
1562                 c.Check(err, IsNil)
1563                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1564                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1565                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1566                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1567                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1568                         "/keepout": {realTemp + "/keep1/tmp0", false},
1569                 })
1570                 os.RemoveAll(cr.ArvMountPoint)
1571                 cr.CleanupDirs()
1572                 checkEmpty()
1573         }
1574
1575         {
1576                 i = 0
1577                 cr.ArvMountPoint = ""
1578                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1579                 cr.Container.Mounts = map[string]arvados.Mount{
1580                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1581                         "/keepout": {Kind: "collection", Writable: true},
1582                 }
1583                 cr.Container.OutputPath = "/keepout"
1584
1585                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1586                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1587
1588                 bindmounts, err := cr.SetupMounts()
1589                 c.Check(err, IsNil)
1590                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1591                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
1592                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1593                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1594                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1595                         "/keepout": {realTemp + "/keep1/tmp0", false},
1596                 })
1597                 os.RemoveAll(cr.ArvMountPoint)
1598                 cr.CleanupDirs()
1599                 checkEmpty()
1600         }
1601
1602         for _, test := range []struct {
1603                 in  interface{}
1604                 out string
1605         }{
1606                 {in: "foo", out: `"foo"`},
1607                 {in: nil, out: `null`},
1608                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1609         } {
1610                 i = 0
1611                 cr.ArvMountPoint = ""
1612                 cr.Container.Mounts = map[string]arvados.Mount{
1613                         "/mnt/test.json": {Kind: "json", Content: test.in},
1614                 }
1615                 bindmounts, err := cr.SetupMounts()
1616                 c.Check(err, IsNil)
1617                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1618                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1619                 })
1620                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1621                 c.Check(err, IsNil)
1622                 c.Check(content, DeepEquals, []byte(test.out))
1623                 os.RemoveAll(cr.ArvMountPoint)
1624                 cr.CleanupDirs()
1625                 checkEmpty()
1626         }
1627
1628         for _, test := range []struct {
1629                 in  interface{}
1630                 out string
1631         }{
1632                 {in: "foo", out: `foo`},
1633                 {in: nil, out: "error"},
1634                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1635         } {
1636                 i = 0
1637                 cr.ArvMountPoint = ""
1638                 cr.Container.Mounts = map[string]arvados.Mount{
1639                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1640                 }
1641                 bindmounts, err := cr.SetupMounts()
1642                 if test.out == "error" {
1643                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1644                 } else {
1645                         c.Check(err, IsNil)
1646                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1647                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1648                         })
1649                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1650                         c.Check(err, IsNil)
1651                         c.Check(content, DeepEquals, []byte(test.out))
1652                 }
1653                 os.RemoveAll(cr.ArvMountPoint)
1654                 cr.CleanupDirs()
1655                 checkEmpty()
1656         }
1657
1658         // Read-only mount points are allowed underneath output_dir mount point
1659         {
1660                 i = 0
1661                 cr.ArvMountPoint = ""
1662                 cr.Container.Mounts = make(map[string]arvados.Mount)
1663                 cr.Container.Mounts = map[string]arvados.Mount{
1664                         "/tmp":     {Kind: "tmp"},
1665                         "/tmp/foo": {Kind: "collection"},
1666                 }
1667                 cr.Container.OutputPath = "/tmp"
1668
1669                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1670
1671                 bindmounts, err := cr.SetupMounts()
1672                 c.Check(err, IsNil)
1673                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1674                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
1675                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1676                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1677                         "/tmp":     {realTemp + "/tmp2", false},
1678                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1679                 })
1680                 os.RemoveAll(cr.ArvMountPoint)
1681                 cr.CleanupDirs()
1682                 checkEmpty()
1683         }
1684
1685         // Writable mount points copied to output_dir mount point
1686         {
1687                 i = 0
1688                 cr.ArvMountPoint = ""
1689                 cr.Container.Mounts = make(map[string]arvados.Mount)
1690                 cr.Container.Mounts = map[string]arvados.Mount{
1691                         "/tmp": {Kind: "tmp"},
1692                         "/tmp/foo": {Kind: "collection",
1693                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1694                                 Writable:         true},
1695                         "/tmp/bar": {Kind: "collection",
1696                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1697                                 Path:             "baz",
1698                                 Writable:         true},
1699                 }
1700                 cr.Container.OutputPath = "/tmp"
1701
1702                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1703                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1704
1705                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1706                 rf.Write([]byte("bar"))
1707                 rf.Close()
1708
1709                 _, err := cr.SetupMounts()
1710                 c.Check(err, IsNil)
1711                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1712                 c.Check(err, IsNil)
1713                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1714                 c.Check(err, IsNil)
1715                 os.RemoveAll(cr.ArvMountPoint)
1716                 cr.CleanupDirs()
1717                 checkEmpty()
1718         }
1719
1720         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1721         {
1722                 i = 0
1723                 cr.ArvMountPoint = ""
1724                 cr.Container.Mounts = make(map[string]arvados.Mount)
1725                 cr.Container.Mounts = map[string]arvados.Mount{
1726                         "/tmp":     {Kind: "tmp"},
1727                         "/tmp/foo": {Kind: "tmp"},
1728                 }
1729                 cr.Container.OutputPath = "/tmp"
1730
1731                 _, err := cr.SetupMounts()
1732                 c.Check(err, NotNil)
1733                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1734                 os.RemoveAll(cr.ArvMountPoint)
1735                 cr.CleanupDirs()
1736                 checkEmpty()
1737         }
1738
1739         // Only mount point of kind 'collection' is allowed for stdin
1740         {
1741                 i = 0
1742                 cr.ArvMountPoint = ""
1743                 cr.Container.Mounts = make(map[string]arvados.Mount)
1744                 cr.Container.Mounts = map[string]arvados.Mount{
1745                         "stdin": {Kind: "tmp"},
1746                 }
1747
1748                 _, err := cr.SetupMounts()
1749                 c.Check(err, NotNil)
1750                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1751                 os.RemoveAll(cr.ArvMountPoint)
1752                 cr.CleanupDirs()
1753                 checkEmpty()
1754         }
1755
1756         // git_tree mounts
1757         {
1758                 i = 0
1759                 cr.ArvMountPoint = ""
1760                 git_client.InstallProtocol("https", git_http.NewClient(arvados.InsecureHTTPClient))
1761                 cr.token = arvadostest.ActiveToken
1762                 cr.Container.Mounts = make(map[string]arvados.Mount)
1763                 cr.Container.Mounts = map[string]arvados.Mount{
1764                         "/tip": {
1765                                 Kind:   "git_tree",
1766                                 UUID:   arvadostest.Repository2UUID,
1767                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1768                                 Path:   "/",
1769                         },
1770                         "/non-tip": {
1771                                 Kind:   "git_tree",
1772                                 UUID:   arvadostest.Repository2UUID,
1773                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1774                                 Path:   "/",
1775                         },
1776                 }
1777                 cr.Container.OutputPath = "/tmp"
1778
1779                 bindmounts, err := cr.SetupMounts()
1780                 c.Check(err, IsNil)
1781
1782                 for path, mount := range bindmounts {
1783                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1784                 }
1785
1786                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1787                 c.Check(err, IsNil)
1788                 c.Check(string(data), Equals, "\000\001\002\003")
1789                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1790                 c.Check(err, FitsTypeOf, &os.PathError{})
1791                 c.Check(os.IsNotExist(err), Equals, true)
1792
1793                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1794                 c.Check(err, IsNil)
1795                 c.Check(string(data), Equals, "\000\001\002\003")
1796                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1797                 c.Check(err, IsNil)
1798                 c.Check(string(data), Equals, "testfile\n")
1799
1800                 cr.CleanupDirs()
1801                 checkEmpty()
1802         }
1803 }
1804
1805 func (s *TestSuite) TestStdout(c *C) {
1806         helperRecord := `{
1807                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1808                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1809                 "cwd": "/bin",
1810                 "environment": {"FROBIZ": "bilbo"},
1811                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1812                 "output_path": "/tmp",
1813                 "priority": 1,
1814                 "runtime_constraints": {},
1815                 "state": "Locked"
1816         }`
1817
1818         s.fullRunHelper(c, helperRecord, nil, func() int {
1819                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1820                 return 0
1821         })
1822
1823         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1824         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1825         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1826 }
1827
1828 // Used by the TestStdoutWithWrongPath*()
1829 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func() int) (*ArvTestClient, *ContainerRunner, error) {
1830         err := json.Unmarshal([]byte(record), &s.api.Container)
1831         c.Assert(err, IsNil)
1832         s.executor.runFunc = fn
1833         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1834         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1835                 return s.api, &KeepTestClient{}, nil, nil
1836         }
1837         return s.api, s.runner, s.runner.Run()
1838 }
1839
1840 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1841         _, _, err := s.stdoutErrorRunHelper(c, `{
1842     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1843     "output_path": "/tmp",
1844     "state": "Locked"
1845 }`, func() int { return 0 })
1846         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1847 }
1848
1849 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1850         _, _, err := s.stdoutErrorRunHelper(c, `{
1851     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1852     "output_path": "/tmp",
1853     "state": "Locked"
1854 }`, func() int { return 0 })
1855         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1856 }
1857
1858 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1859         _, _, err := s.stdoutErrorRunHelper(c, `{
1860     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1861     "output_path": "/tmp",
1862     "state": "Locked"
1863 }`, func() int { return 0 })
1864         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1865 }
1866
1867 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1868         s.fullRunHelper(c, `{
1869     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1870     "container_image": "`+arvadostest.DockerImage112PDH+`",
1871     "cwd": "/bin",
1872     "environment": {},
1873     "mounts": {"/tmp": {"kind": "tmp"} },
1874     "output_path": "/tmp",
1875     "priority": 1,
1876     "runtime_constraints": {"API": true},
1877     "state": "Locked"
1878 }`, nil, func() int {
1879                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1880                 return 3
1881         })
1882         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1883         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1884         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1885 }
1886
1887 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1888         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1889         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1890         s.fullRunHelper(c, `{
1891     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1892     "container_image": "`+arvadostest.DockerImage112PDH+`",
1893     "cwd": "/bin",
1894     "environment": {},
1895     "mounts": {"/tmp": {"kind": "tmp"} },
1896     "output_path": "/tmp",
1897     "priority": 1,
1898     "runtime_constraints": {"API": true},
1899     "state": "Locked"
1900 }`, nil, func() int {
1901                 s.api.Container.Output = arvadostest.DockerImage112PDH
1902                 return 0
1903         })
1904
1905         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1906         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1907         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1908 }
1909
1910 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1911         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1912                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1913                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1914                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1915         }
1916         s.executor.runFunc = func() int {
1917                 time.Sleep(time.Second)
1918                 return 137
1919         }
1920         record := `{
1921     "command": ["sleep", "1"],
1922     "container_image": "` + arvadostest.DockerImage112PDH + `",
1923     "cwd": "/bin",
1924     "environment": {},
1925     "mounts": {"/tmp": {"kind": "tmp"} },
1926     "output_path": "/tmp",
1927     "priority": 1,
1928     "runtime_constraints": {"API": true},
1929     "state": "Locked"
1930 }`
1931         err := json.Unmarshal([]byte(record), &s.api.Container)
1932         c.Assert(err, IsNil)
1933         err = s.runner.Run()
1934         c.Assert(err, IsNil)
1935         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1936         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1937         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1938         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1939         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1940 }
1941
1942 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1943         helperRecord := `{
1944                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1945                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1946                 "cwd": "/bin",
1947                 "environment": {"FROBIZ": "bilbo"},
1948                 "mounts": {
1949         "/tmp": {"kind": "tmp"},
1950         "/tmp/foo": {"kind": "collection",
1951                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1952                      "exclude_from_output": true
1953         },
1954         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1955     },
1956                 "output_path": "/tmp",
1957                 "priority": 1,
1958                 "runtime_constraints": {},
1959                 "state": "Locked"
1960         }`
1961
1962         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1963
1964         s.fullRunHelper(c, helperRecord, extraMounts, func() int {
1965                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1966                 return 0
1967         })
1968
1969         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1970         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1971         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1972 }
1973
1974 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1975         helperRecord := `{
1976                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1977                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1978                 "cwd": "/bin",
1979                 "environment": {"FROBIZ": "bilbo"},
1980                 "mounts": {
1981         "/tmp": {"kind": "tmp"},
1982         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1983         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1984         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1985         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1986         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1987     },
1988                 "output_path": "/tmp",
1989                 "priority": 1,
1990                 "runtime_constraints": {},
1991                 "state": "Locked",
1992                 "uuid": "zzzzz-dz642-202301130848001"
1993         }`
1994
1995         extraMounts := []string{
1996                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1997                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1998                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1999         }
2000
2001         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, func() int {
2002                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
2003                 return 0
2004         })
2005
2006         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
2007                 "/tmp":                   {realtemp + "/tmp1", false},
2008                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
2009                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
2010                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
2011                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
2012         })
2013
2014         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
2015         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
2016         output_count := uint(0)
2017         for _, v := range s.runner.ContainerArvClient.(*ArvTestClient).Content {
2018                 if v["collection"] == nil {
2019                         continue
2020                 }
2021                 collection := v["collection"].(arvadosclient.Dict)
2022                 if collection["name"].(string) != "output for zzzzz-dz642-202301130848001" {
2023                         continue
2024                 }
2025                 c.Check(v["ensure_unique_name"], Equals, true)
2026                 c.Check(collection["manifest_text"].(string), Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
2027 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
2028 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
2029 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
2030 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
2031 `)
2032                 output_count++
2033         }
2034         c.Check(output_count, Not(Equals), uint(0))
2035 }
2036
2037 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
2038         helperRecord := `{
2039                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
2040                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2041                 "cwd": "/bin",
2042                 "environment": {"FROBIZ": "bilbo"},
2043                 "mounts": {
2044         "/tmp": {"kind": "tmp"},
2045         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
2046         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
2047     },
2048                 "output_path": "/tmp",
2049                 "priority": 1,
2050                 "runtime_constraints": {},
2051                 "state": "Locked",
2052                 "uuid": "zzzzz-dz642-202301130848002"
2053         }`
2054
2055         extraMounts := []string{
2056                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
2057         }
2058
2059         s.fullRunHelper(c, helperRecord, extraMounts, func() int {
2060                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
2061                 return 0
2062         })
2063
2064         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2065         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2066         output_count := uint(0)
2067         for _, v := range s.runner.ContainerArvClient.(*ArvTestClient).Content {
2068                 if v["collection"] == nil {
2069                         continue
2070                 }
2071                 collection := v["collection"].(arvadosclient.Dict)
2072                 if collection["name"].(string) != "output for zzzzz-dz642-202301130848002" {
2073                         continue
2074                 }
2075                 c.Check(collection["manifest_text"].(string), Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
2076 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
2077 `)
2078                 output_count++
2079         }
2080         c.Check(output_count, Not(Equals), uint(0))
2081 }
2082
2083 func (s *TestSuite) TestOutputError(c *C) {
2084         helperRecord := `{
2085                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
2086                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2087                 "cwd": "/bin",
2088                 "environment": {"FROBIZ": "bilbo"},
2089                 "mounts": {
2090                         "/tmp": {"kind": "tmp"}
2091                 },
2092                 "output_path": "/tmp",
2093                 "priority": 1,
2094                 "runtime_constraints": {},
2095                 "state": "Locked"
2096         }`
2097         s.fullRunHelper(c, helperRecord, nil, func() int {
2098                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
2099                 return 0
2100         })
2101
2102         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
2103 }
2104
2105 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
2106         helperRecord := `{
2107                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
2108                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2109                 "cwd": "/bin",
2110                 "environment": {"FROBIZ": "bilbo"},
2111                 "mounts": {
2112         "/tmp": {"kind": "tmp"},
2113         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
2114         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
2115     },
2116                 "output_path": "/tmp",
2117                 "priority": 1,
2118                 "runtime_constraints": {},
2119                 "state": "Locked"
2120         }`
2121
2122         extraMounts := []string{
2123                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
2124         }
2125
2126         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, func() int {
2127                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
2128                 return 0
2129         })
2130
2131         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
2132         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
2133         for _, v := range api.Content {
2134                 if v["collection"] != nil {
2135                         collection := v["collection"].(arvadosclient.Dict)
2136                         if strings.Index(collection["name"].(string), "output") == 0 {
2137                                 manifest := collection["manifest_text"].(string)
2138                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
2139 `)
2140                         }
2141                 }
2142         }
2143 }
2144
2145 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
2146         helperRecord := `{
2147                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
2148                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2149                 "cwd": "/bin",
2150                 "environment": {"FROBIZ": "bilbo"},
2151                 "mounts": {
2152         "/tmp": {"kind": "tmp"},
2153         "stdin": {"kind": "json", "content": "foo"},
2154         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
2155     },
2156                 "output_path": "/tmp",
2157                 "priority": 1,
2158                 "runtime_constraints": {},
2159                 "state": "Locked"
2160         }`
2161
2162         api, _, _ := s.fullRunHelper(c, helperRecord, nil, func() int {
2163                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
2164                 return 0
2165         })
2166
2167         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
2168         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
2169         for _, v := range api.Content {
2170                 if v["collection"] != nil {
2171                         collection := v["collection"].(arvadosclient.Dict)
2172                         if strings.Index(collection["name"].(string), "output") == 0 {
2173                                 manifest := collection["manifest_text"].(string)
2174                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
2175 `)
2176                         }
2177                 }
2178         }
2179 }
2180
2181 func (s *TestSuite) TestStderrMount(c *C) {
2182         api, cr, _ := s.fullRunHelper(c, `{
2183     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
2184     "container_image": "`+arvadostest.DockerImage112PDH+`",
2185     "cwd": ".",
2186     "environment": {},
2187     "mounts": {"/tmp": {"kind": "tmp"},
2188                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
2189                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
2190     "output_path": "/tmp",
2191     "priority": 1,
2192     "runtime_constraints": {},
2193     "state": "Locked"
2194 }`, nil, func() int {
2195                 fmt.Fprintln(s.executor.created.Stdout, "hello")
2196                 fmt.Fprintln(s.executor.created.Stderr, "oops")
2197                 return 1
2198         })
2199
2200         final := api.CalledWith("container.state", "Complete")
2201         c.Assert(final, NotNil)
2202         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
2203         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
2204
2205         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
2206 }
2207
2208 func (s *TestSuite) TestNumberRoundTrip(c *C) {
2209         s.api.callraw = true
2210         err := s.runner.fetchContainerRecord()
2211         c.Assert(err, IsNil)
2212         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
2213         c.Logf("%#v", s.runner.Container)
2214         c.Check(err, IsNil)
2215         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
2216 }
2217
2218 func (s *TestSuite) TestFullBrokenDocker(c *C) {
2219         nextState := ""
2220         for _, setup := range []func(){
2221                 func() {
2222                         c.Log("// waitErr = ocl runtime error")
2223                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
2224                         nextState = "Cancelled"
2225                 },
2226                 func() {
2227                         c.Log("// loadErr = cannot connect")
2228                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
2229                         s.runner.brokenNodeHook = c.MkDir() + "/broken-node-hook"
2230                         err := ioutil.WriteFile(s.runner.brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
2231                         c.Assert(err, IsNil)
2232                         nextState = "Queued"
2233                 },
2234         } {
2235                 s.SetUpTest(c)
2236                 setup()
2237                 s.fullRunHelper(c, `{
2238     "command": ["echo", "hello world"],
2239     "container_image": "`+arvadostest.DockerImage112PDH+`",
2240     "cwd": ".",
2241     "environment": {},
2242     "mounts": {"/tmp": {"kind": "tmp"} },
2243     "output_path": "/tmp",
2244     "priority": 1,
2245     "runtime_constraints": {},
2246     "state": "Locked"
2247 }`, nil, func() int { return 0 })
2248                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
2249                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
2250                 if s.runner.brokenNodeHook != "" {
2251                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
2252                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
2253                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
2254                 } else {
2255                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
2256                 }
2257         }
2258 }
2259
2260 func (s *TestSuite) TestBadCommand(c *C) {
2261         for _, startError := range []string{
2262                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
2263                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
2264                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
2265         } {
2266                 s.SetUpTest(c)
2267                 s.executor.startErr = errors.New(startError)
2268                 s.fullRunHelper(c, `{
2269     "command": ["echo", "hello world"],
2270     "container_image": "`+arvadostest.DockerImage112PDH+`",
2271     "cwd": ".",
2272     "environment": {},
2273     "mounts": {"/tmp": {"kind": "tmp"} },
2274     "output_path": "/tmp",
2275     "priority": 1,
2276     "runtime_constraints": {},
2277     "state": "Locked"
2278 }`, nil, func() int { return 0 })
2279                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
2280                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
2281         }
2282 }
2283
2284 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
2285         helperRecord := `{
2286                 "command": ["true"],
2287                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2288                 "cwd": "/bin",
2289                 "mounts": {
2290                     "/tmp": {"kind": "tmp"},
2291                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2292                 },
2293                 "secret_mounts": {
2294                 },
2295                 "output_path": "/tmp",
2296                 "priority": 1,
2297                 "runtime_constraints": {},
2298                 "state": "Locked"
2299         }`
2300
2301         s.fullRunHelper(c, helperRecord, nil, func() int {
2302                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2303                 c.Check(err, IsNil)
2304                 c.Check(string(content), Equals, "mypassword")
2305                 return 0
2306         })
2307
2308         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2309         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2310         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
2311         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
2312
2313         // under secret mounts, not captured in output
2314         helperRecord = `{
2315                 "command": ["true"],
2316                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2317                 "cwd": "/bin",
2318                 "mounts": {
2319                     "/tmp": {"kind": "tmp"}
2320                 },
2321                 "secret_mounts": {
2322                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2323                 },
2324                 "output_path": "/tmp",
2325                 "priority": 1,
2326                 "runtime_constraints": {},
2327                 "state": "Locked"
2328         }`
2329
2330         s.SetUpTest(c)
2331         s.fullRunHelper(c, helperRecord, nil, func() int {
2332                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2333                 c.Check(err, IsNil)
2334                 c.Check(string(content), Equals, "mypassword")
2335                 return 0
2336         })
2337
2338         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2339         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2340         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2341         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2342
2343         // under secret mounts, output dir is a collection, not captured in output
2344         helperRecord = `{
2345                 "command": ["true"],
2346                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2347                 "cwd": "/bin",
2348                 "mounts": {
2349                     "/tmp": {"kind": "collection", "writable": true}
2350                 },
2351                 "secret_mounts": {
2352                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2353                 },
2354                 "output_path": "/tmp",
2355                 "priority": 1,
2356                 "runtime_constraints": {},
2357                 "state": "Locked"
2358         }`
2359
2360         s.SetUpTest(c)
2361         _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, func() int {
2362                 // secret.conf should be provisioned as a separate
2363                 // bind mount, i.e., it should not appear in the
2364                 // (fake) fuse filesystem as viewed from the host.
2365                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2366                 if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) {
2367                         c.Logf("secret.conf: content %q, err %#v", content, err)
2368                 }
2369                 err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700)
2370                 c.Check(err, IsNil)
2371                 return 0
2372         })
2373
2374         content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text")
2375         c.Check(err, IsNil)
2376         c.Check(string(content), Equals, "mypassword")
2377         c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true})
2378         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2379         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2380         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil)
2381 }
2382
2383 func (s *TestSuite) TestCalculateCost(c *C) {
2384         defer func(s string) { lockdir = s }(lockdir)
2385         lockdir = c.MkDir()
2386         now := time.Now()
2387         cr := s.runner
2388         cr.costStartTime = now.Add(-time.Hour)
2389         var logbuf bytes.Buffer
2390         cr.CrunchLog.Immediate = log.New(&logbuf, "", 0)
2391
2392         // if there's no InstanceType env var, cost is calculated as 0
2393         os.Unsetenv("InstanceType")
2394         cost := cr.calculateCost(now)
2395         c.Check(cost, Equals, 0.0)
2396
2397         // with InstanceType env var and loadPrices() hasn't run (or
2398         // hasn't found any data), cost is calculated based on
2399         // InstanceType env var
2400         os.Setenv("InstanceType", `{"Price":1.2}`)
2401         cost = cr.calculateCost(now)
2402         c.Check(cost, Equals, 1.2)
2403
2404         // first update tells us the spot price was $1/h until 30
2405         // minutes ago when it increased to $2/h
2406         j, err := json.Marshal([]cloud.InstancePrice{
2407                 {StartTime: now.Add(-4 * time.Hour), Price: 1.0},
2408                 {StartTime: now.Add(-time.Hour / 2), Price: 2.0},
2409         })
2410         c.Assert(err, IsNil)
2411         os.WriteFile(lockdir+"/"+pricesfile, j, 0777)
2412         cr.loadPrices()
2413         cost = cr.calculateCost(now)
2414         c.Check(cost, Equals, 1.5)
2415
2416         // next update (via --list + SIGUSR2) tells us the spot price
2417         // increased to $3/h 15 minutes ago
2418         j, err = json.Marshal([]cloud.InstancePrice{
2419                 {StartTime: now.Add(-time.Hour / 3), Price: 2.0}, // dup of -time.Hour/2 price
2420                 {StartTime: now.Add(-time.Hour / 4), Price: 3.0},
2421         })
2422         c.Assert(err, IsNil)
2423         os.WriteFile(lockdir+"/"+pricesfile, j, 0777)
2424         cr.loadPrices()
2425         cost = cr.calculateCost(now)
2426         c.Check(cost, Equals, 1.0/2+2.0/4+3.0/4)
2427
2428         cost = cr.calculateCost(now.Add(-time.Hour / 2))
2429         c.Check(cost, Equals, 0.5)
2430
2431         c.Logf("%s", logbuf.String())
2432         c.Check(logbuf.String(), Matches, `(?ms).*Instance price changed to 1\.00 at 20.* changed to 2\.00 .* changed to 3\.00 .*`)
2433         c.Check(logbuf.String(), Not(Matches), `(?ms).*changed to 2\.00 .* changed to 2\.00 .*`)
2434 }
2435
2436 func (s *TestSuite) TestSIGUSR2CostUpdate(c *C) {
2437         pid := os.Getpid()
2438         now := time.Now()
2439         pricesJSON, err := json.Marshal([]cloud.InstancePrice{
2440                 {StartTime: now.Add(-4 * time.Hour), Price: 2.4},
2441                 {StartTime: now.Add(-2 * time.Hour), Price: 2.6},
2442         })
2443         c.Assert(err, IsNil)
2444
2445         os.Setenv("InstanceType", `{"Price":2.2}`)
2446         defer func(s string) { lockdir = s }(lockdir)
2447         lockdir = c.MkDir()
2448
2449         // We can't use s.api.CalledWith because timing differences will yield
2450         // different cost values across runs. getCostUpdate iterates over API
2451         // calls until it finds one that sets the cost, then writes that value
2452         // to the next index of costUpdates.
2453         deadline := now.Add(time.Second)
2454         costUpdates := make([]float64, 2)
2455         costIndex := 0
2456         apiIndex := 0
2457         getCostUpdate := func() {
2458                 for ; time.Now().Before(deadline); time.Sleep(time.Second / 10) {
2459                         for apiIndex < len(s.api.Content) {
2460                                 update := s.api.Content[apiIndex]
2461                                 apiIndex++
2462                                 var ok bool
2463                                 var cost float64
2464                                 if update, ok = update["container"].(arvadosclient.Dict); !ok {
2465                                         continue
2466                                 }
2467                                 if cost, ok = update["cost"].(float64); !ok {
2468                                         continue
2469                                 }
2470                                 c.Logf("API call #%d updates cost to %v", apiIndex-1, cost)
2471                                 costUpdates[costIndex] = cost
2472                                 costIndex++
2473                                 return
2474                         }
2475                 }
2476         }
2477
2478         s.fullRunHelper(c, `{
2479                 "command": ["true"],
2480                 "container_image": "`+arvadostest.DockerImage112PDH+`",
2481                 "cwd": ".",
2482                 "environment": {},
2483                 "mounts": {"/tmp": {"kind": "tmp"} },
2484                 "output_path": "/tmp",
2485                 "priority": 1,
2486                 "runtime_constraints": {},
2487                 "state": "Locked",
2488                 "uuid": "zzzzz-dz642-20230320101530a"
2489         }`, nil, func() int {
2490                 s.runner.costStartTime = now.Add(-3 * time.Hour)
2491                 err := syscall.Kill(pid, syscall.SIGUSR2)
2492                 c.Check(err, IsNil, Commentf("error sending first SIGUSR2 to runner"))
2493                 getCostUpdate()
2494
2495                 err = os.WriteFile(path.Join(lockdir, pricesfile), pricesJSON, 0o700)
2496                 c.Check(err, IsNil, Commentf("error writing JSON prices file"))
2497                 err = syscall.Kill(pid, syscall.SIGUSR2)
2498                 c.Check(err, IsNil, Commentf("error sending second SIGUSR2 to runner"))
2499                 getCostUpdate()
2500
2501                 return 0
2502         })
2503         // Comparing with format strings makes it easy to ignore minor variations
2504         // in cost across runs while keeping diagnostics pretty.
2505         c.Check(fmt.Sprintf("%.3f", costUpdates[0]), Equals, "6.600")
2506         c.Check(fmt.Sprintf("%.3f", costUpdates[1]), Equals, "7.600")
2507 }
2508
2509 type FakeProcess struct {
2510         cmdLine []string
2511 }
2512
2513 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2514         return fp.cmdLine, nil
2515 }