lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/md5"
  11         "encoding/json"
  12         "errors"
  13         "fmt"
  14         "io"
  15         "io/ioutil"
  16         "log"
  17         "math/rand"
  18         "net/http"
  19         "net/http/httptest"
  20         "net/http/httputil"
  21         "net/url"
  22         "os"
  23         "os/exec"
  24         "path"
  25         "regexp"
  26         "runtime/pprof"
  27         "strconv"
  28         "strings"
  29         "sync"
  30         "sync/atomic"
  31         "syscall"
  32         "testing"
  33         "time"
  34
  35         "git.arvados.org/arvados.git/lib/cloud"
  36         "git.arvados.org/arvados.git/lib/cmd"
  37         "git.arvados.org/arvados.git/sdk/go/arvados"
  38         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  39         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  40         "git.arvados.org/arvados.git/sdk/go/manifest"
  41
  42         . "gopkg.in/check.v1"
  43         git_client "gopkg.in/src-d/go-git.v4/plumbing/transport/client"
  44         git_http "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
  45 )
  46
  47 // Gocheck boilerplate
  48 func TestCrunchExec(t *testing.T) {
  49         TestingT(t)
  50 }
  51
  52 const logLineStart = `(?m)(.*\n)*\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d+Z `
  53
  54 var _ = Suite(&TestSuite{})
  55
  56 type TestSuite struct {
  57         client                   *arvados.Client
  58         api                      *ArvTestClient
  59         runner                   *ContainerRunner
  60         executor                 *stubExecutor
  61         keepmount                string
  62         keepmountTmp             []string
  63         testDispatcherKeepClient KeepTestClient
  64         testContainerKeepClient  KeepTestClient
  65 }
  66
  67 func (s *TestSuite) SetUpTest(c *C) {
  68         s.client = arvados.NewClientFromEnv()
  69         s.executor = &stubExecutor{}
  70         var err error
  71         s.api = &ArvTestClient{}
  72         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  73         c.Assert(err, IsNil)
  74         s.runner.executor = s.executor
  75         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  76                 return s.api, &s.testContainerKeepClient, s.client, nil
  77         }
  78         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  79                 s.runner.ArvMountPoint = s.keepmount
  80                 for i, opt := range cmd {
  81                         if opt == "--mount-tmp" {
  82                                 err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700)
  83                                 if err != nil {
  84                                         return nil, err
  85                                 }
  86                                 s.keepmountTmp = append(s.keepmountTmp, cmd[i+1])
  87                         }
  88                 }
  89                 return nil, nil
  90         }
  91         s.keepmount = c.MkDir()
  92         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  93         s.keepmountTmp = nil
  94         c.Assert(err, IsNil)
  95         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  96         c.Assert(err, IsNil)
  97         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  98         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  99         c.Assert(err, IsNil)
 100         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
 101         c.Assert(err, IsNil)
 102         s.runner.ArvMountPoint = s.keepmount
 103 }
 104
 105 type ArvTestClient struct {
 106         Total   int64
 107         Calls   int
 108         Content []arvadosclient.Dict
 109         arvados.Container
 110         secretMounts []byte
 111         Logs         map[string]*bytes.Buffer
 112         sync.Mutex
 113         WasSetRunning bool
 114         callraw       bool
 115 }
 116
 117 type KeepTestClient struct {
 118         Called         bool
 119         Content        []byte
 120         StorageClasses []string
 121 }
 122
 123 type stubExecutor struct {
 124         imageLoaded bool
 125         loaded      string
 126         loadErr     error
 127         exitCode    int
 128         createErr   error
 129         created     containerSpec
 130         startErr    error
 131         waitSleep   time.Duration
 132         waitErr     error
 133         stopErr     error
 134         stopped     bool
 135         closed      bool
 136         runFunc     func() int
 137         exit        chan int
 138 }
 139
 140 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 141         containerClient *arvados.Client) error {
 142         e.loaded = tarball
 143         return e.loadErr
 144 }
 145 func (e *stubExecutor) Runtime() string                 { return "stub" }
 146 func (e *stubExecutor) Version() string                 { return "stub " + cmd.Version.String() }
 147 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 148 func (e *stubExecutor) Start() error {
 149         e.exit = make(chan int, 1)
 150         go func() { e.exit <- e.runFunc() }()
 151         return e.startErr
 152 }
 153 func (e *stubExecutor) CgroupID() string { return "cgroupid" }
 154 func (e *stubExecutor) Stop() error      { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 155 func (e *stubExecutor) Close()           { e.closed = true }
 156 func (e *stubExecutor) Wait(context.Context) (int, error) {
 157         return <-e.exit, e.waitErr
 158 }
 159 func (e *stubExecutor) InjectCommand(ctx context.Context, _, _ string, _ bool, _ []string) (*exec.Cmd, error) {
 160         return nil, errors.New("unimplemented")
 161 }
 162 func (e *stubExecutor) IPAddress() (string, error) { return "", errors.New("unimplemented") }
 163
 164 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 165
 166 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 167 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 168 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 169
 170 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 171 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 172
 173 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 174 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 175 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 176 `
 177
 178 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 179
 180 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 181 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 182
 183 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 184 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 185
 186 func (client *ArvTestClient) Create(resourceType string,
 187         parameters arvadosclient.Dict,
 188         output interface{}) error {
 189
 190         client.Mutex.Lock()
 191         defer client.Mutex.Unlock()
 192
 193         client.Calls++
 194         client.Content = append(client.Content, parameters)
 195
 196         if resourceType == "logs" {
 197                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 198                 if client.Logs == nil {
 199                         client.Logs = make(map[string]*bytes.Buffer)
 200                 }
 201                 if client.Logs[et] == nil {
 202                         client.Logs[et] = &bytes.Buffer{}
 203                 }
 204                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 205         }
 206
 207         if resourceType == "collections" && output != nil {
 208                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 209                 md5sum := md5.Sum([]byte(mt))
 210                 outmap := output.(*arvados.Collection)
 211                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5sum, len(mt))
 212                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%015x", md5sum[:7])
 213         }
 214
 215         return nil
 216 }
 217
 218 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 219         switch {
 220         case method == "GET" && resourceType == "containers" && action == "auth":
 221                 return json.Unmarshal([]byte(`{
 222                         "kind": "arvados#api_client_authorization",
 223                         "uuid": "`+fakeAuthUUID+`",
 224                         "api_token": "`+fakeAuthToken+`"
 225                         }`), output)
 226         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 227                 if client.secretMounts != nil {
 228                         return json.Unmarshal(client.secretMounts, output)
 229                 }
 230                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 231         default:
 232                 return fmt.Errorf("Not found")
 233         }
 234 }
 235
 236 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 237         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 238         var j []byte
 239         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 240                 j = []byte(`{
 241                         "kind": "arvados#nodeList",
 242                         "items": [{
 243                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 244                                 "hostname": "compute2",
 245                                 "properties": {"total_cpu_cores": 16}
 246                         }]}`)
 247         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 248                 if uuid == "" {
 249                         j, err = json.Marshal(map[string]interface{}{
 250                                 "items": []interface{}{client.Container},
 251                                 "kind":  "arvados#nodeList",
 252                         })
 253                 } else {
 254                         j, err = json.Marshal(client.Container)
 255                 }
 256         } else {
 257                 j = []byte(`{
 258                         "command": ["sleep", "1"],
 259                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 260                         "cwd": ".",
 261                         "environment": {},
 262                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 263                         "output_path": "/tmp",
 264                         "priority": 1,
 265                         "runtime_constraints": {}
 266                 }`)
 267         }
 268         return ioutil.NopCloser(bytes.NewReader(j)), err
 269 }
 270
 271 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 272         if resourceType == "collections" {
 273                 if uuid == hwPDH {
 274                         output.(*arvados.Collection).ManifestText = hwManifest
 275                 } else if uuid == otherPDH {
 276                         output.(*arvados.Collection).ManifestText = otherManifest
 277                 } else if uuid == normalizedWithSubdirsPDH {
 278                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 279                 } else if uuid == denormalizedWithSubdirsPDH {
 280                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 281                 }
 282         }
 283         if resourceType == "containers" {
 284                 (*output.(*arvados.Container)) = client.Container
 285         }
 286         return nil
 287 }
 288
 289 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 290         client.Mutex.Lock()
 291         defer client.Mutex.Unlock()
 292         client.Calls++
 293         client.Content = append(client.Content, parameters)
 294         if resourceType == "containers" {
 295                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 296                         client.WasSetRunning = true
 297                 }
 298         } else if resourceType == "collections" && output != nil {
 299                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 300                 output.(*arvados.Collection).UUID = uuid
 301                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 302         }
 303         return nil
 304 }
 305
 306 var discoveryMap = map[string]interface{}{
 307         "defaultTrashLifetime":               float64(1209600),
 308         "crunchLimitLogBytesPerJob":          float64(67108864),
 309         "crunchLogThrottleBytes":             float64(65536),
 310         "crunchLogThrottlePeriod":            float64(60),
 311         "crunchLogThrottleLines":             float64(1024),
 312         "crunchLogPartialLineThrottlePeriod": float64(5),
 313         "crunchLogBytesPerEvent":             float64(4096),
 314         "crunchLogSecondsBetweenEvents":      float64(1),
 315 }
 316
 317 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 318         return discoveryMap[key], nil
 319 }
 320
 321 // CalledWith returns the parameters from the first API call whose
 322 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 323 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 324 // no call matches, it returns nil.
 325 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 326 call:
 327         for _, content := range client.Content {
 328                 var v interface{} = content
 329                 for _, k := range strings.Split(jpath, ".") {
 330                         if dict, ok := v.(arvadosclient.Dict); !ok {
 331                                 continue call
 332                         } else {
 333                                 v = dict[k]
 334                         }
 335                 }
 336                 if v == expect {
 337                         return content
 338                 }
 339         }
 340         return nil
 341 }
 342
 343 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 344         return locator, nil
 345 }
 346
 347 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 348         client.Content = opts.Data
 349         return arvados.BlockWriteResponse{
 350                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 351         }, nil
 352 }
 353
 354 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 355         return 0, errors.New("not implemented")
 356 }
 357
 358 func (client *KeepTestClient) ClearBlockCache() {
 359 }
 360
 361 func (client *KeepTestClient) Close() {
 362         client.Content = nil
 363 }
 364
 365 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 366         client.StorageClasses = sc
 367 }
 368
 369 type FileWrapper struct {
 370         io.ReadCloser
 371         len int64
 372 }
 373
 374 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 375         return nil, errors.New("not implemented")
 376 }
 377
 378 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 379         return 0, errors.New("not implemented")
 380 }
 381
 382 func (fw FileWrapper) Size() int64 {
 383         return fw.len
 384 }
 385
 386 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 387         return nil, errors.New("not implemented")
 388 }
 389
 390 func (fw FileWrapper) Truncate(int64) error {
 391         return errors.New("not implemented")
 392 }
 393
 394 func (fw FileWrapper) Write([]byte) (int, error) {
 395         return 0, errors.New("not implemented")
 396 }
 397
 398 func (fw FileWrapper) Sync() error {
 399         return errors.New("not implemented")
 400 }
 401
 402 func (fw FileWrapper) Snapshot() (*arvados.Subtree, error) {
 403         return nil, errors.New("not implemented")
 404 }
 405
 406 func (fw FileWrapper) Splice(*arvados.Subtree) error {
 407         return errors.New("not implemented")
 408 }
 409
 410 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 411         if filename == hwImageID+".tar" {
 412                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 413                 client.Called = true
 414                 return FileWrapper{rdr, 1321984}, nil
 415         } else if filename == "/file1_in_main.txt" {
 416                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 417                 client.Called = true
 418                 return FileWrapper{rdr, 3}, nil
 419         }
 420         return nil, nil
 421 }
 422
 423 type apiStubServer struct {
 424         server    *httptest.Server
 425         proxy     *httputil.ReverseProxy
 426         intercept func(http.ResponseWriter, *http.Request) bool
 427
 428         container arvados.Container
 429         logs      map[string]string
 430 }
 431
 432 func apiStub() (*arvados.Client, *apiStubServer) {
 433         client := arvados.NewClientFromEnv()
 434         apistub := &apiStubServer{}
 435         apistub.server = httptest.NewTLSServer(apistub)
 436         apistub.proxy = httputil.NewSingleHostReverseProxy(&url.URL{Scheme: "https", Host: client.APIHost})
 437         if client.Insecure {
 438                 apistub.proxy.Transport = arvados.InsecureHTTPClient.Transport
 439         }
 440         client.APIHost = apistub.server.Listener.Addr().String()
 441         return client, apistub
 442 }
 443
 444 func (apistub *apiStubServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 445         if apistub.intercept != nil && apistub.intercept(w, r) {
 446                 return
 447         }
 448         if r.Method == "POST" && r.URL.Path == "/arvados/v1/logs" {
 449                 var body struct {
 450                         Log struct {
 451                                 EventType  string `json:"event_type"`
 452                                 Properties struct {
 453                                         Text string
 454                                 }
 455                         }
 456                 }
 457                 json.NewDecoder(r.Body).Decode(&body)
 458                 apistub.logs[body.Log.EventType] += body.Log.Properties.Text
 459                 return
 460         }
 461         if r.Method == "GET" && r.URL.Path == "/arvados/v1/collections/"+hwPDH {
 462                 json.NewEncoder(w).Encode(arvados.Collection{ManifestText: hwManifest})
 463                 return
 464         }
 465         if r.Method == "GET" && r.URL.Path == "/arvados/v1/collections/"+otherPDH {
 466                 json.NewEncoder(w).Encode(arvados.Collection{ManifestText: otherManifest})
 467                 return
 468         }
 469         if r.Method == "GET" && r.URL.Path == "/arvados/v1/collections/"+normalizedWithSubdirsPDH {
 470                 json.NewEncoder(w).Encode(arvados.Collection{ManifestText: normalizedManifestWithSubdirs})
 471                 return
 472         }
 473         if r.Method == "GET" && r.URL.Path == "/arvados/v1/collections/"+denormalizedWithSubdirsPDH {
 474                 json.NewEncoder(w).Encode(arvados.Collection{ManifestText: denormalizedManifestWithSubdirs})
 475                 return
 476         }
 477         if r.Method == "GET" && r.URL.Path == "/arvados/v1/containers/"+apistub.container.UUID {
 478                 json.NewEncoder(w).Encode(apistub.container)
 479                 return
 480         }
 481         apistub.proxy.ServeHTTP(w, r)
 482 }
 483
 484 func (s *TestSuite) TestLoadImage(c *C) {
 485         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 486         s.runner.Container.Mounts = map[string]arvados.Mount{
 487                 "/out": {Kind: "tmp", Writable: true},
 488         }
 489         s.runner.Container.OutputPath = "/out"
 490
 491         _, err := s.runner.SetupMounts()
 492         c.Assert(err, IsNil)
 493
 494         imageID, err := s.runner.LoadImage()
 495         c.Check(err, IsNil)
 496         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 497         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 498
 499         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 500         s.executor.imageLoaded = false
 501         s.executor.loaded = ""
 502         s.executor.loadErr = errors.New("bork")
 503         imageID, err = s.runner.LoadImage()
 504         c.Check(err, ErrorMatches, ".*bork")
 505         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 506
 507         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 508         s.executor.imageLoaded = false
 509         s.executor.loaded = ""
 510         s.executor.loadErr = nil
 511         imageID, err = s.runner.LoadImage()
 512         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 513         c.Check(s.executor.loaded, Equals, "")
 514 }
 515
 516 type ArvErrorTestClient struct{}
 517
 518 func (ArvErrorTestClient) Create(resourceType string,
 519         parameters arvadosclient.Dict,
 520         output interface{}) error {
 521         return nil
 522 }
 523
 524 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 525         if method == "GET" && resourceType == "containers" && action == "auth" {
 526                 return nil
 527         }
 528         return errors.New("ArvError")
 529 }
 530
 531 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 532         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 533         return nil, errors.New("ArvError")
 534 }
 535
 536 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 537         return errors.New("ArvError")
 538 }
 539
 540 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 541         return nil
 542 }
 543
 544 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 545         return discoveryMap[key], nil
 546 }
 547
 548 type KeepErrorTestClient struct {
 549         KeepTestClient
 550 }
 551
 552 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 553         return nil, errors.New("KeepError")
 554 }
 555
 556 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 557         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 558 }
 559
 560 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 561         return "", errors.New("KeepError")
 562 }
 563
 564 type KeepReadErrorTestClient struct {
 565         KeepTestClient
 566 }
 567
 568 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 569         return 0, errors.New("KeepError")
 570 }
 571
 572 type ErrorReader struct {
 573         FileWrapper
 574 }
 575
 576 func (ErrorReader) Read(p []byte) (n int, err error) {
 577         return 0, errors.New("ErrorReader")
 578 }
 579
 580 func (ErrorReader) Seek(int64, int) (int64, error) {
 581         return 0, errors.New("ErrorReader")
 582 }
 583
 584 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 585         return ErrorReader{}, nil
 586 }
 587
 588 type ClosableBuffer struct {
 589         bytes.Buffer
 590 }
 591
 592 func (*ClosableBuffer) Close() error {
 593         return nil
 594 }
 595
 596 type TestLogs struct {
 597         Stdout ClosableBuffer
 598         Stderr ClosableBuffer
 599 }
 600
 601 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 602         if logstr == "stdout" {
 603                 return &tl.Stdout, nil
 604         }
 605         if logstr == "stderr" {
 606                 return &tl.Stderr, nil
 607         }
 608         return nil, errors.New("???")
 609 }
 610
 611 func dockerLog(fd byte, msg string) []byte {
 612         by := []byte(msg)
 613         header := make([]byte, 8+len(by))
 614         header[0] = fd
 615         header[7] = byte(len(by))
 616         copy(header[8:], by)
 617         return header
 618 }
 619
 620 func (s *TestSuite) TestRunContainer(c *C) {
 621         s.executor.runFunc = func() int {
 622                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 623                 return 0
 624         }
 625
 626         var logs TestLogs
 627         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 628         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 629         s.runner.Container.Command = []string{"./hw"}
 630         s.runner.Container.OutputStorageClasses = []string{"default"}
 631
 632         imageID, err := s.runner.LoadImage()
 633         c.Assert(err, IsNil)
 634
 635         err = s.runner.CreateContainer(imageID, nil)
 636         c.Assert(err, IsNil)
 637
 638         err = s.runner.StartContainer()
 639         c.Assert(err, IsNil)
 640
 641         err = s.runner.WaitFinish()
 642         c.Assert(err, IsNil)
 643
 644         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 645         c.Check(logs.Stderr.String(), Equals, "")
 646 }
 647
 648 func (s *TestSuite) TestCommitLogs(c *C) {
 649         api := &ArvTestClient{}
 650         kc := &KeepTestClient{}
 651         defer kc.Close()
 652         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 653         c.Assert(err, IsNil)
 654         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 655
 656         cr.CrunchLog.Print("Hello world!")
 657         cr.CrunchLog.Print("Goodbye")
 658         cr.finalState = "Complete"
 659
 660         err = cr.CommitLogs()
 661         c.Check(err, IsNil)
 662
 663         c.Check(api.Calls, Equals, 2)
 664         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 665         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 666         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 667         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 668 }
 669
 670 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 671         api := &ArvTestClient{}
 672         kc := &KeepTestClient{}
 673         defer kc.Close()
 674         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 675         c.Assert(err, IsNil)
 676
 677         err = cr.UpdateContainerRunning("")
 678         c.Check(err, IsNil)
 679
 680         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 681 }
 682
 683 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 684         api := &ArvTestClient{}
 685         kc := &KeepTestClient{}
 686         defer kc.Close()
 687         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 688         c.Assert(err, IsNil)
 689
 690         cr.LogsPDH = new(string)
 691         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 692
 693         cr.ExitCode = new(int)
 694         *cr.ExitCode = 42
 695         cr.finalState = "Complete"
 696
 697         err = cr.UpdateContainerFinal()
 698         c.Check(err, IsNil)
 699
 700         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 701         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 702         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 703 }
 704
 705 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 706         api := &ArvTestClient{}
 707         kc := &KeepTestClient{}
 708         defer kc.Close()
 709         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 710         c.Assert(err, IsNil)
 711         cr.cCancelled = true
 712         cr.finalState = "Cancelled"
 713
 714         err = cr.UpdateContainerFinal()
 715         c.Check(err, IsNil)
 716
 717         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 718         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 719         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 720 }
 721
 722 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 723 // dress rehearsal of the Run() function, starting from a JSON container record.
 724 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, fn func() int) (*ArvTestClient, *ContainerRunner, string) {
 725         err := json.Unmarshal([]byte(record), &s.api.Container)
 726         c.Assert(err, IsNil)
 727         initialState := s.api.Container.State
 728
 729         var sm struct {
 730                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 731         }
 732         err = json.Unmarshal([]byte(record), &sm)
 733         c.Check(err, IsNil)
 734         secretMounts, err := json.Marshal(sm)
 735         c.Assert(err, IsNil)
 736         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 737
 738         s.executor.runFunc = fn
 739
 740         s.runner.statInterval = 100 * time.Millisecond
 741         s.runner.containerWatchdogInterval = time.Second
 742
 743         realTemp := c.MkDir()
 744         tempcount := 0
 745         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 746                 tempcount++
 747                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 748                 err := os.Mkdir(d, os.ModePerm)
 749                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 750                         // Test case must have pre-populated the tempdir
 751                         err = nil
 752                 }
 753                 return d, err
 754         }
 755         client, _ := apiStub()
 756         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 757                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, client, nil
 758         }
 759
 760         if extraMounts != nil && len(extraMounts) > 0 {
 761                 err := s.runner.SetupArvMountPoint("keep")
 762                 c.Check(err, IsNil)
 763
 764                 for _, m := range extraMounts {
 765                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 766                 }
 767         }
 768
 769         err = s.runner.Run()
 770         if s.api.CalledWith("container.state", "Complete") != nil {
 771                 c.Check(err, IsNil)
 772         }
 773         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 774                 c.Check(s.api.WasSetRunning, Equals, true)
 775                 var lastupdate arvadosclient.Dict
 776                 for _, content := range s.api.Content {
 777                         if content["container"] != nil {
 778                                 lastupdate = content["container"].(arvadosclient.Dict)
 779                         }
 780                 }
 781                 if lastupdate["log"] == nil {
 782                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 783                 }
 784         }
 785
 786         if err != nil {
 787                 for k, v := range s.api.Logs {
 788                         c.Log(k)
 789                         c.Log(v.String())
 790                 }
 791         }
 792
 793         return s.api, s.runner, realTemp
 794 }
 795
 796 func (s *TestSuite) TestFullRunHello(c *C) {
 797         s.runner.enableMemoryLimit = true
 798         s.runner.networkMode = "default"
 799         s.fullRunHelper(c, `{
 800     "command": ["echo", "hello world"],
 801     "container_image": "`+arvadostest.DockerImage112PDH+`",
 802     "cwd": ".",
 803     "environment": {"foo":"bar","baz":"waz"},
 804     "mounts": {"/tmp": {"kind": "tmp"} },
 805     "output_path": "/tmp",
 806     "priority": 1,
 807     "runtime_constraints": {"vcpus":1,"ram":1000000},
 808     "state": "Locked",
 809     "output_storage_classes": ["default"]
 810 }`, nil, func() int {
 811                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 812                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 813                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 814                 c.Check(s.executor.created.VCPUs, Equals, 1)
 815                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 816                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 817                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 818                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
 819                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 820                 return 0
 821         })
 822
 823         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 824         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 825         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 826         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 827         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 828 }
 829
 830 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 831         var ran bool
 832         s.fullRunHelper(c, `{
 833     "command": ["sleep", "3"],
 834     "container_image": "`+arvadostest.DockerImage112PDH+`",
 835     "cwd": ".",
 836     "environment": {},
 837     "mounts": {"/tmp": {"kind": "tmp"} },
 838     "output_path": "/tmp",
 839     "priority": 1,
 840     "runtime_constraints": {},
 841     "scheduling_parameters":{"max_run_time": 1},
 842     "state": "Running"
 843 }`, nil, func() int {
 844                 ran = true
 845                 return 2
 846         })
 847         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 848         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 849         c.Check(ran, Equals, false)
 850 }
 851
 852 func ec2MetadataServerStub(c *C, token *string, failureRate float64, stoptime *atomic.Value) *httptest.Server {
 853         failedOnce := false
 854         return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 855                 if !failedOnce || rand.Float64() < failureRate {
 856                         w.WriteHeader(http.StatusServiceUnavailable)
 857                         failedOnce = true
 858                         return
 859                 }
 860                 switch r.URL.Path {
 861                 case "/latest/api/token":
 862                         fmt.Fprintln(w, *token)
 863                 case "/latest/meta-data/spot/instance-action":
 864                         if r.Header.Get("X-aws-ec2-metadata-token") != *token {
 865                                 w.WriteHeader(http.StatusUnauthorized)
 866                         } else if t, _ := stoptime.Load().(time.Time); t.IsZero() {
 867                                 w.WriteHeader(http.StatusNotFound)
 868                         } else {
 869                                 fmt.Fprintf(w, `{"action":"stop","time":"%s"}`, t.Format(time.RFC3339))
 870                         }
 871                 default:
 872                         w.WriteHeader(http.StatusNotFound)
 873                 }
 874         }))
 875 }
 876
 877 func (s *TestSuite) TestSpotInterruptionNotice(c *C) {
 878         s.testSpotInterruptionNotice(c, 0.1)
 879 }
 880
 881 func (s *TestSuite) TestSpotInterruptionNoticeNotAvailable(c *C) {
 882         s.testSpotInterruptionNotice(c, 1)
 883 }
 884
 885 func (s *TestSuite) testSpotInterruptionNotice(c *C, failureRate float64) {
 886         var stoptime atomic.Value
 887         token := "fake-ec2-metadata-token"
 888         stub := ec2MetadataServerStub(c, &token, failureRate, &stoptime)
 889         defer stub.Close()
 890
 891         defer func(i time.Duration, u string) {
 892                 spotInterruptionCheckInterval = i
 893                 ec2MetadataBaseURL = u
 894         }(spotInterruptionCheckInterval, ec2MetadataBaseURL)
 895         spotInterruptionCheckInterval = time.Second / 8
 896         ec2MetadataBaseURL = stub.URL
 897
 898         go s.runner.checkSpotInterruptionNotices()
 899         s.fullRunHelper(c, `{
 900     "command": ["sleep", "3"],
 901     "container_image": "`+arvadostest.DockerImage112PDH+`",
 902     "cwd": ".",
 903     "environment": {},
 904     "mounts": {"/tmp": {"kind": "tmp"} },
 905     "output_path": "/tmp",
 906     "priority": 1,
 907     "runtime_constraints": {},
 908     "state": "Locked"
 909 }`, nil, func() int {
 910                 time.Sleep(time.Second)
 911                 stoptime.Store(time.Now().Add(time.Minute).UTC())
 912                 token = "different-fake-ec2-metadata-token"
 913                 time.Sleep(time.Second)
 914                 return 0
 915         })
 916         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Checking for spot interruptions every 125ms using instance metadata at http://.*`)
 917         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Error checking spot interruptions: 503 Service Unavailable.*`)
 918         if failureRate == 1 {
 919                 c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Giving up on checking spot interruptions after too many consecutive failures.*`)
 920         } else {
 921                 text := `Cloud provider scheduled instance stop at ` + stoptime.Load().(time.Time).Format(time.RFC3339)
 922                 c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*`+text+`.*`)
 923                 c.Check(s.api.CalledWith("container.runtime_status.warning", "preemption notice"), NotNil)
 924                 c.Check(s.api.CalledWith("container.runtime_status.warningDetail", text), NotNil)
 925                 c.Check(s.api.CalledWith("container.runtime_status.preemptionNotice", text), NotNil)
 926         }
 927 }
 928
 929 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 930         s.fullRunHelper(c, `{
 931     "command": ["sleep", "3"],
 932     "container_image": "`+arvadostest.DockerImage112PDH+`",
 933     "cwd": ".",
 934     "environment": {},
 935     "mounts": {"/tmp": {"kind": "tmp"} },
 936     "output_path": "/tmp",
 937     "priority": 1,
 938     "runtime_constraints": {},
 939     "scheduling_parameters":{"max_run_time": 1},
 940     "state": "Locked"
 941 }`, nil, func() int {
 942                 time.Sleep(3 * time.Second)
 943                 return 0
 944         })
 945
 946         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 947         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 948 }
 949
 950 func (s *TestSuite) TestContainerWaitFails(c *C) {
 951         s.fullRunHelper(c, `{
 952     "command": ["sleep", "3"],
 953     "container_image": "`+arvadostest.DockerImage112PDH+`",
 954     "cwd": ".",
 955     "mounts": {"/tmp": {"kind": "tmp"} },
 956     "output_path": "/tmp",
 957     "priority": 1,
 958     "state": "Locked"
 959 }`, nil, func() int {
 960                 s.executor.waitErr = errors.New("Container is not running")
 961                 return 0
 962         })
 963
 964         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 965         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 966 }
 967
 968 func (s *TestSuite) TestCrunchstat(c *C) {
 969         s.fullRunHelper(c, `{
 970                 "command": ["sleep", "1"],
 971                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 972                 "cwd": ".",
 973                 "environment": {},
 974                 "mounts": {"/tmp": {"kind": "tmp"} },
 975                 "output_path": "/tmp",
 976                 "priority": 1,
 977                 "runtime_constraints": {},
 978                 "state": "Locked"
 979         }`, nil, func() int {
 980                 time.Sleep(time.Second)
 981                 return 0
 982         })
 983
 984         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 985         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 986
 987         // We didn't actually start a container, so crunchstat didn't
 988         // find accounting files and therefore didn't log any stats.
 989         // It should have logged a "can't find accounting files"
 990         // message after one poll interval, though, so we can confirm
 991         // it's alive:
 992         c.Assert(s.api.Logs["crunchstat"], NotNil)
 993         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 994
 995         // The "files never appeared" log assures us that we called
 996         // (*crunchstat.Reporter)Stop(), and that we set it up with
 997         // the correct container ID "abcde":
 998         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 999 }
1000
1001 func (s *TestSuite) TestNodeInfoLog(c *C) {
1002         os.Setenv("SLURMD_NODENAME", "compute2")
1003         s.fullRunHelper(c, `{
1004                 "command": ["sleep", "1"],
1005                 "container_image": "`+arvadostest.DockerImage112PDH+`",
1006                 "cwd": ".",
1007                 "environment": {},
1008                 "mounts": {"/tmp": {"kind": "tmp"} },
1009                 "output_path": "/tmp",
1010                 "priority": 1,
1011                 "runtime_constraints": {},
1012                 "state": "Locked"
1013         }`, nil, func() int {
1014                 time.Sleep(time.Second)
1015                 return 0
1016         })
1017
1018         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1019         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1020
1021         c.Assert(s.api.Logs["node"], NotNil)
1022         json := s.api.Logs["node"].String()
1023         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
1024         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
1025         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
1026
1027         c.Assert(s.api.Logs["node-info"], NotNil)
1028         json = s.api.Logs["node-info"].String()
1029         c.Check(json, Matches, `(?ms).*Host Information.*`)
1030         c.Check(json, Matches, `(?ms).*CPU Information.*`)
1031         c.Check(json, Matches, `(?ms).*Memory Information.*`)
1032         c.Check(json, Matches, `(?ms).*Disk Space.*`)
1033         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
1034 }
1035
1036 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
1037         s.fullRunHelper(c, `{
1038                 "command": ["sleep", "1"],
1039                 "container_image": "`+arvadostest.DockerImage112PDH+`",
1040                 "cwd": ".",
1041                 "environment": {},
1042                 "mounts": {"/tmp": {"kind": "tmp"} },
1043                 "output_path": "/tmp",
1044                 "priority": 1,
1045                 "runtime_constraints": {},
1046                 "state": "Locked"
1047         }`, nil, func() int {
1048                 return 0
1049         })
1050
1051         c.Assert(s.api.Logs["crunch-run"], NotNil)
1052         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
1053         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`)
1054         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container: zzzzz-zzzzz-zzzzzzzzzzzzzzz.*`)
1055         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Using container runtime: stub.*`)
1056 }
1057
1058 func (s *TestSuite) testLogRSSThresholds(c *C, ram int, expected []int, notExpected int) {
1059         s.runner.cgroupRoot = "testdata/fakestat"
1060         s.fullRunHelper(c, `{
1061                 "command": ["true"],
1062                 "container_image": "`+arvadostest.DockerImage112PDH+`",
1063                 "cwd": ".",
1064                 "environment": {},
1065                 "mounts": {"/tmp": {"kind": "tmp"} },
1066                 "output_path": "/tmp",
1067                 "priority": 1,
1068                 "runtime_constraints": {"ram": `+strconv.Itoa(ram)+`},
1069                 "state": "Locked"
1070         }`, nil, func() int { return 0 })
1071         logs := s.api.Logs["crunch-run"].String()
1072         pattern := logLineStart + `Container using over %d%% of memory \(rss 734003200/%d bytes\)`
1073         var threshold int
1074         for _, threshold = range expected {
1075                 c.Check(logs, Matches, fmt.Sprintf(pattern, threshold, ram))
1076         }
1077         if notExpected > threshold {
1078                 c.Check(logs, Not(Matches), fmt.Sprintf(pattern, notExpected, ram))
1079         }
1080 }
1081
1082 func (s *TestSuite) TestLogNoRSSThresholds(c *C) {
1083         s.testLogRSSThresholds(c, 7340032000, []int{}, 90)
1084 }
1085
1086 func (s *TestSuite) TestLogSomeRSSThresholds(c *C) {
1087         onePercentRSS := 7340032
1088         s.testLogRSSThresholds(c, 102*onePercentRSS, []int{90, 95}, 99)
1089 }
1090
1091 func (s *TestSuite) TestLogAllRSSThresholds(c *C) {
1092         s.testLogRSSThresholds(c, 734003299, []int{90, 95, 99}, 0)
1093 }
1094
1095 func (s *TestSuite) TestLogMaximaAfterRun(c *C) {
1096         s.runner.cgroupRoot = "testdata/fakestat"
1097         s.runner.parentTemp = c.MkDir()
1098         s.fullRunHelper(c, `{
1099         "command": ["true"],
1100         "container_image": "`+arvadostest.DockerImage112PDH+`",
1101         "cwd": ".",
1102         "environment": {},
1103         "mounts": {"/tmp": {"kind": "tmp"} },
1104         "output_path": "/tmp",
1105         "priority": 1,
1106         "runtime_constraints": {"ram": 7340032000},
1107         "state": "Locked"
1108     }`, nil, func() int { return 0 })
1109         logs := s.api.Logs["crunch-run"].String()
1110         for _, expected := range []string{
1111                 `Maximum disk usage was \d+%, \d+/\d+ bytes`,
1112                 `Maximum container memory cache usage was 73400320 bytes`,
1113                 `Maximum container memory swap usage was 320 bytes`,
1114                 `Maximum container memory pgmajfault usage was 20 faults`,
1115                 `Maximum container memory rss usage was 10%, 734003200/7340032000 bytes`,
1116                 `Maximum crunch-run memory rss usage was \d+ bytes`,
1117         } {
1118                 c.Check(logs, Matches, logLineStart+expected)
1119         }
1120 }
1121
1122 func (s *TestSuite) TestCommitNodeInfoBeforeStart(c *C) {
1123         var collection_create, container_update arvadosclient.Dict
1124         s.fullRunHelper(c, `{
1125                 "command": ["true"],
1126                 "container_image": "`+arvadostest.DockerImage112PDH+`",
1127                 "cwd": ".",
1128                 "environment": {},
1129                 "mounts": {"/tmp": {"kind": "tmp"} },
1130                 "output_path": "/tmp",
1131                 "priority": 1,
1132                 "runtime_constraints": {},
1133                 "state": "Locked",
1134                 "uuid": "zzzzz-dz642-202301121543210"
1135         }`, nil, func() int {
1136                 collection_create = s.api.CalledWith("ensure_unique_name", true)
1137                 container_update = s.api.CalledWith("container.state", "Running")
1138                 return 0
1139         })
1140
1141         c.Assert(collection_create, NotNil)
1142         log_collection := collection_create["collection"].(arvadosclient.Dict)
1143         c.Check(log_collection["name"], Equals, "logs for zzzzz-dz642-202301121543210")
1144         manifest_text := log_collection["manifest_text"].(string)
1145         // We check that the file size is at least two digits as an easy way to
1146         // check the file isn't empty.
1147         c.Check(manifest_text, Matches, `\. .+ \d+:\d{2,}:node-info\.txt( .+)?\n`)
1148         c.Check(manifest_text, Matches, `\. .+ \d+:\d{2,}:node\.json( .+)?\n`)
1149
1150         c.Assert(container_update, NotNil)
1151         // As of Arvados 2.5.0, the container update must specify its log in PDH
1152         // format for the API server to propagate it to container requests, which
1153         // is what we care about for this test.
1154         expect_pdh := fmt.Sprintf("%x+%d", md5.Sum([]byte(manifest_text)), len(manifest_text))
1155         c.Check(container_update["container"].(arvadosclient.Dict)["log"], Equals, expect_pdh)
1156 }
1157
1158 func (s *TestSuite) TestContainerRecordLog(c *C) {
1159         s.fullRunHelper(c, `{
1160                 "command": ["sleep", "1"],
1161                 "container_image": "`+arvadostest.DockerImage112PDH+`",
1162                 "cwd": ".",
1163                 "environment": {},
1164                 "mounts": {"/tmp": {"kind": "tmp"} },
1165                 "output_path": "/tmp",
1166                 "priority": 1,
1167                 "runtime_constraints": {},
1168                 "state": "Locked"
1169         }`, nil,
1170                 func() int {
1171                         time.Sleep(time.Second)
1172                         return 0
1173                 })
1174
1175         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1176         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1177
1178         c.Assert(s.api.Logs["container"], NotNil)
1179         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
1180 }
1181
1182 func (s *TestSuite) TestFullRunStderr(c *C) {
1183         s.fullRunHelper(c, `{
1184     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
1185     "container_image": "`+arvadostest.DockerImage112PDH+`",
1186     "cwd": ".",
1187     "environment": {},
1188     "mounts": {"/tmp": {"kind": "tmp"} },
1189     "output_path": "/tmp",
1190     "priority": 1,
1191     "runtime_constraints": {},
1192     "state": "Locked"
1193 }`, nil, func() int {
1194                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1195                 fmt.Fprintln(s.executor.created.Stderr, "world")
1196                 return 1
1197         })
1198
1199         final := s.api.CalledWith("container.state", "Complete")
1200         c.Assert(final, NotNil)
1201         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1202         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1203
1204         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
1205         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
1206 }
1207
1208 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
1209         s.fullRunHelper(c, `{
1210     "command": ["pwd"],
1211     "container_image": "`+arvadostest.DockerImage112PDH+`",
1212     "cwd": ".",
1213     "environment": {},
1214     "mounts": {"/tmp": {"kind": "tmp"} },
1215     "output_path": "/tmp",
1216     "priority": 1,
1217     "runtime_constraints": {},
1218     "state": "Locked"
1219 }`, nil, func() int {
1220                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
1221                 return 0
1222         })
1223
1224         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1225         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1226         c.Log(s.api.Logs["stdout"])
1227         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
1228 }
1229
1230 func (s *TestSuite) TestFullRunSetCwd(c *C) {
1231         s.fullRunHelper(c, `{
1232     "command": ["pwd"],
1233     "container_image": "`+arvadostest.DockerImage112PDH+`",
1234     "cwd": "/bin",
1235     "environment": {},
1236     "mounts": {"/tmp": {"kind": "tmp"} },
1237     "output_path": "/tmp",
1238     "priority": 1,
1239     "runtime_constraints": {},
1240     "state": "Locked"
1241 }`, nil, func() int {
1242                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1243                 return 0
1244         })
1245
1246         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1247         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1248         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1249 }
1250
1251 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
1252         s.fullRunHelper(c, `{
1253     "command": ["pwd"],
1254     "container_image": "`+arvadostest.DockerImage112PDH+`",
1255     "cwd": "/bin",
1256     "environment": {},
1257     "mounts": {"/tmp": {"kind": "tmp"} },
1258     "output_path": "/tmp",
1259     "priority": 1,
1260     "runtime_constraints": {},
1261     "state": "Locked",
1262     "output_storage_classes": ["foo", "bar"]
1263 }`, nil, func() int {
1264                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1265                 return 0
1266         })
1267
1268         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1269         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1270         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1271         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1272         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1273 }
1274
1275 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
1276         s.fullRunHelper(c, `{
1277     "command": ["pwd"],
1278     "container_image": "`+arvadostest.DockerImage112PDH+`",
1279     "cwd": "/bin",
1280     "environment": {},
1281     "mounts": {"/tmp": {"kind": "tmp"} },
1282     "output_path": "/tmp",
1283     "priority": 1,
1284     "runtime_constraints": {"cuda": {"device_count": 2}},
1285     "state": "Locked",
1286     "output_storage_classes": ["foo", "bar"]
1287 }`, nil, func() int {
1288                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1289                 return 0
1290         })
1291         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1292 }
1293
1294 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1295         s.fullRunHelper(c, `{
1296     "command": ["pwd"],
1297     "container_image": "`+arvadostest.DockerImage112PDH+`",
1298     "cwd": "/bin",
1299     "environment": {},
1300     "mounts": {"/tmp": {"kind": "tmp"} },
1301     "output_path": "/tmp",
1302     "priority": 1,
1303     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1304     "state": "Locked",
1305     "output_storage_classes": ["foo", "bar"]
1306 }`, nil, func() int {
1307                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1308                 return 0
1309         })
1310         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1311 }
1312
1313 func (s *TestSuite) TestStopOnSignal(c *C) {
1314         s.executor.runFunc = func() int {
1315                 s.executor.created.Stdout.Write([]byte("foo\n"))
1316                 s.runner.SigChan <- syscall.SIGINT
1317                 time.Sleep(10 * time.Second)
1318                 return 0
1319         }
1320         s.testStopContainer(c)
1321 }
1322
1323 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1324         s.executor.runFunc = func() int {
1325                 s.executor.created.Stdout.Write([]byte("foo\n"))
1326                 s.runner.ArvMountExit <- nil
1327                 close(s.runner.ArvMountExit)
1328                 time.Sleep(10 * time.Second)
1329                 return 0
1330         }
1331         s.runner.ArvMountExit = make(chan error)
1332         s.testStopContainer(c)
1333 }
1334
1335 func (s *TestSuite) testStopContainer(c *C) {
1336         record := `{
1337     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1338     "container_image": "` + arvadostest.DockerImage112PDH + `",
1339     "cwd": ".",
1340     "environment": {},
1341     "mounts": {"/tmp": {"kind": "tmp"} },
1342     "output_path": "/tmp",
1343     "priority": 1,
1344     "runtime_constraints": {},
1345     "state": "Locked"
1346 }`
1347
1348         err := json.Unmarshal([]byte(record), &s.api.Container)
1349         c.Assert(err, IsNil)
1350
1351         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1352         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1353                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1354         }
1355
1356         done := make(chan error)
1357         go func() {
1358                 done <- s.runner.Run()
1359         }()
1360         select {
1361         case <-time.After(20 * time.Second):
1362                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1363                 c.Fatal("timed out")
1364         case err = <-done:
1365                 c.Check(err, IsNil)
1366         }
1367         for k, v := range s.api.Logs {
1368                 c.Log(k)
1369                 c.Log(v.String(), "\n")
1370         }
1371
1372         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1373         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1374         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1375 }
1376
1377 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1378         s.fullRunHelper(c, `{
1379     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1380     "container_image": "`+arvadostest.DockerImage112PDH+`",
1381     "cwd": "/bin",
1382     "environment": {"FROBIZ": "bilbo"},
1383     "mounts": {"/tmp": {"kind": "tmp"} },
1384     "output_path": "/tmp",
1385     "priority": 1,
1386     "runtime_constraints": {},
1387     "state": "Locked"
1388 }`, nil, func() int {
1389                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1390                 return 0
1391         })
1392
1393         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1394         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1395         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1396 }
1397
1398 type ArvMountCmdLine struct {
1399         Cmd   []string
1400         token string
1401 }
1402
1403 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1404         am.Cmd = c
1405         am.token = token
1406         return nil, nil
1407 }
1408
1409 func stubCert(temp string) string {
1410         path := temp + "/ca-certificates.crt"
1411         crt, _ := os.Create(path)
1412         crt.Close()
1413         arvadosclient.CertFiles = []string{path}
1414         return path
1415 }
1416
1417 func (s *TestSuite) TestSetupMounts(c *C) {
1418         cr := s.runner
1419         am := &ArvMountCmdLine{}
1420         cr.RunArvMount = am.ArvMountTest
1421         cr.containerClient, _ = apiStub()
1422         cr.ContainerArvClient = &ArvTestClient{}
1423         cr.ContainerKeepClient = &KeepTestClient{}
1424         cr.Container.OutputStorageClasses = []string{"default"}
1425
1426         realTemp := c.MkDir()
1427         certTemp := c.MkDir()
1428         stubCertPath := stubCert(certTemp)
1429         cr.parentTemp = realTemp
1430
1431         i := 0
1432         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1433                 i++
1434                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1435                 err := os.Mkdir(d, os.ModePerm)
1436                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1437                         // Test case must have pre-populated the tempdir
1438                         err = nil
1439                 }
1440                 return d, err
1441         }
1442
1443         checkEmpty := func() {
1444                 // Should be deleted.
1445                 _, err := os.Stat(realTemp)
1446                 c.Assert(os.IsNotExist(err), Equals, true)
1447
1448                 // Now recreate it for the next test.
1449                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1450         }
1451
1452         {
1453                 i = 0
1454                 cr.ArvMountPoint = ""
1455                 cr.Container.Mounts = make(map[string]arvados.Mount)
1456                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1457                 cr.Container.OutputPath = "/tmp"
1458                 cr.statInterval = 5 * time.Second
1459                 bindmounts, err := cr.SetupMounts()
1460                 c.Check(err, IsNil)
1461                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1462                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1463                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1464                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1465                 os.RemoveAll(cr.ArvMountPoint)
1466                 cr.CleanupDirs()
1467                 checkEmpty()
1468         }
1469
1470         {
1471                 i = 0
1472                 cr.ArvMountPoint = ""
1473                 cr.Container.Mounts = make(map[string]arvados.Mount)
1474                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1475                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1476                 cr.Container.OutputPath = "/out"
1477                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1478
1479                 bindmounts, err := cr.SetupMounts()
1480                 c.Check(err, IsNil)
1481                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1482                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1483                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1484                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1485                 os.RemoveAll(cr.ArvMountPoint)
1486                 cr.CleanupDirs()
1487                 checkEmpty()
1488         }
1489
1490         {
1491                 i = 0
1492                 cr.ArvMountPoint = ""
1493                 cr.Container.Mounts = make(map[string]arvados.Mount)
1494                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1495                 cr.Container.OutputPath = "/tmp"
1496                 cr.Container.RuntimeConstraints.API = true
1497                 cr.Container.OutputStorageClasses = []string{"default"}
1498
1499                 bindmounts, err := cr.SetupMounts()
1500                 c.Check(err, IsNil)
1501                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1502                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1503                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1504                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1505                 os.RemoveAll(cr.ArvMountPoint)
1506                 cr.CleanupDirs()
1507                 checkEmpty()
1508
1509                 cr.Container.RuntimeConstraints.API = false
1510         }
1511
1512         {
1513                 i = 0
1514                 cr.ArvMountPoint = ""
1515                 cr.Container.Mounts = map[string]arvados.Mount{
1516                         "/keeptmp": {Kind: "collection", Writable: true},
1517                 }
1518                 cr.Container.OutputPath = "/keeptmp"
1519
1520                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1521
1522                 bindmounts, err := cr.SetupMounts()
1523                 c.Check(err, IsNil)
1524                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1525                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1526                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1527                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1528                 os.RemoveAll(cr.ArvMountPoint)
1529                 cr.CleanupDirs()
1530                 checkEmpty()
1531         }
1532
1533         {
1534                 i = 0
1535                 cr.ArvMountPoint = ""
1536                 cr.Container.Mounts = map[string]arvados.Mount{
1537                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1538                         "/keepout": {Kind: "collection", Writable: true},
1539                 }
1540                 cr.Container.OutputPath = "/keepout"
1541
1542                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1543                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1544
1545                 bindmounts, err := cr.SetupMounts()
1546                 c.Check(err, IsNil)
1547                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1548                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1549                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1550                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1551                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1552                         "/keepout": {realTemp + "/keep1/tmp0", false},
1553                 })
1554                 os.RemoveAll(cr.ArvMountPoint)
1555                 cr.CleanupDirs()
1556                 checkEmpty()
1557         }
1558
1559         {
1560                 i = 0
1561                 cr.ArvMountPoint = ""
1562                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1563                 cr.Container.Mounts = map[string]arvados.Mount{
1564                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1565                         "/keepout": {Kind: "collection", Writable: true},
1566                 }
1567                 cr.Container.OutputPath = "/keepout"
1568
1569                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1570                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1571
1572                 bindmounts, err := cr.SetupMounts()
1573                 c.Check(err, IsNil)
1574                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1575                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
1576                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1577                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1578                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1579                         "/keepout": {realTemp + "/keep1/tmp0", false},
1580                 })
1581                 os.RemoveAll(cr.ArvMountPoint)
1582                 cr.CleanupDirs()
1583                 checkEmpty()
1584         }
1585
1586         for _, test := range []struct {
1587                 in  interface{}
1588                 out string
1589         }{
1590                 {in: "foo", out: `"foo"`},
1591                 {in: nil, out: `null`},
1592                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1593         } {
1594                 i = 0
1595                 cr.ArvMountPoint = ""
1596                 cr.Container.Mounts = map[string]arvados.Mount{
1597                         "/mnt/test.json": {Kind: "json", Content: test.in},
1598                 }
1599                 bindmounts, err := cr.SetupMounts()
1600                 c.Check(err, IsNil)
1601                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1602                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1603                 })
1604                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1605                 c.Check(err, IsNil)
1606                 c.Check(content, DeepEquals, []byte(test.out))
1607                 os.RemoveAll(cr.ArvMountPoint)
1608                 cr.CleanupDirs()
1609                 checkEmpty()
1610         }
1611
1612         for _, test := range []struct {
1613                 in  interface{}
1614                 out string
1615         }{
1616                 {in: "foo", out: `foo`},
1617                 {in: nil, out: "error"},
1618                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1619         } {
1620                 i = 0
1621                 cr.ArvMountPoint = ""
1622                 cr.Container.Mounts = map[string]arvados.Mount{
1623                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1624                 }
1625                 bindmounts, err := cr.SetupMounts()
1626                 if test.out == "error" {
1627                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1628                 } else {
1629                         c.Check(err, IsNil)
1630                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1631                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1632                         })
1633                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1634                         c.Check(err, IsNil)
1635                         c.Check(content, DeepEquals, []byte(test.out))
1636                 }
1637                 os.RemoveAll(cr.ArvMountPoint)
1638                 cr.CleanupDirs()
1639                 checkEmpty()
1640         }
1641
1642         // Read-only mount points are allowed underneath output_dir mount point
1643         {
1644                 i = 0
1645                 cr.ArvMountPoint = ""
1646                 cr.Container.Mounts = make(map[string]arvados.Mount)
1647                 cr.Container.Mounts = map[string]arvados.Mount{
1648                         "/tmp":     {Kind: "tmp"},
1649                         "/tmp/foo": {Kind: "collection"},
1650                 }
1651                 cr.Container.OutputPath = "/tmp"
1652
1653                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1654
1655                 bindmounts, err := cr.SetupMounts()
1656                 c.Check(err, IsNil)
1657                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1658                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
1659                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1660                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1661                         "/tmp":     {realTemp + "/tmp2", false},
1662                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1663                 })
1664                 os.RemoveAll(cr.ArvMountPoint)
1665                 cr.CleanupDirs()
1666                 checkEmpty()
1667         }
1668
1669         // Writable mount points copied to output_dir mount point
1670         {
1671                 i = 0
1672                 cr.ArvMountPoint = ""
1673                 cr.Container.Mounts = make(map[string]arvados.Mount)
1674                 cr.Container.Mounts = map[string]arvados.Mount{
1675                         "/tmp": {Kind: "tmp"},
1676                         "/tmp/foo": {Kind: "collection",
1677                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1678                                 Writable:         true},
1679                         "/tmp/bar": {Kind: "collection",
1680                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1681                                 Path:             "baz",
1682                                 Writable:         true},
1683                 }
1684                 cr.Container.OutputPath = "/tmp"
1685
1686                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1687                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1688
1689                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1690                 rf.Write([]byte("bar"))
1691                 rf.Close()
1692
1693                 _, err := cr.SetupMounts()
1694                 c.Check(err, IsNil)
1695                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1696                 c.Check(err, IsNil)
1697                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1698                 c.Check(err, IsNil)
1699                 os.RemoveAll(cr.ArvMountPoint)
1700                 cr.CleanupDirs()
1701                 checkEmpty()
1702         }
1703
1704         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1705         {
1706                 i = 0
1707                 cr.ArvMountPoint = ""
1708                 cr.Container.Mounts = make(map[string]arvados.Mount)
1709                 cr.Container.Mounts = map[string]arvados.Mount{
1710                         "/tmp":     {Kind: "tmp"},
1711                         "/tmp/foo": {Kind: "tmp"},
1712                 }
1713                 cr.Container.OutputPath = "/tmp"
1714
1715                 _, err := cr.SetupMounts()
1716                 c.Check(err, NotNil)
1717                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1718                 os.RemoveAll(cr.ArvMountPoint)
1719                 cr.CleanupDirs()
1720                 checkEmpty()
1721         }
1722
1723         // Only mount point of kind 'collection' is allowed for stdin
1724         {
1725                 i = 0
1726                 cr.ArvMountPoint = ""
1727                 cr.Container.Mounts = make(map[string]arvados.Mount)
1728                 cr.Container.Mounts = map[string]arvados.Mount{
1729                         "stdin": {Kind: "tmp"},
1730                 }
1731
1732                 _, err := cr.SetupMounts()
1733                 c.Check(err, NotNil)
1734                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1735                 os.RemoveAll(cr.ArvMountPoint)
1736                 cr.CleanupDirs()
1737                 checkEmpty()
1738         }
1739
1740         // git_tree mounts
1741         {
1742                 i = 0
1743                 cr.ArvMountPoint = ""
1744                 git_client.InstallProtocol("https", git_http.NewClient(arvados.InsecureHTTPClient))
1745                 cr.token = arvadostest.ActiveToken
1746                 cr.Container.Mounts = make(map[string]arvados.Mount)
1747                 cr.Container.Mounts = map[string]arvados.Mount{
1748                         "/tip": {
1749                                 Kind:   "git_tree",
1750                                 UUID:   arvadostest.Repository2UUID,
1751                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1752                                 Path:   "/",
1753                         },
1754                         "/non-tip": {
1755                                 Kind:   "git_tree",
1756                                 UUID:   arvadostest.Repository2UUID,
1757                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1758                                 Path:   "/",
1759                         },
1760                 }
1761                 cr.Container.OutputPath = "/tmp"
1762
1763                 bindmounts, err := cr.SetupMounts()
1764                 c.Check(err, IsNil)
1765
1766                 for path, mount := range bindmounts {
1767                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1768                 }
1769
1770                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1771                 c.Check(err, IsNil)
1772                 c.Check(string(data), Equals, "\000\001\002\003")
1773                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1774                 c.Check(err, FitsTypeOf, &os.PathError{})
1775                 c.Check(os.IsNotExist(err), Equals, true)
1776
1777                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1778                 c.Check(err, IsNil)
1779                 c.Check(string(data), Equals, "\000\001\002\003")
1780                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1781                 c.Check(err, IsNil)
1782                 c.Check(string(data), Equals, "testfile\n")
1783
1784                 cr.CleanupDirs()
1785                 checkEmpty()
1786         }
1787 }
1788
1789 func (s *TestSuite) TestStdout(c *C) {
1790         helperRecord := `{
1791                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1792                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1793                 "cwd": "/bin",
1794                 "environment": {"FROBIZ": "bilbo"},
1795                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1796                 "output_path": "/tmp",
1797                 "priority": 1,
1798                 "runtime_constraints": {},
1799                 "state": "Locked"
1800         }`
1801
1802         s.fullRunHelper(c, helperRecord, nil, func() int {
1803                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1804                 return 0
1805         })
1806
1807         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1808         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1809         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1810 }
1811
1812 // Used by the TestStdoutWithWrongPath*()
1813 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func() int) (*ArvTestClient, *ContainerRunner, error) {
1814         err := json.Unmarshal([]byte(record), &s.api.Container)
1815         c.Assert(err, IsNil)
1816         s.executor.runFunc = fn
1817         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1818         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1819                 return s.api, &KeepTestClient{}, nil, nil
1820         }
1821         return s.api, s.runner, s.runner.Run()
1822 }
1823
1824 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1825         _, _, err := s.stdoutErrorRunHelper(c, `{
1826     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1827     "output_path": "/tmp",
1828     "state": "Locked"
1829 }`, func() int { return 0 })
1830         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1831 }
1832
1833 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1834         _, _, err := s.stdoutErrorRunHelper(c, `{
1835     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1836     "output_path": "/tmp",
1837     "state": "Locked"
1838 }`, func() int { return 0 })
1839         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1840 }
1841
1842 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1843         _, _, err := s.stdoutErrorRunHelper(c, `{
1844     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1845     "output_path": "/tmp",
1846     "state": "Locked"
1847 }`, func() int { return 0 })
1848         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1849 }
1850
1851 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1852         s.fullRunHelper(c, `{
1853     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1854     "container_image": "`+arvadostest.DockerImage112PDH+`",
1855     "cwd": "/bin",
1856     "environment": {},
1857     "mounts": {"/tmp": {"kind": "tmp"} },
1858     "output_path": "/tmp",
1859     "priority": 1,
1860     "runtime_constraints": {"API": true},
1861     "state": "Locked"
1862 }`, nil, func() int {
1863                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1864                 return 3
1865         })
1866         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1867         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1868         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1869 }
1870
1871 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1872         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1873         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1874         s.fullRunHelper(c, `{
1875     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1876     "container_image": "`+arvadostest.DockerImage112PDH+`",
1877     "cwd": "/bin",
1878     "environment": {},
1879     "mounts": {"/tmp": {"kind": "tmp"} },
1880     "output_path": "/tmp",
1881     "priority": 1,
1882     "runtime_constraints": {"API": true},
1883     "state": "Locked"
1884 }`, nil, func() int {
1885                 s.api.Container.Output = arvadostest.DockerImage112PDH
1886                 return 0
1887         })
1888
1889         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1890         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1891         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1892 }
1893
1894 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1895         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1896                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1897                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1898                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1899         }
1900         s.executor.runFunc = func() int {
1901                 time.Sleep(time.Second)
1902                 return 137
1903         }
1904         record := `{
1905     "command": ["sleep", "1"],
1906     "container_image": "` + arvadostest.DockerImage112PDH + `",
1907     "cwd": "/bin",
1908     "environment": {},
1909     "mounts": {"/tmp": {"kind": "tmp"} },
1910     "output_path": "/tmp",
1911     "priority": 1,
1912     "runtime_constraints": {"API": true},
1913     "state": "Locked"
1914 }`
1915         err := json.Unmarshal([]byte(record), &s.api.Container)
1916         c.Assert(err, IsNil)
1917         err = s.runner.Run()
1918         c.Assert(err, IsNil)
1919         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1920         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1921         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1922         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1923         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1924 }
1925
1926 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1927         helperRecord := `{
1928                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1929                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1930                 "cwd": "/bin",
1931                 "environment": {"FROBIZ": "bilbo"},
1932                 "mounts": {
1933         "/tmp": {"kind": "tmp"},
1934         "/tmp/foo": {"kind": "collection",
1935                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1936                      "exclude_from_output": true
1937         },
1938         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1939     },
1940                 "output_path": "/tmp",
1941                 "priority": 1,
1942                 "runtime_constraints": {},
1943                 "state": "Locked"
1944         }`
1945
1946         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1947
1948         s.fullRunHelper(c, helperRecord, extraMounts, func() int {
1949                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1950                 return 0
1951         })
1952
1953         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1954         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1955         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1956 }
1957
1958 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1959         helperRecord := `{
1960                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1961                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1962                 "cwd": "/bin",
1963                 "environment": {"FROBIZ": "bilbo"},
1964                 "mounts": {
1965         "/tmp": {"kind": "tmp"},
1966         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1967         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1968         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1969         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1970         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1971     },
1972                 "output_path": "/tmp",
1973                 "priority": 1,
1974                 "runtime_constraints": {},
1975                 "state": "Locked",
1976                 "uuid": "zzzzz-dz642-202301130848001"
1977         }`
1978
1979         extraMounts := []string{
1980                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1981                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1982                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1983         }
1984
1985         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, func() int {
1986                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1987                 return 0
1988         })
1989
1990         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1991                 "/tmp":                   {realtemp + "/tmp1", false},
1992                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1993                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1994                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1995                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1996         })
1997
1998         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1999         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
2000         output_count := uint(0)
2001         for _, v := range s.runner.ContainerArvClient.(*ArvTestClient).Content {
2002                 if v["collection"] == nil {
2003                         continue
2004                 }
2005                 collection := v["collection"].(arvadosclient.Dict)
2006                 if collection["name"].(string) != "output for zzzzz-dz642-202301130848001" {
2007                         continue
2008                 }
2009                 c.Check(v["ensure_unique_name"], Equals, true)
2010                 c.Check(collection["manifest_text"].(string), Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
2011 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
2012 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
2013 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
2014 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
2015 `)
2016                 output_count++
2017         }
2018         c.Check(output_count, Not(Equals), uint(0))
2019 }
2020
2021 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
2022         helperRecord := `{
2023                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
2024                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2025                 "cwd": "/bin",
2026                 "environment": {"FROBIZ": "bilbo"},
2027                 "mounts": {
2028         "/tmp": {"kind": "tmp"},
2029         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
2030         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
2031     },
2032                 "output_path": "/tmp",
2033                 "priority": 1,
2034                 "runtime_constraints": {},
2035                 "state": "Locked",
2036                 "uuid": "zzzzz-dz642-202301130848002"
2037         }`
2038
2039         extraMounts := []string{
2040                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
2041         }
2042
2043         s.fullRunHelper(c, helperRecord, extraMounts, func() int {
2044                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
2045                 return 0
2046         })
2047
2048         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2049         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2050         output_count := uint(0)
2051         for _, v := range s.runner.ContainerArvClient.(*ArvTestClient).Content {
2052                 if v["collection"] == nil {
2053                         continue
2054                 }
2055                 collection := v["collection"].(arvadosclient.Dict)
2056                 if collection["name"].(string) != "output for zzzzz-dz642-202301130848002" {
2057                         continue
2058                 }
2059                 c.Check(collection["manifest_text"].(string), Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
2060 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
2061 `)
2062                 output_count++
2063         }
2064         c.Check(output_count, Not(Equals), uint(0))
2065 }
2066
2067 func (s *TestSuite) TestOutputError(c *C) {
2068         helperRecord := `{
2069                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
2070                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2071                 "cwd": "/bin",
2072                 "environment": {"FROBIZ": "bilbo"},
2073                 "mounts": {
2074                         "/tmp": {"kind": "tmp"}
2075                 },
2076                 "output_path": "/tmp",
2077                 "priority": 1,
2078                 "runtime_constraints": {},
2079                 "state": "Locked"
2080         }`
2081         s.fullRunHelper(c, helperRecord, nil, func() int {
2082                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
2083                 return 0
2084         })
2085
2086         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
2087 }
2088
2089 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
2090         helperRecord := `{
2091                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
2092                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2093                 "cwd": "/bin",
2094                 "environment": {"FROBIZ": "bilbo"},
2095                 "mounts": {
2096         "/tmp": {"kind": "tmp"},
2097         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
2098         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
2099     },
2100                 "output_path": "/tmp",
2101                 "priority": 1,
2102                 "runtime_constraints": {},
2103                 "state": "Locked"
2104         }`
2105
2106         extraMounts := []string{
2107                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
2108         }
2109
2110         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, func() int {
2111                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
2112                 return 0
2113         })
2114
2115         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
2116         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
2117         for _, v := range api.Content {
2118                 if v["collection"] != nil {
2119                         collection := v["collection"].(arvadosclient.Dict)
2120                         if strings.Index(collection["name"].(string), "output") == 0 {
2121                                 manifest := collection["manifest_text"].(string)
2122                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
2123 `)
2124                         }
2125                 }
2126         }
2127 }
2128
2129 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
2130         helperRecord := `{
2131                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
2132                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2133                 "cwd": "/bin",
2134                 "environment": {"FROBIZ": "bilbo"},
2135                 "mounts": {
2136         "/tmp": {"kind": "tmp"},
2137         "stdin": {"kind": "json", "content": "foo"},
2138         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
2139     },
2140                 "output_path": "/tmp",
2141                 "priority": 1,
2142                 "runtime_constraints": {},
2143                 "state": "Locked"
2144         }`
2145
2146         api, _, _ := s.fullRunHelper(c, helperRecord, nil, func() int {
2147                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
2148                 return 0
2149         })
2150
2151         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
2152         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
2153         for _, v := range api.Content {
2154                 if v["collection"] != nil {
2155                         collection := v["collection"].(arvadosclient.Dict)
2156                         if strings.Index(collection["name"].(string), "output") == 0 {
2157                                 manifest := collection["manifest_text"].(string)
2158                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
2159 `)
2160                         }
2161                 }
2162         }
2163 }
2164
2165 func (s *TestSuite) TestStderrMount(c *C) {
2166         api, cr, _ := s.fullRunHelper(c, `{
2167     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
2168     "container_image": "`+arvadostest.DockerImage112PDH+`",
2169     "cwd": ".",
2170     "environment": {},
2171     "mounts": {"/tmp": {"kind": "tmp"},
2172                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
2173                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
2174     "output_path": "/tmp",
2175     "priority": 1,
2176     "runtime_constraints": {},
2177     "state": "Locked"
2178 }`, nil, func() int {
2179                 fmt.Fprintln(s.executor.created.Stdout, "hello")
2180                 fmt.Fprintln(s.executor.created.Stderr, "oops")
2181                 return 1
2182         })
2183
2184         final := api.CalledWith("container.state", "Complete")
2185         c.Assert(final, NotNil)
2186         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
2187         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
2188
2189         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
2190 }
2191
2192 func (s *TestSuite) TestNumberRoundTrip(c *C) {
2193         s.api.callraw = true
2194         err := s.runner.fetchContainerRecord()
2195         c.Assert(err, IsNil)
2196         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
2197         c.Logf("%#v", s.runner.Container)
2198         c.Check(err, IsNil)
2199         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
2200 }
2201
2202 func (s *TestSuite) TestFullBrokenDocker(c *C) {
2203         nextState := ""
2204         for _, setup := range []func(){
2205                 func() {
2206                         c.Log("// waitErr = ocl runtime error")
2207                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
2208                         nextState = "Cancelled"
2209                 },
2210                 func() {
2211                         c.Log("// loadErr = cannot connect")
2212                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
2213                         s.runner.brokenNodeHook = c.MkDir() + "/broken-node-hook"
2214                         err := ioutil.WriteFile(s.runner.brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
2215                         c.Assert(err, IsNil)
2216                         nextState = "Queued"
2217                 },
2218         } {
2219                 s.SetUpTest(c)
2220                 setup()
2221                 s.fullRunHelper(c, `{
2222     "command": ["echo", "hello world"],
2223     "container_image": "`+arvadostest.DockerImage112PDH+`",
2224     "cwd": ".",
2225     "environment": {},
2226     "mounts": {"/tmp": {"kind": "tmp"} },
2227     "output_path": "/tmp",
2228     "priority": 1,
2229     "runtime_constraints": {},
2230     "state": "Locked"
2231 }`, nil, func() int { return 0 })
2232                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
2233                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
2234                 if s.runner.brokenNodeHook != "" {
2235                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
2236                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
2237                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
2238                 } else {
2239                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
2240                 }
2241         }
2242 }
2243
2244 func (s *TestSuite) TestBadCommand(c *C) {
2245         for _, startError := range []string{
2246                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
2247                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
2248                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
2249         } {
2250                 s.SetUpTest(c)
2251                 s.executor.startErr = errors.New(startError)
2252                 s.fullRunHelper(c, `{
2253     "command": ["echo", "hello world"],
2254     "container_image": "`+arvadostest.DockerImage112PDH+`",
2255     "cwd": ".",
2256     "environment": {},
2257     "mounts": {"/tmp": {"kind": "tmp"} },
2258     "output_path": "/tmp",
2259     "priority": 1,
2260     "runtime_constraints": {},
2261     "state": "Locked"
2262 }`, nil, func() int { return 0 })
2263                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
2264                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
2265         }
2266 }
2267
2268 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
2269         helperRecord := `{
2270                 "command": ["true"],
2271                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2272                 "cwd": "/bin",
2273                 "mounts": {
2274                     "/tmp": {"kind": "tmp"},
2275                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2276                 },
2277                 "secret_mounts": {
2278                 },
2279                 "output_path": "/tmp",
2280                 "priority": 1,
2281                 "runtime_constraints": {},
2282                 "state": "Locked"
2283         }`
2284
2285         s.fullRunHelper(c, helperRecord, nil, func() int {
2286                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2287                 c.Check(err, IsNil)
2288                 c.Check(string(content), Equals, "mypassword")
2289                 return 0
2290         })
2291
2292         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2293         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2294         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
2295         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
2296
2297         // under secret mounts, not captured in output
2298         helperRecord = `{
2299                 "command": ["true"],
2300                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2301                 "cwd": "/bin",
2302                 "mounts": {
2303                     "/tmp": {"kind": "tmp"}
2304                 },
2305                 "secret_mounts": {
2306                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2307                 },
2308                 "output_path": "/tmp",
2309                 "priority": 1,
2310                 "runtime_constraints": {},
2311                 "state": "Locked"
2312         }`
2313
2314         s.SetUpTest(c)
2315         s.fullRunHelper(c, helperRecord, nil, func() int {
2316                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2317                 c.Check(err, IsNil)
2318                 c.Check(string(content), Equals, "mypassword")
2319                 return 0
2320         })
2321
2322         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2323         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2324         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2325         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2326
2327         // under secret mounts, output dir is a collection, not captured in output
2328         helperRecord = `{
2329                 "command": ["true"],
2330                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2331                 "cwd": "/bin",
2332                 "mounts": {
2333                     "/tmp": {"kind": "collection", "writable": true}
2334                 },
2335                 "secret_mounts": {
2336                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2337                 },
2338                 "output_path": "/tmp",
2339                 "priority": 1,
2340                 "runtime_constraints": {},
2341                 "state": "Locked"
2342         }`
2343
2344         s.SetUpTest(c)
2345         _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, func() int {
2346                 // secret.conf should be provisioned as a separate
2347                 // bind mount, i.e., it should not appear in the
2348                 // (fake) fuse filesystem as viewed from the host.
2349                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2350                 if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) {
2351                         c.Logf("secret.conf: content %q, err %#v", content, err)
2352                 }
2353                 err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700)
2354                 c.Check(err, IsNil)
2355                 return 0
2356         })
2357
2358         content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text")
2359         c.Check(err, IsNil)
2360         c.Check(string(content), Equals, "mypassword")
2361         c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true})
2362         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2363         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2364         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil)
2365 }
2366
2367 func (s *TestSuite) TestCalculateCost(c *C) {
2368         defer func(s string) { lockdir = s }(lockdir)
2369         lockdir = c.MkDir()
2370         now := time.Now()
2371         cr := s.runner
2372         cr.costStartTime = now.Add(-time.Hour)
2373         var logbuf bytes.Buffer
2374         cr.CrunchLog.Immediate = log.New(&logbuf, "", 0)
2375
2376         // if there's no InstanceType env var, cost is calculated as 0
2377         os.Unsetenv("InstanceType")
2378         cost := cr.calculateCost(now)
2379         c.Check(cost, Equals, 0.0)
2380
2381         // with InstanceType env var and loadPrices() hasn't run (or
2382         // hasn't found any data), cost is calculated based on
2383         // InstanceType env var
2384         os.Setenv("InstanceType", `{"Price":1.2}`)
2385         defer os.Unsetenv("InstanceType")
2386         cost = cr.calculateCost(now)
2387         c.Check(cost, Equals, 1.2)
2388
2389         // first update tells us the spot price was $1/h until 30
2390         // minutes ago when it increased to $2/h
2391         j, err := json.Marshal([]cloud.InstancePrice{
2392                 {StartTime: now.Add(-4 * time.Hour), Price: 1.0},
2393                 {StartTime: now.Add(-time.Hour / 2), Price: 2.0},
2394         })
2395         c.Assert(err, IsNil)
2396         os.WriteFile(lockdir+"/"+pricesfile, j, 0777)
2397         cr.loadPrices()
2398         cost = cr.calculateCost(now)
2399         c.Check(cost, Equals, 1.5)
2400
2401         // next update (via --list + SIGUSR2) tells us the spot price
2402         // increased to $3/h 15 minutes ago
2403         j, err = json.Marshal([]cloud.InstancePrice{
2404                 {StartTime: now.Add(-time.Hour / 3), Price: 2.0}, // dup of -time.Hour/2 price
2405                 {StartTime: now.Add(-time.Hour / 4), Price: 3.0},
2406         })
2407         c.Assert(err, IsNil)
2408         os.WriteFile(lockdir+"/"+pricesfile, j, 0777)
2409         cr.loadPrices()
2410         cost = cr.calculateCost(now)
2411         c.Check(cost, Equals, 1.0/2+2.0/4+3.0/4)
2412
2413         cost = cr.calculateCost(now.Add(-time.Hour / 2))
2414         c.Check(cost, Equals, 0.5)
2415
2416         c.Logf("%s", logbuf.String())
2417         c.Check(logbuf.String(), Matches, `(?ms).*Instance price changed to 1\.00 at 20.* changed to 2\.00 .* changed to 3\.00 .*`)
2418         c.Check(logbuf.String(), Not(Matches), `(?ms).*changed to 2\.00 .* changed to 2\.00 .*`)
2419 }
2420
2421 func (s *TestSuite) TestSIGUSR2CostUpdate(c *C) {
2422         pid := os.Getpid()
2423         now := time.Now()
2424         pricesJSON, err := json.Marshal([]cloud.InstancePrice{
2425                 {StartTime: now.Add(-4 * time.Hour), Price: 2.4},
2426                 {StartTime: now.Add(-2 * time.Hour), Price: 2.6},
2427         })
2428         c.Assert(err, IsNil)
2429
2430         os.Setenv("InstanceType", `{"Price":2.2}`)
2431         defer os.Unsetenv("InstanceType")
2432         defer func(s string) { lockdir = s }(lockdir)
2433         lockdir = c.MkDir()
2434
2435         // We can't use s.api.CalledWith because timing differences will yield
2436         // different cost values across runs. getCostUpdate iterates over API
2437         // calls until it finds one that sets the cost, then writes that value
2438         // to the next index of costUpdates.
2439         deadline := now.Add(time.Second)
2440         costUpdates := make([]float64, 2)
2441         costIndex := 0
2442         apiIndex := 0
2443         getCostUpdate := func() {
2444                 for ; time.Now().Before(deadline); time.Sleep(time.Second / 10) {
2445                         for apiIndex < len(s.api.Content) {
2446                                 update := s.api.Content[apiIndex]
2447                                 apiIndex++
2448                                 var ok bool
2449                                 var cost float64
2450                                 if update, ok = update["container"].(arvadosclient.Dict); !ok {
2451                                         continue
2452                                 }
2453                                 if cost, ok = update["cost"].(float64); !ok {
2454                                         continue
2455                                 }
2456                                 c.Logf("API call #%d updates cost to %v", apiIndex-1, cost)
2457                                 costUpdates[costIndex] = cost
2458                                 costIndex++
2459                                 return
2460                         }
2461                 }
2462         }
2463
2464         s.fullRunHelper(c, `{
2465                 "command": ["true"],
2466                 "container_image": "`+arvadostest.DockerImage112PDH+`",
2467                 "cwd": ".",
2468                 "environment": {},
2469                 "mounts": {"/tmp": {"kind": "tmp"} },
2470                 "output_path": "/tmp",
2471                 "priority": 1,
2472                 "runtime_constraints": {},
2473                 "state": "Locked",
2474                 "uuid": "zzzzz-dz642-20230320101530a"
2475         }`, nil, func() int {
2476                 s.runner.costStartTime = now.Add(-3 * time.Hour)
2477                 err := syscall.Kill(pid, syscall.SIGUSR2)
2478                 c.Check(err, IsNil, Commentf("error sending first SIGUSR2 to runner"))
2479                 getCostUpdate()
2480
2481                 err = os.WriteFile(path.Join(lockdir, pricesfile), pricesJSON, 0o700)
2482                 c.Check(err, IsNil, Commentf("error writing JSON prices file"))
2483                 err = syscall.Kill(pid, syscall.SIGUSR2)
2484                 c.Check(err, IsNil, Commentf("error sending second SIGUSR2 to runner"))
2485                 getCostUpdate()
2486
2487                 return 0
2488         })
2489         // Comparing with format strings makes it easy to ignore minor variations
2490         // in cost across runs while keeping diagnostics pretty.
2491         c.Check(fmt.Sprintf("%.3f", costUpdates[0]), Equals, "6.600")
2492         c.Check(fmt.Sprintf("%.3f", costUpdates[1]), Equals, "7.600")
2493 }
2494
2495 type FakeProcess struct {
2496         cmdLine []string
2497 }
2498
2499 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2500         return fp.cmdLine, nil
2501 }