lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "log"
  16         "math/rand"
  17         "net/http"
  18         "net/http/httptest"
  19         "os"
  20         "os/exec"
  21         "path"
  22         "regexp"
  23         "runtime/pprof"
  24         "strconv"
  25         "strings"
  26         "sync"
  27         "sync/atomic"
  28         "syscall"
  29         "testing"
  30         "time"
  31
  32         "git.arvados.org/arvados.git/lib/cloud"
  33         "git.arvados.org/arvados.git/lib/cmd"
  34         "git.arvados.org/arvados.git/sdk/go/arvados"
  35         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  36         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  37         "git.arvados.org/arvados.git/sdk/go/manifest"
  38         "golang.org/x/net/context"
  39
  40         . "gopkg.in/check.v1"
  41 )
  42
  43 // Gocheck boilerplate
  44 func TestCrunchExec(t *testing.T) {
  45         TestingT(t)
  46 }
  47
  48 const logLineStart = `(?m)(.*\n)*\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d+Z `
  49
  50 var _ = Suite(&TestSuite{})
  51
  52 type TestSuite struct {
  53         client                   *arvados.Client
  54         api                      *ArvTestClient
  55         runner                   *ContainerRunner
  56         executor                 *stubExecutor
  57         keepmount                string
  58         keepmountTmp             []string
  59         testDispatcherKeepClient KeepTestClient
  60         testContainerKeepClient  KeepTestClient
  61 }
  62
  63 func (s *TestSuite) SetUpTest(c *C) {
  64         s.client = arvados.NewClientFromEnv()
  65         s.executor = &stubExecutor{}
  66         var err error
  67         s.api = &ArvTestClient{}
  68         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  69         c.Assert(err, IsNil)
  70         s.runner.executor = s.executor
  71         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  72                 return s.api, &s.testContainerKeepClient, s.client, nil
  73         }
  74         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  75                 s.runner.ArvMountPoint = s.keepmount
  76                 for i, opt := range cmd {
  77                         if opt == "--mount-tmp" {
  78                                 err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700)
  79                                 if err != nil {
  80                                         return nil, err
  81                                 }
  82                                 s.keepmountTmp = append(s.keepmountTmp, cmd[i+1])
  83                         }
  84                 }
  85                 return nil, nil
  86         }
  87         s.keepmount = c.MkDir()
  88         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  89         s.keepmountTmp = nil
  90         c.Assert(err, IsNil)
  91         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  92         c.Assert(err, IsNil)
  93         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  94         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  95         c.Assert(err, IsNil)
  96         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  97         c.Assert(err, IsNil)
  98         s.runner.ArvMountPoint = s.keepmount
  99 }
 100
 101 type ArvTestClient struct {
 102         Total   int64
 103         Calls   int
 104         Content []arvadosclient.Dict
 105         arvados.Container
 106         secretMounts []byte
 107         Logs         map[string]*bytes.Buffer
 108         sync.Mutex
 109         WasSetRunning bool
 110         callraw       bool
 111 }
 112
 113 type KeepTestClient struct {
 114         Called         bool
 115         Content        []byte
 116         StorageClasses []string
 117 }
 118
 119 type stubExecutor struct {
 120         imageLoaded bool
 121         loaded      string
 122         loadErr     error
 123         exitCode    int
 124         createErr   error
 125         created     containerSpec
 126         startErr    error
 127         waitSleep   time.Duration
 128         waitErr     error
 129         stopErr     error
 130         stopped     bool
 131         closed      bool
 132         runFunc     func() int
 133         exit        chan int
 134 }
 135
 136 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 137         containerClient *arvados.Client) error {
 138         e.loaded = tarball
 139         return e.loadErr
 140 }
 141 func (e *stubExecutor) Runtime() string                 { return "stub" }
 142 func (e *stubExecutor) Version() string                 { return "stub " + cmd.Version.String() }
 143 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 144 func (e *stubExecutor) Start() error {
 145         e.exit = make(chan int, 1)
 146         go func() { e.exit <- e.runFunc() }()
 147         return e.startErr
 148 }
 149 func (e *stubExecutor) CgroupID() string { return "cgroupid" }
 150 func (e *stubExecutor) Stop() error      { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 151 func (e *stubExecutor) Close()           { e.closed = true }
 152 func (e *stubExecutor) Wait(context.Context) (int, error) {
 153         return <-e.exit, e.waitErr
 154 }
 155 func (e *stubExecutor) InjectCommand(ctx context.Context, _, _ string, _ bool, _ []string) (*exec.Cmd, error) {
 156         return nil, errors.New("unimplemented")
 157 }
 158 func (e *stubExecutor) IPAddress() (string, error) { return "", errors.New("unimplemented") }
 159
 160 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 161
 162 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 163 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 164 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 165
 166 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 167 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 168
 169 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 170 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 171 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 172 `
 173
 174 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 175
 176 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 177 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 178
 179 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 180 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 181
 182 func (client *ArvTestClient) Create(resourceType string,
 183         parameters arvadosclient.Dict,
 184         output interface{}) error {
 185
 186         client.Mutex.Lock()
 187         defer client.Mutex.Unlock()
 188
 189         client.Calls++
 190         client.Content = append(client.Content, parameters)
 191
 192         if resourceType == "logs" {
 193                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 194                 if client.Logs == nil {
 195                         client.Logs = make(map[string]*bytes.Buffer)
 196                 }
 197                 if client.Logs[et] == nil {
 198                         client.Logs[et] = &bytes.Buffer{}
 199                 }
 200                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 201         }
 202
 203         if resourceType == "collections" && output != nil {
 204                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 205                 md5sum := md5.Sum([]byte(mt))
 206                 outmap := output.(*arvados.Collection)
 207                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5sum, len(mt))
 208                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%015x", md5sum[:7])
 209         }
 210
 211         return nil
 212 }
 213
 214 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 215         switch {
 216         case method == "GET" && resourceType == "containers" && action == "auth":
 217                 return json.Unmarshal([]byte(`{
 218                         "kind": "arvados#api_client_authorization",
 219                         "uuid": "`+fakeAuthUUID+`",
 220                         "api_token": "`+fakeAuthToken+`"
 221                         }`), output)
 222         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 223                 if client.secretMounts != nil {
 224                         return json.Unmarshal(client.secretMounts, output)
 225                 }
 226                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 227         default:
 228                 return fmt.Errorf("Not found")
 229         }
 230 }
 231
 232 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 233         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 234         var j []byte
 235         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 236                 j = []byte(`{
 237                         "kind": "arvados#nodeList",
 238                         "items": [{
 239                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 240                                 "hostname": "compute2",
 241                                 "properties": {"total_cpu_cores": 16}
 242                         }]}`)
 243         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 244                 if uuid == "" {
 245                         j, err = json.Marshal(map[string]interface{}{
 246                                 "items": []interface{}{client.Container},
 247                                 "kind":  "arvados#nodeList",
 248                         })
 249                 } else {
 250                         j, err = json.Marshal(client.Container)
 251                 }
 252         } else {
 253                 j = []byte(`{
 254                         "command": ["sleep", "1"],
 255                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 256                         "cwd": ".",
 257                         "environment": {},
 258                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 259                         "output_path": "/tmp",
 260                         "priority": 1,
 261                         "runtime_constraints": {}
 262                 }`)
 263         }
 264         return ioutil.NopCloser(bytes.NewReader(j)), err
 265 }
 266
 267 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 268         if resourceType == "collections" {
 269                 if uuid == hwPDH {
 270                         output.(*arvados.Collection).ManifestText = hwManifest
 271                 } else if uuid == otherPDH {
 272                         output.(*arvados.Collection).ManifestText = otherManifest
 273                 } else if uuid == normalizedWithSubdirsPDH {
 274                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 275                 } else if uuid == denormalizedWithSubdirsPDH {
 276                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 277                 }
 278         }
 279         if resourceType == "containers" {
 280                 (*output.(*arvados.Container)) = client.Container
 281         }
 282         return nil
 283 }
 284
 285 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 286         client.Mutex.Lock()
 287         defer client.Mutex.Unlock()
 288         client.Calls++
 289         client.Content = append(client.Content, parameters)
 290         if resourceType == "containers" {
 291                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 292                         client.WasSetRunning = true
 293                 }
 294         } else if resourceType == "collections" && output != nil {
 295                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 296                 output.(*arvados.Collection).UUID = uuid
 297                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 298         }
 299         return nil
 300 }
 301
 302 var discoveryMap = map[string]interface{}{
 303         "defaultTrashLifetime":               float64(1209600),
 304         "crunchLimitLogBytesPerJob":          float64(67108864),
 305         "crunchLogThrottleBytes":             float64(65536),
 306         "crunchLogThrottlePeriod":            float64(60),
 307         "crunchLogThrottleLines":             float64(1024),
 308         "crunchLogPartialLineThrottlePeriod": float64(5),
 309         "crunchLogBytesPerEvent":             float64(4096),
 310         "crunchLogSecondsBetweenEvents":      float64(1),
 311 }
 312
 313 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 314         return discoveryMap[key], nil
 315 }
 316
 317 // CalledWith returns the parameters from the first API call whose
 318 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 319 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 320 // no call matches, it returns nil.
 321 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 322 call:
 323         for _, content := range client.Content {
 324                 var v interface{} = content
 325                 for _, k := range strings.Split(jpath, ".") {
 326                         if dict, ok := v.(arvadosclient.Dict); !ok {
 327                                 continue call
 328                         } else {
 329                                 v = dict[k]
 330                         }
 331                 }
 332                 if v == expect {
 333                         return content
 334                 }
 335         }
 336         return nil
 337 }
 338
 339 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 340         return locator, nil
 341 }
 342
 343 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 344         client.Content = opts.Data
 345         return arvados.BlockWriteResponse{
 346                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 347         }, nil
 348 }
 349
 350 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 351         return 0, errors.New("not implemented")
 352 }
 353
 354 func (client *KeepTestClient) ClearBlockCache() {
 355 }
 356
 357 func (client *KeepTestClient) Close() {
 358         client.Content = nil
 359 }
 360
 361 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 362         client.StorageClasses = sc
 363 }
 364
 365 type FileWrapper struct {
 366         io.ReadCloser
 367         len int64
 368 }
 369
 370 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 371         return nil, errors.New("not implemented")
 372 }
 373
 374 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 375         return 0, errors.New("not implemented")
 376 }
 377
 378 func (fw FileWrapper) Size() int64 {
 379         return fw.len
 380 }
 381
 382 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 383         return nil, errors.New("not implemented")
 384 }
 385
 386 func (fw FileWrapper) Truncate(int64) error {
 387         return errors.New("not implemented")
 388 }
 389
 390 func (fw FileWrapper) Write([]byte) (int, error) {
 391         return 0, errors.New("not implemented")
 392 }
 393
 394 func (fw FileWrapper) Sync() error {
 395         return errors.New("not implemented")
 396 }
 397
 398 func (fw FileWrapper) Snapshot() (*arvados.Subtree, error) {
 399         return nil, errors.New("not implemented")
 400 }
 401
 402 func (fw FileWrapper) Splice(*arvados.Subtree) error {
 403         return errors.New("not implemented")
 404 }
 405
 406 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 407         if filename == hwImageID+".tar" {
 408                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 409                 client.Called = true
 410                 return FileWrapper{rdr, 1321984}, nil
 411         } else if filename == "/file1_in_main.txt" {
 412                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 413                 client.Called = true
 414                 return FileWrapper{rdr, 3}, nil
 415         }
 416         return nil, nil
 417 }
 418
 419 func (s *TestSuite) TestLoadImage(c *C) {
 420         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 421         s.runner.Container.Mounts = map[string]arvados.Mount{
 422                 "/out": {Kind: "tmp", Writable: true},
 423         }
 424         s.runner.Container.OutputPath = "/out"
 425
 426         _, err := s.runner.SetupMounts()
 427         c.Assert(err, IsNil)
 428
 429         imageID, err := s.runner.LoadImage()
 430         c.Check(err, IsNil)
 431         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 432         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 433
 434         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 435         s.executor.imageLoaded = false
 436         s.executor.loaded = ""
 437         s.executor.loadErr = errors.New("bork")
 438         imageID, err = s.runner.LoadImage()
 439         c.Check(err, ErrorMatches, ".*bork")
 440         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 441
 442         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 443         s.executor.imageLoaded = false
 444         s.executor.loaded = ""
 445         s.executor.loadErr = nil
 446         imageID, err = s.runner.LoadImage()
 447         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 448         c.Check(s.executor.loaded, Equals, "")
 449 }
 450
 451 type ArvErrorTestClient struct{}
 452
 453 func (ArvErrorTestClient) Create(resourceType string,
 454         parameters arvadosclient.Dict,
 455         output interface{}) error {
 456         return nil
 457 }
 458
 459 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 460         if method == "GET" && resourceType == "containers" && action == "auth" {
 461                 return nil
 462         }
 463         return errors.New("ArvError")
 464 }
 465
 466 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 467         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 468         return nil, errors.New("ArvError")
 469 }
 470
 471 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 472         return errors.New("ArvError")
 473 }
 474
 475 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 476         return nil
 477 }
 478
 479 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 480         return discoveryMap[key], nil
 481 }
 482
 483 type KeepErrorTestClient struct {
 484         KeepTestClient
 485 }
 486
 487 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 488         return nil, errors.New("KeepError")
 489 }
 490
 491 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 492         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 493 }
 494
 495 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 496         return "", errors.New("KeepError")
 497 }
 498
 499 type KeepReadErrorTestClient struct {
 500         KeepTestClient
 501 }
 502
 503 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 504         return 0, errors.New("KeepError")
 505 }
 506
 507 type ErrorReader struct {
 508         FileWrapper
 509 }
 510
 511 func (ErrorReader) Read(p []byte) (n int, err error) {
 512         return 0, errors.New("ErrorReader")
 513 }
 514
 515 func (ErrorReader) Seek(int64, int) (int64, error) {
 516         return 0, errors.New("ErrorReader")
 517 }
 518
 519 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 520         return ErrorReader{}, nil
 521 }
 522
 523 type ClosableBuffer struct {
 524         bytes.Buffer
 525 }
 526
 527 func (*ClosableBuffer) Close() error {
 528         return nil
 529 }
 530
 531 type TestLogs struct {
 532         Stdout ClosableBuffer
 533         Stderr ClosableBuffer
 534 }
 535
 536 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 537         if logstr == "stdout" {
 538                 return &tl.Stdout, nil
 539         }
 540         if logstr == "stderr" {
 541                 return &tl.Stderr, nil
 542         }
 543         return nil, errors.New("???")
 544 }
 545
 546 func dockerLog(fd byte, msg string) []byte {
 547         by := []byte(msg)
 548         header := make([]byte, 8+len(by))
 549         header[0] = fd
 550         header[7] = byte(len(by))
 551         copy(header[8:], by)
 552         return header
 553 }
 554
 555 func (s *TestSuite) TestRunContainer(c *C) {
 556         s.executor.runFunc = func() int {
 557                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 558                 return 0
 559         }
 560
 561         var logs TestLogs
 562         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 563         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 564         s.runner.Container.Command = []string{"./hw"}
 565         s.runner.Container.OutputStorageClasses = []string{"default"}
 566
 567         imageID, err := s.runner.LoadImage()
 568         c.Assert(err, IsNil)
 569
 570         err = s.runner.CreateContainer(imageID, nil)
 571         c.Assert(err, IsNil)
 572
 573         err = s.runner.StartContainer()
 574         c.Assert(err, IsNil)
 575
 576         err = s.runner.WaitFinish()
 577         c.Assert(err, IsNil)
 578
 579         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 580         c.Check(logs.Stderr.String(), Equals, "")
 581 }
 582
 583 func (s *TestSuite) TestCommitLogs(c *C) {
 584         api := &ArvTestClient{}
 585         kc := &KeepTestClient{}
 586         defer kc.Close()
 587         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 588         c.Assert(err, IsNil)
 589         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 590
 591         cr.CrunchLog.Print("Hello world!")
 592         cr.CrunchLog.Print("Goodbye")
 593         cr.finalState = "Complete"
 594
 595         err = cr.CommitLogs()
 596         c.Check(err, IsNil)
 597
 598         c.Check(api.Calls, Equals, 2)
 599         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 600         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 601         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 602         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 603 }
 604
 605 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 606         api := &ArvTestClient{}
 607         kc := &KeepTestClient{}
 608         defer kc.Close()
 609         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 610         c.Assert(err, IsNil)
 611
 612         err = cr.UpdateContainerRunning("")
 613         c.Check(err, IsNil)
 614
 615         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 616 }
 617
 618 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 619         api := &ArvTestClient{}
 620         kc := &KeepTestClient{}
 621         defer kc.Close()
 622         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 623         c.Assert(err, IsNil)
 624
 625         cr.LogsPDH = new(string)
 626         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 627
 628         cr.ExitCode = new(int)
 629         *cr.ExitCode = 42
 630         cr.finalState = "Complete"
 631
 632         err = cr.UpdateContainerFinal()
 633         c.Check(err, IsNil)
 634
 635         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 636         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 637         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 638 }
 639
 640 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 641         api := &ArvTestClient{}
 642         kc := &KeepTestClient{}
 643         defer kc.Close()
 644         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 645         c.Assert(err, IsNil)
 646         cr.cCancelled = true
 647         cr.finalState = "Cancelled"
 648
 649         err = cr.UpdateContainerFinal()
 650         c.Check(err, IsNil)
 651
 652         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 653         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 654         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 655 }
 656
 657 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 658 // dress rehearsal of the Run() function, starting from a JSON container record.
 659 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, fn func() int) (*ArvTestClient, *ContainerRunner, string) {
 660         err := json.Unmarshal([]byte(record), &s.api.Container)
 661         c.Assert(err, IsNil)
 662         initialState := s.api.Container.State
 663
 664         var sm struct {
 665                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 666         }
 667         err = json.Unmarshal([]byte(record), &sm)
 668         c.Check(err, IsNil)
 669         secretMounts, err := json.Marshal(sm)
 670         c.Assert(err, IsNil)
 671         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 672
 673         s.executor.runFunc = fn
 674
 675         s.runner.statInterval = 100 * time.Millisecond
 676         s.runner.containerWatchdogInterval = time.Second
 677
 678         realTemp := c.MkDir()
 679         tempcount := 0
 680         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 681                 tempcount++
 682                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 683                 err := os.Mkdir(d, os.ModePerm)
 684                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 685                         // Test case must have pre-populated the tempdir
 686                         err = nil
 687                 }
 688                 return d, err
 689         }
 690         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 691                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 692         }
 693
 694         if extraMounts != nil && len(extraMounts) > 0 {
 695                 err := s.runner.SetupArvMountPoint("keep")
 696                 c.Check(err, IsNil)
 697
 698                 for _, m := range extraMounts {
 699                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 700                 }
 701         }
 702
 703         err = s.runner.Run()
 704         if s.api.CalledWith("container.state", "Complete") != nil {
 705                 c.Check(err, IsNil)
 706         }
 707         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 708                 c.Check(s.api.WasSetRunning, Equals, true)
 709                 var lastupdate arvadosclient.Dict
 710                 for _, content := range s.api.Content {
 711                         if content["container"] != nil {
 712                                 lastupdate = content["container"].(arvadosclient.Dict)
 713                         }
 714                 }
 715                 if lastupdate["log"] == nil {
 716                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 717                 }
 718         }
 719
 720         if err != nil {
 721                 for k, v := range s.api.Logs {
 722                         c.Log(k)
 723                         c.Log(v.String())
 724                 }
 725         }
 726
 727         return s.api, s.runner, realTemp
 728 }
 729
 730 func (s *TestSuite) TestFullRunHello(c *C) {
 731         s.runner.enableMemoryLimit = true
 732         s.runner.networkMode = "default"
 733         s.fullRunHelper(c, `{
 734     "command": ["echo", "hello world"],
 735     "container_image": "`+arvadostest.DockerImage112PDH+`",
 736     "cwd": ".",
 737     "environment": {"foo":"bar","baz":"waz"},
 738     "mounts": {"/tmp": {"kind": "tmp"} },
 739     "output_path": "/tmp",
 740     "priority": 1,
 741     "runtime_constraints": {"vcpus":1,"ram":1000000},
 742     "state": "Locked",
 743     "output_storage_classes": ["default"]
 744 }`, nil, func() int {
 745                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 746                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 747                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 748                 c.Check(s.executor.created.VCPUs, Equals, 1)
 749                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 750                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 751                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 752                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
 753                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 754                 return 0
 755         })
 756
 757         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 758         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 759         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 760         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 761         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 762 }
 763
 764 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 765         var ran bool
 766         s.fullRunHelper(c, `{
 767     "command": ["sleep", "3"],
 768     "container_image": "`+arvadostest.DockerImage112PDH+`",
 769     "cwd": ".",
 770     "environment": {},
 771     "mounts": {"/tmp": {"kind": "tmp"} },
 772     "output_path": "/tmp",
 773     "priority": 1,
 774     "runtime_constraints": {},
 775     "scheduling_parameters":{"max_run_time": 1},
 776     "state": "Running"
 777 }`, nil, func() int {
 778                 ran = true
 779                 return 2
 780         })
 781         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 782         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 783         c.Check(ran, Equals, false)
 784 }
 785
 786 func ec2MetadataServerStub(c *C, token *string, failureRate float64, stoptime *atomic.Value) *httptest.Server {
 787         failedOnce := false
 788         return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 789                 if !failedOnce || rand.Float64() < failureRate {
 790                         w.WriteHeader(http.StatusServiceUnavailable)
 791                         failedOnce = true
 792                         return
 793                 }
 794                 switch r.URL.Path {
 795                 case "/latest/api/token":
 796                         fmt.Fprintln(w, *token)
 797                 case "/latest/meta-data/spot/instance-action":
 798                         if r.Header.Get("X-aws-ec2-metadata-token") != *token {
 799                                 w.WriteHeader(http.StatusUnauthorized)
 800                         } else if t, _ := stoptime.Load().(time.Time); t.IsZero() {
 801                                 w.WriteHeader(http.StatusNotFound)
 802                         } else {
 803                                 fmt.Fprintf(w, `{"action":"stop","time":"%s"}`, t.Format(time.RFC3339))
 804                         }
 805                 default:
 806                         w.WriteHeader(http.StatusNotFound)
 807                 }
 808         }))
 809 }
 810
 811 func (s *TestSuite) TestSpotInterruptionNotice(c *C) {
 812         s.testSpotInterruptionNotice(c, 0.1)
 813 }
 814
 815 func (s *TestSuite) TestSpotInterruptionNoticeNotAvailable(c *C) {
 816         s.testSpotInterruptionNotice(c, 1)
 817 }
 818
 819 func (s *TestSuite) testSpotInterruptionNotice(c *C, failureRate float64) {
 820         var stoptime atomic.Value
 821         token := "fake-ec2-metadata-token"
 822         stub := ec2MetadataServerStub(c, &token, failureRate, &stoptime)
 823         defer stub.Close()
 824
 825         defer func(i time.Duration, u string) {
 826                 spotInterruptionCheckInterval = i
 827                 ec2MetadataBaseURL = u
 828         }(spotInterruptionCheckInterval, ec2MetadataBaseURL)
 829         spotInterruptionCheckInterval = time.Second / 8
 830         ec2MetadataBaseURL = stub.URL
 831
 832         go s.runner.checkSpotInterruptionNotices()
 833         s.fullRunHelper(c, `{
 834     "command": ["sleep", "3"],
 835     "container_image": "`+arvadostest.DockerImage112PDH+`",
 836     "cwd": ".",
 837     "environment": {},
 838     "mounts": {"/tmp": {"kind": "tmp"} },
 839     "output_path": "/tmp",
 840     "priority": 1,
 841     "runtime_constraints": {},
 842     "state": "Locked"
 843 }`, nil, func() int {
 844                 time.Sleep(time.Second)
 845                 stoptime.Store(time.Now().Add(time.Minute).UTC())
 846                 token = "different-fake-ec2-metadata-token"
 847                 time.Sleep(time.Second)
 848                 return 0
 849         })
 850         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Checking for spot interruptions every 125ms using instance metadata at http://.*`)
 851         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Error checking spot interruptions: 503 Service Unavailable.*`)
 852         if failureRate == 1 {
 853                 c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Giving up on checking spot interruptions after too many consecutive failures.*`)
 854         } else {
 855                 text := `Cloud provider scheduled instance stop at ` + stoptime.Load().(time.Time).Format(time.RFC3339)
 856                 c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*`+text+`.*`)
 857                 c.Check(s.api.CalledWith("container.runtime_status.warning", "preemption notice"), NotNil)
 858                 c.Check(s.api.CalledWith("container.runtime_status.warningDetail", text), NotNil)
 859                 c.Check(s.api.CalledWith("container.runtime_status.preemptionNotice", text), NotNil)
 860         }
 861 }
 862
 863 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 864         s.fullRunHelper(c, `{
 865     "command": ["sleep", "3"],
 866     "container_image": "`+arvadostest.DockerImage112PDH+`",
 867     "cwd": ".",
 868     "environment": {},
 869     "mounts": {"/tmp": {"kind": "tmp"} },
 870     "output_path": "/tmp",
 871     "priority": 1,
 872     "runtime_constraints": {},
 873     "scheduling_parameters":{"max_run_time": 1},
 874     "state": "Locked"
 875 }`, nil, func() int {
 876                 time.Sleep(3 * time.Second)
 877                 return 0
 878         })
 879
 880         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 881         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 882 }
 883
 884 func (s *TestSuite) TestContainerWaitFails(c *C) {
 885         s.fullRunHelper(c, `{
 886     "command": ["sleep", "3"],
 887     "container_image": "`+arvadostest.DockerImage112PDH+`",
 888     "cwd": ".",
 889     "mounts": {"/tmp": {"kind": "tmp"} },
 890     "output_path": "/tmp",
 891     "priority": 1,
 892     "state": "Locked"
 893 }`, nil, func() int {
 894                 s.executor.waitErr = errors.New("Container is not running")
 895                 return 0
 896         })
 897
 898         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 899         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 900 }
 901
 902 func (s *TestSuite) TestCrunchstat(c *C) {
 903         s.fullRunHelper(c, `{
 904                 "command": ["sleep", "1"],
 905                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 906                 "cwd": ".",
 907                 "environment": {},
 908                 "mounts": {"/tmp": {"kind": "tmp"} },
 909                 "output_path": "/tmp",
 910                 "priority": 1,
 911                 "runtime_constraints": {},
 912                 "state": "Locked"
 913         }`, nil, func() int {
 914                 time.Sleep(time.Second)
 915                 return 0
 916         })
 917
 918         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 919         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 920
 921         // We didn't actually start a container, so crunchstat didn't
 922         // find accounting files and therefore didn't log any stats.
 923         // It should have logged a "can't find accounting files"
 924         // message after one poll interval, though, so we can confirm
 925         // it's alive:
 926         c.Assert(s.api.Logs["crunchstat"], NotNil)
 927         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 928
 929         // The "files never appeared" log assures us that we called
 930         // (*crunchstat.Reporter)Stop(), and that we set it up with
 931         // the correct container ID "abcde":
 932         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 933 }
 934
 935 func (s *TestSuite) TestNodeInfoLog(c *C) {
 936         os.Setenv("SLURMD_NODENAME", "compute2")
 937         s.fullRunHelper(c, `{
 938                 "command": ["sleep", "1"],
 939                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 940                 "cwd": ".",
 941                 "environment": {},
 942                 "mounts": {"/tmp": {"kind": "tmp"} },
 943                 "output_path": "/tmp",
 944                 "priority": 1,
 945                 "runtime_constraints": {},
 946                 "state": "Locked"
 947         }`, nil, func() int {
 948                 time.Sleep(time.Second)
 949                 return 0
 950         })
 951
 952         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 953         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 954
 955         c.Assert(s.api.Logs["node"], NotNil)
 956         json := s.api.Logs["node"].String()
 957         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 958         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 959         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 960
 961         c.Assert(s.api.Logs["node-info"], NotNil)
 962         json = s.api.Logs["node-info"].String()
 963         c.Check(json, Matches, `(?ms).*Host Information.*`)
 964         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 965         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 966         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 967         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 968 }
 969
 970 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 971         s.fullRunHelper(c, `{
 972                 "command": ["sleep", "1"],
 973                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 974                 "cwd": ".",
 975                 "environment": {},
 976                 "mounts": {"/tmp": {"kind": "tmp"} },
 977                 "output_path": "/tmp",
 978                 "priority": 1,
 979                 "runtime_constraints": {},
 980                 "state": "Locked"
 981         }`, nil, func() int {
 982                 return 0
 983         })
 984
 985         c.Assert(s.api.Logs["crunch-run"], NotNil)
 986         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 987         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`)
 988         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container: zzzzz-zzzzz-zzzzzzzzzzzzzzz.*`)
 989         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Using container runtime: stub.*`)
 990 }
 991
 992 func (s *TestSuite) testLogRSSThresholds(c *C, ram int, expected []int, notExpected int) {
 993         s.runner.cgroupRoot = "testdata/fakestat"
 994         s.fullRunHelper(c, `{
 995                 "command": ["true"],
 996                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 997                 "cwd": ".",
 998                 "environment": {},
 999                 "mounts": {"/tmp": {"kind": "tmp"} },
1000                 "output_path": "/tmp",
1001                 "priority": 1,
1002                 "runtime_constraints": {"ram": `+strconv.Itoa(ram)+`},
1003                 "state": "Locked"
1004         }`, nil, func() int { return 0 })
1005         logs := s.api.Logs["crunch-run"].String()
1006         pattern := logLineStart + `Container using over %d%% of memory \(rss 734003200/%d bytes\)`
1007         var threshold int
1008         for _, threshold = range expected {
1009                 c.Check(logs, Matches, fmt.Sprintf(pattern, threshold, ram))
1010         }
1011         if notExpected > threshold {
1012                 c.Check(logs, Not(Matches), fmt.Sprintf(pattern, notExpected, ram))
1013         }
1014 }
1015
1016 func (s *TestSuite) TestLogNoRSSThresholds(c *C) {
1017         s.testLogRSSThresholds(c, 7340032000, []int{}, 90)
1018 }
1019
1020 func (s *TestSuite) TestLogSomeRSSThresholds(c *C) {
1021         onePercentRSS := 7340032
1022         s.testLogRSSThresholds(c, 102*onePercentRSS, []int{90, 95}, 99)
1023 }
1024
1025 func (s *TestSuite) TestLogAllRSSThresholds(c *C) {
1026         s.testLogRSSThresholds(c, 734003299, []int{90, 95, 99}, 0)
1027 }
1028
1029 func (s *TestSuite) TestLogMaximaAfterRun(c *C) {
1030         s.runner.cgroupRoot = "testdata/fakestat"
1031         s.runner.parentTemp = c.MkDir()
1032         s.fullRunHelper(c, `{
1033         "command": ["true"],
1034         "container_image": "`+arvadostest.DockerImage112PDH+`",
1035         "cwd": ".",
1036         "environment": {},
1037         "mounts": {"/tmp": {"kind": "tmp"} },
1038         "output_path": "/tmp",
1039         "priority": 1,
1040         "runtime_constraints": {"ram": 7340032000},
1041         "state": "Locked"
1042     }`, nil, func() int { return 0 })
1043         logs := s.api.Logs["crunch-run"].String()
1044         for _, expected := range []string{
1045                 `Maximum disk usage was \d+%, \d+/\d+ bytes`,
1046                 `Maximum container memory cache usage was 73400320 bytes`,
1047                 `Maximum container memory swap usage was 320 bytes`,
1048                 `Maximum container memory pgmajfault usage was 20 faults`,
1049                 `Maximum container memory rss usage was 10%, 734003200/7340032000 bytes`,
1050                 `Maximum crunch-run memory rss usage was \d+ bytes`,
1051         } {
1052                 c.Check(logs, Matches, logLineStart+expected)
1053         }
1054 }
1055
1056 func (s *TestSuite) TestCommitNodeInfoBeforeStart(c *C) {
1057         var collection_create, container_update arvadosclient.Dict
1058         s.fullRunHelper(c, `{
1059                 "command": ["true"],
1060                 "container_image": "`+arvadostest.DockerImage112PDH+`",
1061                 "cwd": ".",
1062                 "environment": {},
1063                 "mounts": {"/tmp": {"kind": "tmp"} },
1064                 "output_path": "/tmp",
1065                 "priority": 1,
1066                 "runtime_constraints": {},
1067                 "state": "Locked",
1068                 "uuid": "zzzzz-dz642-202301121543210"
1069         }`, nil, func() int {
1070                 collection_create = s.api.CalledWith("ensure_unique_name", true)
1071                 container_update = s.api.CalledWith("container.state", "Running")
1072                 return 0
1073         })
1074
1075         c.Assert(collection_create, NotNil)
1076         log_collection := collection_create["collection"].(arvadosclient.Dict)
1077         c.Check(log_collection["name"], Equals, "logs for zzzzz-dz642-202301121543210")
1078         manifest_text := log_collection["manifest_text"].(string)
1079         // We check that the file size is at least two digits as an easy way to
1080         // check the file isn't empty.
1081         c.Check(manifest_text, Matches, `\. .+ \d+:\d{2,}:node-info\.txt( .+)?\n`)
1082         c.Check(manifest_text, Matches, `\. .+ \d+:\d{2,}:node\.json( .+)?\n`)
1083
1084         c.Assert(container_update, NotNil)
1085         // As of Arvados 2.5.0, the container update must specify its log in PDH
1086         // format for the API server to propagate it to container requests, which
1087         // is what we care about for this test.
1088         expect_pdh := fmt.Sprintf("%x+%d", md5.Sum([]byte(manifest_text)), len(manifest_text))
1089         c.Check(container_update["container"].(arvadosclient.Dict)["log"], Equals, expect_pdh)
1090 }
1091
1092 func (s *TestSuite) TestContainerRecordLog(c *C) {
1093         s.fullRunHelper(c, `{
1094                 "command": ["sleep", "1"],
1095                 "container_image": "`+arvadostest.DockerImage112PDH+`",
1096                 "cwd": ".",
1097                 "environment": {},
1098                 "mounts": {"/tmp": {"kind": "tmp"} },
1099                 "output_path": "/tmp",
1100                 "priority": 1,
1101                 "runtime_constraints": {},
1102                 "state": "Locked"
1103         }`, nil,
1104                 func() int {
1105                         time.Sleep(time.Second)
1106                         return 0
1107                 })
1108
1109         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1110         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1111
1112         c.Assert(s.api.Logs["container"], NotNil)
1113         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
1114 }
1115
1116 func (s *TestSuite) TestFullRunStderr(c *C) {
1117         s.fullRunHelper(c, `{
1118     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
1119     "container_image": "`+arvadostest.DockerImage112PDH+`",
1120     "cwd": ".",
1121     "environment": {},
1122     "mounts": {"/tmp": {"kind": "tmp"} },
1123     "output_path": "/tmp",
1124     "priority": 1,
1125     "runtime_constraints": {},
1126     "state": "Locked"
1127 }`, nil, func() int {
1128                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1129                 fmt.Fprintln(s.executor.created.Stderr, "world")
1130                 return 1
1131         })
1132
1133         final := s.api.CalledWith("container.state", "Complete")
1134         c.Assert(final, NotNil)
1135         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1136         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1137
1138         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
1139         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
1140 }
1141
1142 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
1143         s.fullRunHelper(c, `{
1144     "command": ["pwd"],
1145     "container_image": "`+arvadostest.DockerImage112PDH+`",
1146     "cwd": ".",
1147     "environment": {},
1148     "mounts": {"/tmp": {"kind": "tmp"} },
1149     "output_path": "/tmp",
1150     "priority": 1,
1151     "runtime_constraints": {},
1152     "state": "Locked"
1153 }`, nil, func() int {
1154                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
1155                 return 0
1156         })
1157
1158         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1159         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1160         c.Log(s.api.Logs["stdout"])
1161         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
1162 }
1163
1164 func (s *TestSuite) TestFullRunSetCwd(c *C) {
1165         s.fullRunHelper(c, `{
1166     "command": ["pwd"],
1167     "container_image": "`+arvadostest.DockerImage112PDH+`",
1168     "cwd": "/bin",
1169     "environment": {},
1170     "mounts": {"/tmp": {"kind": "tmp"} },
1171     "output_path": "/tmp",
1172     "priority": 1,
1173     "runtime_constraints": {},
1174     "state": "Locked"
1175 }`, nil, func() int {
1176                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1177                 return 0
1178         })
1179
1180         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1181         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1182         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1183 }
1184
1185 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
1186         s.fullRunHelper(c, `{
1187     "command": ["pwd"],
1188     "container_image": "`+arvadostest.DockerImage112PDH+`",
1189     "cwd": "/bin",
1190     "environment": {},
1191     "mounts": {"/tmp": {"kind": "tmp"} },
1192     "output_path": "/tmp",
1193     "priority": 1,
1194     "runtime_constraints": {},
1195     "state": "Locked",
1196     "output_storage_classes": ["foo", "bar"]
1197 }`, nil, func() int {
1198                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1199                 return 0
1200         })
1201
1202         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1203         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1204         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1205         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1206         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1207 }
1208
1209 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
1210         s.fullRunHelper(c, `{
1211     "command": ["pwd"],
1212     "container_image": "`+arvadostest.DockerImage112PDH+`",
1213     "cwd": "/bin",
1214     "environment": {},
1215     "mounts": {"/tmp": {"kind": "tmp"} },
1216     "output_path": "/tmp",
1217     "priority": 1,
1218     "runtime_constraints": {"cuda": {"device_count": 2}},
1219     "state": "Locked",
1220     "output_storage_classes": ["foo", "bar"]
1221 }`, nil, func() int {
1222                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1223                 return 0
1224         })
1225         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1226 }
1227
1228 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1229         s.fullRunHelper(c, `{
1230     "command": ["pwd"],
1231     "container_image": "`+arvadostest.DockerImage112PDH+`",
1232     "cwd": "/bin",
1233     "environment": {},
1234     "mounts": {"/tmp": {"kind": "tmp"} },
1235     "output_path": "/tmp",
1236     "priority": 1,
1237     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1238     "state": "Locked",
1239     "output_storage_classes": ["foo", "bar"]
1240 }`, nil, func() int {
1241                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1242                 return 0
1243         })
1244         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1245 }
1246
1247 func (s *TestSuite) TestStopOnSignal(c *C) {
1248         s.executor.runFunc = func() int {
1249                 s.executor.created.Stdout.Write([]byte("foo\n"))
1250                 s.runner.SigChan <- syscall.SIGINT
1251                 time.Sleep(10 * time.Second)
1252                 return 0
1253         }
1254         s.testStopContainer(c)
1255 }
1256
1257 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1258         s.executor.runFunc = func() int {
1259                 s.executor.created.Stdout.Write([]byte("foo\n"))
1260                 s.runner.ArvMountExit <- nil
1261                 close(s.runner.ArvMountExit)
1262                 time.Sleep(10 * time.Second)
1263                 return 0
1264         }
1265         s.runner.ArvMountExit = make(chan error)
1266         s.testStopContainer(c)
1267 }
1268
1269 func (s *TestSuite) testStopContainer(c *C) {
1270         record := `{
1271     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1272     "container_image": "` + arvadostest.DockerImage112PDH + `",
1273     "cwd": ".",
1274     "environment": {},
1275     "mounts": {"/tmp": {"kind": "tmp"} },
1276     "output_path": "/tmp",
1277     "priority": 1,
1278     "runtime_constraints": {},
1279     "state": "Locked"
1280 }`
1281
1282         err := json.Unmarshal([]byte(record), &s.api.Container)
1283         c.Assert(err, IsNil)
1284
1285         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1286         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1287                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1288         }
1289
1290         done := make(chan error)
1291         go func() {
1292                 done <- s.runner.Run()
1293         }()
1294         select {
1295         case <-time.After(20 * time.Second):
1296                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1297                 c.Fatal("timed out")
1298         case err = <-done:
1299                 c.Check(err, IsNil)
1300         }
1301         for k, v := range s.api.Logs {
1302                 c.Log(k)
1303                 c.Log(v.String(), "\n")
1304         }
1305
1306         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1307         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1308         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1309 }
1310
1311 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1312         s.fullRunHelper(c, `{
1313     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1314     "container_image": "`+arvadostest.DockerImage112PDH+`",
1315     "cwd": "/bin",
1316     "environment": {"FROBIZ": "bilbo"},
1317     "mounts": {"/tmp": {"kind": "tmp"} },
1318     "output_path": "/tmp",
1319     "priority": 1,
1320     "runtime_constraints": {},
1321     "state": "Locked"
1322 }`, nil, func() int {
1323                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1324                 return 0
1325         })
1326
1327         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1328         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1329         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1330 }
1331
1332 type ArvMountCmdLine struct {
1333         Cmd   []string
1334         token string
1335 }
1336
1337 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1338         am.Cmd = c
1339         am.token = token
1340         return nil, nil
1341 }
1342
1343 func stubCert(temp string) string {
1344         path := temp + "/ca-certificates.crt"
1345         crt, _ := os.Create(path)
1346         crt.Close()
1347         arvadosclient.CertFiles = []string{path}
1348         return path
1349 }
1350
1351 func (s *TestSuite) TestSetupMounts(c *C) {
1352         cr := s.runner
1353         am := &ArvMountCmdLine{}
1354         cr.RunArvMount = am.ArvMountTest
1355         cr.ContainerArvClient = &ArvTestClient{}
1356         cr.ContainerKeepClient = &KeepTestClient{}
1357         cr.Container.OutputStorageClasses = []string{"default"}
1358
1359         realTemp := c.MkDir()
1360         certTemp := c.MkDir()
1361         stubCertPath := stubCert(certTemp)
1362         cr.parentTemp = realTemp
1363
1364         i := 0
1365         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1366                 i++
1367                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1368                 err := os.Mkdir(d, os.ModePerm)
1369                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1370                         // Test case must have pre-populated the tempdir
1371                         err = nil
1372                 }
1373                 return d, err
1374         }
1375
1376         checkEmpty := func() {
1377                 // Should be deleted.
1378                 _, err := os.Stat(realTemp)
1379                 c.Assert(os.IsNotExist(err), Equals, true)
1380
1381                 // Now recreate it for the next test.
1382                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1383         }
1384
1385         {
1386                 i = 0
1387                 cr.ArvMountPoint = ""
1388                 cr.Container.Mounts = make(map[string]arvados.Mount)
1389                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1390                 cr.Container.OutputPath = "/tmp"
1391                 cr.statInterval = 5 * time.Second
1392                 bindmounts, err := cr.SetupMounts()
1393                 c.Check(err, IsNil)
1394                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1395                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1396                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1397                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1398                 os.RemoveAll(cr.ArvMountPoint)
1399                 cr.CleanupDirs()
1400                 checkEmpty()
1401         }
1402
1403         {
1404                 i = 0
1405                 cr.ArvMountPoint = ""
1406                 cr.Container.Mounts = make(map[string]arvados.Mount)
1407                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1408                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1409                 cr.Container.OutputPath = "/out"
1410                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1411
1412                 bindmounts, err := cr.SetupMounts()
1413                 c.Check(err, IsNil)
1414                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1415                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1416                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1417                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1418                 os.RemoveAll(cr.ArvMountPoint)
1419                 cr.CleanupDirs()
1420                 checkEmpty()
1421         }
1422
1423         {
1424                 i = 0
1425                 cr.ArvMountPoint = ""
1426                 cr.Container.Mounts = make(map[string]arvados.Mount)
1427                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1428                 cr.Container.OutputPath = "/tmp"
1429                 cr.Container.RuntimeConstraints.API = true
1430                 cr.Container.OutputStorageClasses = []string{"default"}
1431
1432                 bindmounts, err := cr.SetupMounts()
1433                 c.Check(err, IsNil)
1434                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1435                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1436                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1437                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1438                 os.RemoveAll(cr.ArvMountPoint)
1439                 cr.CleanupDirs()
1440                 checkEmpty()
1441
1442                 cr.Container.RuntimeConstraints.API = false
1443         }
1444
1445         {
1446                 i = 0
1447                 cr.ArvMountPoint = ""
1448                 cr.Container.Mounts = map[string]arvados.Mount{
1449                         "/keeptmp": {Kind: "collection", Writable: true},
1450                 }
1451                 cr.Container.OutputPath = "/keeptmp"
1452
1453                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1454
1455                 bindmounts, err := cr.SetupMounts()
1456                 c.Check(err, IsNil)
1457                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1458                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1459                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1460                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1461                 os.RemoveAll(cr.ArvMountPoint)
1462                 cr.CleanupDirs()
1463                 checkEmpty()
1464         }
1465
1466         {
1467                 i = 0
1468                 cr.ArvMountPoint = ""
1469                 cr.Container.Mounts = map[string]arvados.Mount{
1470                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1471                         "/keepout": {Kind: "collection", Writable: true},
1472                 }
1473                 cr.Container.OutputPath = "/keepout"
1474
1475                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1476                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1477
1478                 bindmounts, err := cr.SetupMounts()
1479                 c.Check(err, IsNil)
1480                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1481                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1482                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1483                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1484                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1485                         "/keepout": {realTemp + "/keep1/tmp0", false},
1486                 })
1487                 os.RemoveAll(cr.ArvMountPoint)
1488                 cr.CleanupDirs()
1489                 checkEmpty()
1490         }
1491
1492         {
1493                 i = 0
1494                 cr.ArvMountPoint = ""
1495                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1496                 cr.Container.Mounts = map[string]arvados.Mount{
1497                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1498                         "/keepout": {Kind: "collection", Writable: true},
1499                 }
1500                 cr.Container.OutputPath = "/keepout"
1501
1502                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1503                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1504
1505                 bindmounts, err := cr.SetupMounts()
1506                 c.Check(err, IsNil)
1507                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1508                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
1509                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1510                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1511                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1512                         "/keepout": {realTemp + "/keep1/tmp0", false},
1513                 })
1514                 os.RemoveAll(cr.ArvMountPoint)
1515                 cr.CleanupDirs()
1516                 checkEmpty()
1517         }
1518
1519         for _, test := range []struct {
1520                 in  interface{}
1521                 out string
1522         }{
1523                 {in: "foo", out: `"foo"`},
1524                 {in: nil, out: `null`},
1525                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1526         } {
1527                 i = 0
1528                 cr.ArvMountPoint = ""
1529                 cr.Container.Mounts = map[string]arvados.Mount{
1530                         "/mnt/test.json": {Kind: "json", Content: test.in},
1531                 }
1532                 bindmounts, err := cr.SetupMounts()
1533                 c.Check(err, IsNil)
1534                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1535                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1536                 })
1537                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1538                 c.Check(err, IsNil)
1539                 c.Check(content, DeepEquals, []byte(test.out))
1540                 os.RemoveAll(cr.ArvMountPoint)
1541                 cr.CleanupDirs()
1542                 checkEmpty()
1543         }
1544
1545         for _, test := range []struct {
1546                 in  interface{}
1547                 out string
1548         }{
1549                 {in: "foo", out: `foo`},
1550                 {in: nil, out: "error"},
1551                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1552         } {
1553                 i = 0
1554                 cr.ArvMountPoint = ""
1555                 cr.Container.Mounts = map[string]arvados.Mount{
1556                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1557                 }
1558                 bindmounts, err := cr.SetupMounts()
1559                 if test.out == "error" {
1560                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1561                 } else {
1562                         c.Check(err, IsNil)
1563                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1564                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1565                         })
1566                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1567                         c.Check(err, IsNil)
1568                         c.Check(content, DeepEquals, []byte(test.out))
1569                 }
1570                 os.RemoveAll(cr.ArvMountPoint)
1571                 cr.CleanupDirs()
1572                 checkEmpty()
1573         }
1574
1575         // Read-only mount points are allowed underneath output_dir mount point
1576         {
1577                 i = 0
1578                 cr.ArvMountPoint = ""
1579                 cr.Container.Mounts = make(map[string]arvados.Mount)
1580                 cr.Container.Mounts = map[string]arvados.Mount{
1581                         "/tmp":     {Kind: "tmp"},
1582                         "/tmp/foo": {Kind: "collection"},
1583                 }
1584                 cr.Container.OutputPath = "/tmp"
1585
1586                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1587
1588                 bindmounts, err := cr.SetupMounts()
1589                 c.Check(err, IsNil)
1590                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1591                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
1592                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1593                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1594                         "/tmp":     {realTemp + "/tmp2", false},
1595                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1596                 })
1597                 os.RemoveAll(cr.ArvMountPoint)
1598                 cr.CleanupDirs()
1599                 checkEmpty()
1600         }
1601
1602         // Writable mount points copied to output_dir mount point
1603         {
1604                 i = 0
1605                 cr.ArvMountPoint = ""
1606                 cr.Container.Mounts = make(map[string]arvados.Mount)
1607                 cr.Container.Mounts = map[string]arvados.Mount{
1608                         "/tmp": {Kind: "tmp"},
1609                         "/tmp/foo": {Kind: "collection",
1610                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1611                                 Writable:         true},
1612                         "/tmp/bar": {Kind: "collection",
1613                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1614                                 Path:             "baz",
1615                                 Writable:         true},
1616                 }
1617                 cr.Container.OutputPath = "/tmp"
1618
1619                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1620                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1621
1622                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1623                 rf.Write([]byte("bar"))
1624                 rf.Close()
1625
1626                 _, err := cr.SetupMounts()
1627                 c.Check(err, IsNil)
1628                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1629                 c.Check(err, IsNil)
1630                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1631                 c.Check(err, IsNil)
1632                 os.RemoveAll(cr.ArvMountPoint)
1633                 cr.CleanupDirs()
1634                 checkEmpty()
1635         }
1636
1637         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1638         {
1639                 i = 0
1640                 cr.ArvMountPoint = ""
1641                 cr.Container.Mounts = make(map[string]arvados.Mount)
1642                 cr.Container.Mounts = map[string]arvados.Mount{
1643                         "/tmp":     {Kind: "tmp"},
1644                         "/tmp/foo": {Kind: "tmp"},
1645                 }
1646                 cr.Container.OutputPath = "/tmp"
1647
1648                 _, err := cr.SetupMounts()
1649                 c.Check(err, NotNil)
1650                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1651                 os.RemoveAll(cr.ArvMountPoint)
1652                 cr.CleanupDirs()
1653                 checkEmpty()
1654         }
1655
1656         // Only mount point of kind 'collection' is allowed for stdin
1657         {
1658                 i = 0
1659                 cr.ArvMountPoint = ""
1660                 cr.Container.Mounts = make(map[string]arvados.Mount)
1661                 cr.Container.Mounts = map[string]arvados.Mount{
1662                         "stdin": {Kind: "tmp"},
1663                 }
1664
1665                 _, err := cr.SetupMounts()
1666                 c.Check(err, NotNil)
1667                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1668                 os.RemoveAll(cr.ArvMountPoint)
1669                 cr.CleanupDirs()
1670                 checkEmpty()
1671         }
1672
1673         // git_tree mounts
1674         {
1675                 i = 0
1676                 cr.ArvMountPoint = ""
1677                 (*GitMountSuite)(nil).useTestGitServer(c)
1678                 cr.token = arvadostest.ActiveToken
1679                 cr.Container.Mounts = make(map[string]arvados.Mount)
1680                 cr.Container.Mounts = map[string]arvados.Mount{
1681                         "/tip": {
1682                                 Kind:   "git_tree",
1683                                 UUID:   arvadostest.Repository2UUID,
1684                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1685                                 Path:   "/",
1686                         },
1687                         "/non-tip": {
1688                                 Kind:   "git_tree",
1689                                 UUID:   arvadostest.Repository2UUID,
1690                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1691                                 Path:   "/",
1692                         },
1693                 }
1694                 cr.Container.OutputPath = "/tmp"
1695
1696                 bindmounts, err := cr.SetupMounts()
1697                 c.Check(err, IsNil)
1698
1699                 for path, mount := range bindmounts {
1700                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1701                 }
1702
1703                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1704                 c.Check(err, IsNil)
1705                 c.Check(string(data), Equals, "\000\001\002\003")
1706                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1707                 c.Check(err, FitsTypeOf, &os.PathError{})
1708                 c.Check(os.IsNotExist(err), Equals, true)
1709
1710                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1711                 c.Check(err, IsNil)
1712                 c.Check(string(data), Equals, "\000\001\002\003")
1713                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1714                 c.Check(err, IsNil)
1715                 c.Check(string(data), Equals, "testfile\n")
1716
1717                 cr.CleanupDirs()
1718                 checkEmpty()
1719         }
1720 }
1721
1722 func (s *TestSuite) TestStdout(c *C) {
1723         helperRecord := `{
1724                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1725                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1726                 "cwd": "/bin",
1727                 "environment": {"FROBIZ": "bilbo"},
1728                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1729                 "output_path": "/tmp",
1730                 "priority": 1,
1731                 "runtime_constraints": {},
1732                 "state": "Locked"
1733         }`
1734
1735         s.fullRunHelper(c, helperRecord, nil, func() int {
1736                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1737                 return 0
1738         })
1739
1740         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1741         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1742         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1743 }
1744
1745 // Used by the TestStdoutWithWrongPath*()
1746 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func() int) (*ArvTestClient, *ContainerRunner, error) {
1747         err := json.Unmarshal([]byte(record), &s.api.Container)
1748         c.Assert(err, IsNil)
1749         s.executor.runFunc = fn
1750         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1751         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1752                 return s.api, &KeepTestClient{}, nil, nil
1753         }
1754         return s.api, s.runner, s.runner.Run()
1755 }
1756
1757 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1758         _, _, err := s.stdoutErrorRunHelper(c, `{
1759     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1760     "output_path": "/tmp",
1761     "state": "Locked"
1762 }`, func() int { return 0 })
1763         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1764 }
1765
1766 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1767         _, _, err := s.stdoutErrorRunHelper(c, `{
1768     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1769     "output_path": "/tmp",
1770     "state": "Locked"
1771 }`, func() int { return 0 })
1772         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1773 }
1774
1775 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1776         _, _, err := s.stdoutErrorRunHelper(c, `{
1777     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1778     "output_path": "/tmp",
1779     "state": "Locked"
1780 }`, func() int { return 0 })
1781         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1782 }
1783
1784 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1785         s.fullRunHelper(c, `{
1786     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1787     "container_image": "`+arvadostest.DockerImage112PDH+`",
1788     "cwd": "/bin",
1789     "environment": {},
1790     "mounts": {"/tmp": {"kind": "tmp"} },
1791     "output_path": "/tmp",
1792     "priority": 1,
1793     "runtime_constraints": {"API": true},
1794     "state": "Locked"
1795 }`, nil, func() int {
1796                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1797                 return 3
1798         })
1799         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1800         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1801         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1802 }
1803
1804 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1805         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1806         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1807         s.fullRunHelper(c, `{
1808     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1809     "container_image": "`+arvadostest.DockerImage112PDH+`",
1810     "cwd": "/bin",
1811     "environment": {},
1812     "mounts": {"/tmp": {"kind": "tmp"} },
1813     "output_path": "/tmp",
1814     "priority": 1,
1815     "runtime_constraints": {"API": true},
1816     "state": "Locked"
1817 }`, nil, func() int {
1818                 s.api.Container.Output = arvadostest.DockerImage112PDH
1819                 return 0
1820         })
1821
1822         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1823         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1824         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1825 }
1826
1827 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1828         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1829                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1830                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1831                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1832         }
1833         s.executor.runFunc = func() int {
1834                 time.Sleep(time.Second)
1835                 return 137
1836         }
1837         record := `{
1838     "command": ["sleep", "1"],
1839     "container_image": "` + arvadostest.DockerImage112PDH + `",
1840     "cwd": "/bin",
1841     "environment": {},
1842     "mounts": {"/tmp": {"kind": "tmp"} },
1843     "output_path": "/tmp",
1844     "priority": 1,
1845     "runtime_constraints": {"API": true},
1846     "state": "Locked"
1847 }`
1848         err := json.Unmarshal([]byte(record), &s.api.Container)
1849         c.Assert(err, IsNil)
1850         err = s.runner.Run()
1851         c.Assert(err, IsNil)
1852         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1853         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1854         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1855         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1856         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1857 }
1858
1859 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1860         helperRecord := `{
1861                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1862                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1863                 "cwd": "/bin",
1864                 "environment": {"FROBIZ": "bilbo"},
1865                 "mounts": {
1866         "/tmp": {"kind": "tmp"},
1867         "/tmp/foo": {"kind": "collection",
1868                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1869                      "exclude_from_output": true
1870         },
1871         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1872     },
1873                 "output_path": "/tmp",
1874                 "priority": 1,
1875                 "runtime_constraints": {},
1876                 "state": "Locked"
1877         }`
1878
1879         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1880
1881         s.fullRunHelper(c, helperRecord, extraMounts, func() int {
1882                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1883                 return 0
1884         })
1885
1886         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1887         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1888         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1889 }
1890
1891 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1892         helperRecord := `{
1893                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1894                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1895                 "cwd": "/bin",
1896                 "environment": {"FROBIZ": "bilbo"},
1897                 "mounts": {
1898         "/tmp": {"kind": "tmp"},
1899         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1900         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1901         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1902         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1903         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1904     },
1905                 "output_path": "/tmp",
1906                 "priority": 1,
1907                 "runtime_constraints": {},
1908                 "state": "Locked",
1909                 "uuid": "zzzzz-dz642-202301130848001"
1910         }`
1911
1912         extraMounts := []string{
1913                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1914                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1915                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1916         }
1917
1918         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, func() int {
1919                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1920                 return 0
1921         })
1922
1923         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1924                 "/tmp":                   {realtemp + "/tmp1", false},
1925                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1926                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1927                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1928                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1929         })
1930
1931         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1932         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1933         output_count := uint(0)
1934         for _, v := range s.runner.ContainerArvClient.(*ArvTestClient).Content {
1935                 if v["collection"] == nil {
1936                         continue
1937                 }
1938                 collection := v["collection"].(arvadosclient.Dict)
1939                 if collection["name"].(string) != "output for zzzzz-dz642-202301130848001" {
1940                         continue
1941                 }
1942                 c.Check(v["ensure_unique_name"], Equals, true)
1943                 c.Check(collection["manifest_text"].(string), Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1944 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1945 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1946 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1947 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1948 `)
1949                 output_count++
1950         }
1951         c.Check(output_count, Not(Equals), uint(0))
1952 }
1953
1954 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1955         helperRecord := `{
1956                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1957                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1958                 "cwd": "/bin",
1959                 "environment": {"FROBIZ": "bilbo"},
1960                 "mounts": {
1961         "/tmp": {"kind": "tmp"},
1962         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1963         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1964     },
1965                 "output_path": "/tmp",
1966                 "priority": 1,
1967                 "runtime_constraints": {},
1968                 "state": "Locked",
1969                 "uuid": "zzzzz-dz642-202301130848002"
1970         }`
1971
1972         extraMounts := []string{
1973                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1974         }
1975
1976         s.fullRunHelper(c, helperRecord, extraMounts, func() int {
1977                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1978                 return 0
1979         })
1980
1981         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1982         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1983         output_count := uint(0)
1984         for _, v := range s.runner.ContainerArvClient.(*ArvTestClient).Content {
1985                 if v["collection"] == nil {
1986                         continue
1987                 }
1988                 collection := v["collection"].(arvadosclient.Dict)
1989                 if collection["name"].(string) != "output for zzzzz-dz642-202301130848002" {
1990                         continue
1991                 }
1992                 c.Check(collection["manifest_text"].(string), Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1993 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1994 `)
1995                 output_count++
1996         }
1997         c.Check(output_count, Not(Equals), uint(0))
1998 }
1999
2000 func (s *TestSuite) TestOutputError(c *C) {
2001         helperRecord := `{
2002                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
2003                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2004                 "cwd": "/bin",
2005                 "environment": {"FROBIZ": "bilbo"},
2006                 "mounts": {
2007                         "/tmp": {"kind": "tmp"}
2008                 },
2009                 "output_path": "/tmp",
2010                 "priority": 1,
2011                 "runtime_constraints": {},
2012                 "state": "Locked"
2013         }`
2014         s.fullRunHelper(c, helperRecord, nil, func() int {
2015                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
2016                 return 0
2017         })
2018
2019         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
2020 }
2021
2022 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
2023         helperRecord := `{
2024                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
2025                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2026                 "cwd": "/bin",
2027                 "environment": {"FROBIZ": "bilbo"},
2028                 "mounts": {
2029         "/tmp": {"kind": "tmp"},
2030         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
2031         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
2032     },
2033                 "output_path": "/tmp",
2034                 "priority": 1,
2035                 "runtime_constraints": {},
2036                 "state": "Locked"
2037         }`
2038
2039         extraMounts := []string{
2040                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
2041         }
2042
2043         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, func() int {
2044                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
2045                 return 0
2046         })
2047
2048         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
2049         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
2050         for _, v := range api.Content {
2051                 if v["collection"] != nil {
2052                         collection := v["collection"].(arvadosclient.Dict)
2053                         if strings.Index(collection["name"].(string), "output") == 0 {
2054                                 manifest := collection["manifest_text"].(string)
2055                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
2056 `)
2057                         }
2058                 }
2059         }
2060 }
2061
2062 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
2063         helperRecord := `{
2064                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
2065                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2066                 "cwd": "/bin",
2067                 "environment": {"FROBIZ": "bilbo"},
2068                 "mounts": {
2069         "/tmp": {"kind": "tmp"},
2070         "stdin": {"kind": "json", "content": "foo"},
2071         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
2072     },
2073                 "output_path": "/tmp",
2074                 "priority": 1,
2075                 "runtime_constraints": {},
2076                 "state": "Locked"
2077         }`
2078
2079         api, _, _ := s.fullRunHelper(c, helperRecord, nil, func() int {
2080                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
2081                 return 0
2082         })
2083
2084         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
2085         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
2086         for _, v := range api.Content {
2087                 if v["collection"] != nil {
2088                         collection := v["collection"].(arvadosclient.Dict)
2089                         if strings.Index(collection["name"].(string), "output") == 0 {
2090                                 manifest := collection["manifest_text"].(string)
2091                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
2092 `)
2093                         }
2094                 }
2095         }
2096 }
2097
2098 func (s *TestSuite) TestStderrMount(c *C) {
2099         api, cr, _ := s.fullRunHelper(c, `{
2100     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
2101     "container_image": "`+arvadostest.DockerImage112PDH+`",
2102     "cwd": ".",
2103     "environment": {},
2104     "mounts": {"/tmp": {"kind": "tmp"},
2105                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
2106                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
2107     "output_path": "/tmp",
2108     "priority": 1,
2109     "runtime_constraints": {},
2110     "state": "Locked"
2111 }`, nil, func() int {
2112                 fmt.Fprintln(s.executor.created.Stdout, "hello")
2113                 fmt.Fprintln(s.executor.created.Stderr, "oops")
2114                 return 1
2115         })
2116
2117         final := api.CalledWith("container.state", "Complete")
2118         c.Assert(final, NotNil)
2119         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
2120         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
2121
2122         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
2123 }
2124
2125 func (s *TestSuite) TestNumberRoundTrip(c *C) {
2126         s.api.callraw = true
2127         err := s.runner.fetchContainerRecord()
2128         c.Assert(err, IsNil)
2129         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
2130         c.Logf("%#v", s.runner.Container)
2131         c.Check(err, IsNil)
2132         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
2133 }
2134
2135 func (s *TestSuite) TestFullBrokenDocker(c *C) {
2136         nextState := ""
2137         for _, setup := range []func(){
2138                 func() {
2139                         c.Log("// waitErr = ocl runtime error")
2140                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
2141                         nextState = "Cancelled"
2142                 },
2143                 func() {
2144                         c.Log("// loadErr = cannot connect")
2145                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
2146                         s.runner.brokenNodeHook = c.MkDir() + "/broken-node-hook"
2147                         err := ioutil.WriteFile(s.runner.brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
2148                         c.Assert(err, IsNil)
2149                         nextState = "Queued"
2150                 },
2151         } {
2152                 s.SetUpTest(c)
2153                 setup()
2154                 s.fullRunHelper(c, `{
2155     "command": ["echo", "hello world"],
2156     "container_image": "`+arvadostest.DockerImage112PDH+`",
2157     "cwd": ".",
2158     "environment": {},
2159     "mounts": {"/tmp": {"kind": "tmp"} },
2160     "output_path": "/tmp",
2161     "priority": 1,
2162     "runtime_constraints": {},
2163     "state": "Locked"
2164 }`, nil, func() int { return 0 })
2165                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
2166                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
2167                 if s.runner.brokenNodeHook != "" {
2168                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
2169                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
2170                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
2171                 } else {
2172                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
2173                 }
2174         }
2175 }
2176
2177 func (s *TestSuite) TestBadCommand(c *C) {
2178         for _, startError := range []string{
2179                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
2180                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
2181                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
2182         } {
2183                 s.SetUpTest(c)
2184                 s.executor.startErr = errors.New(startError)
2185                 s.fullRunHelper(c, `{
2186     "command": ["echo", "hello world"],
2187     "container_image": "`+arvadostest.DockerImage112PDH+`",
2188     "cwd": ".",
2189     "environment": {},
2190     "mounts": {"/tmp": {"kind": "tmp"} },
2191     "output_path": "/tmp",
2192     "priority": 1,
2193     "runtime_constraints": {},
2194     "state": "Locked"
2195 }`, nil, func() int { return 0 })
2196                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
2197                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
2198         }
2199 }
2200
2201 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
2202         helperRecord := `{
2203                 "command": ["true"],
2204                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2205                 "cwd": "/bin",
2206                 "mounts": {
2207                     "/tmp": {"kind": "tmp"},
2208                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2209                 },
2210                 "secret_mounts": {
2211                 },
2212                 "output_path": "/tmp",
2213                 "priority": 1,
2214                 "runtime_constraints": {},
2215                 "state": "Locked"
2216         }`
2217
2218         s.fullRunHelper(c, helperRecord, nil, func() int {
2219                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2220                 c.Check(err, IsNil)
2221                 c.Check(string(content), Equals, "mypassword")
2222                 return 0
2223         })
2224
2225         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2226         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2227         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
2228         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
2229
2230         // under secret mounts, not captured in output
2231         helperRecord = `{
2232                 "command": ["true"],
2233                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2234                 "cwd": "/bin",
2235                 "mounts": {
2236                     "/tmp": {"kind": "tmp"}
2237                 },
2238                 "secret_mounts": {
2239                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2240                 },
2241                 "output_path": "/tmp",
2242                 "priority": 1,
2243                 "runtime_constraints": {},
2244                 "state": "Locked"
2245         }`
2246
2247         s.SetUpTest(c)
2248         s.fullRunHelper(c, helperRecord, nil, func() int {
2249                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2250                 c.Check(err, IsNil)
2251                 c.Check(string(content), Equals, "mypassword")
2252                 return 0
2253         })
2254
2255         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2256         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2257         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2258         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2259
2260         // under secret mounts, output dir is a collection, not captured in output
2261         helperRecord = `{
2262                 "command": ["true"],
2263                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2264                 "cwd": "/bin",
2265                 "mounts": {
2266                     "/tmp": {"kind": "collection", "writable": true}
2267                 },
2268                 "secret_mounts": {
2269                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2270                 },
2271                 "output_path": "/tmp",
2272                 "priority": 1,
2273                 "runtime_constraints": {},
2274                 "state": "Locked"
2275         }`
2276
2277         s.SetUpTest(c)
2278         _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, func() int {
2279                 // secret.conf should be provisioned as a separate
2280                 // bind mount, i.e., it should not appear in the
2281                 // (fake) fuse filesystem as viewed from the host.
2282                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2283                 if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) {
2284                         c.Logf("secret.conf: content %q, err %#v", content, err)
2285                 }
2286                 err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700)
2287                 c.Check(err, IsNil)
2288                 return 0
2289         })
2290
2291         content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text")
2292         c.Check(err, IsNil)
2293         c.Check(string(content), Equals, "mypassword")
2294         c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true})
2295         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2296         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2297         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil)
2298 }
2299
2300 func (s *TestSuite) TestCalculateCost(c *C) {
2301         defer func(s string) { lockdir = s }(lockdir)
2302         lockdir = c.MkDir()
2303         now := time.Now()
2304         cr := s.runner
2305         cr.costStartTime = now.Add(-time.Hour)
2306         var logbuf bytes.Buffer
2307         cr.CrunchLog.Immediate = log.New(&logbuf, "", 0)
2308
2309         // if there's no InstanceType env var, cost is calculated as 0
2310         os.Unsetenv("InstanceType")
2311         cost := cr.calculateCost(now)
2312         c.Check(cost, Equals, 0.0)
2313
2314         // with InstanceType env var and loadPrices() hasn't run (or
2315         // hasn't found any data), cost is calculated based on
2316         // InstanceType env var
2317         os.Setenv("InstanceType", `{"Price":1.2}`)
2318         defer os.Unsetenv("InstanceType")
2319         cost = cr.calculateCost(now)
2320         c.Check(cost, Equals, 1.2)
2321
2322         // first update tells us the spot price was $1/h until 30
2323         // minutes ago when it increased to $2/h
2324         j, err := json.Marshal([]cloud.InstancePrice{
2325                 {StartTime: now.Add(-4 * time.Hour), Price: 1.0},
2326                 {StartTime: now.Add(-time.Hour / 2), Price: 2.0},
2327         })
2328         c.Assert(err, IsNil)
2329         os.WriteFile(lockdir+"/"+pricesfile, j, 0777)
2330         cr.loadPrices()
2331         cost = cr.calculateCost(now)
2332         c.Check(cost, Equals, 1.5)
2333
2334         // next update (via --list + SIGUSR2) tells us the spot price
2335         // increased to $3/h 15 minutes ago
2336         j, err = json.Marshal([]cloud.InstancePrice{
2337                 {StartTime: now.Add(-time.Hour / 3), Price: 2.0}, // dup of -time.Hour/2 price
2338                 {StartTime: now.Add(-time.Hour / 4), Price: 3.0},
2339         })
2340         c.Assert(err, IsNil)
2341         os.WriteFile(lockdir+"/"+pricesfile, j, 0777)
2342         cr.loadPrices()
2343         cost = cr.calculateCost(now)
2344         c.Check(cost, Equals, 1.0/2+2.0/4+3.0/4)
2345
2346         cost = cr.calculateCost(now.Add(-time.Hour / 2))
2347         c.Check(cost, Equals, 0.5)
2348
2349         c.Logf("%s", logbuf.String())
2350         c.Check(logbuf.String(), Matches, `(?ms).*Instance price changed to 1\.00 at 20.* changed to 2\.00 .* changed to 3\.00 .*`)
2351         c.Check(logbuf.String(), Not(Matches), `(?ms).*changed to 2\.00 .* changed to 2\.00 .*`)
2352 }
2353
2354 func (s *TestSuite) TestSIGUSR2CostUpdate(c *C) {
2355         pid := os.Getpid()
2356         now := time.Now()
2357         pricesJSON, err := json.Marshal([]cloud.InstancePrice{
2358                 {StartTime: now.Add(-4 * time.Hour), Price: 2.4},
2359                 {StartTime: now.Add(-2 * time.Hour), Price: 2.6},
2360         })
2361         c.Assert(err, IsNil)
2362
2363         os.Setenv("InstanceType", `{"Price":2.2}`)
2364         defer os.Unsetenv("InstanceType")
2365         defer func(s string) { lockdir = s }(lockdir)
2366         lockdir = c.MkDir()
2367
2368         // We can't use s.api.CalledWith because timing differences will yield
2369         // different cost values across runs. getCostUpdate iterates over API
2370         // calls until it finds one that sets the cost, then writes that value
2371         // to the next index of costUpdates.
2372         deadline := now.Add(time.Second)
2373         costUpdates := make([]float64, 2)
2374         costIndex := 0
2375         apiIndex := 0
2376         getCostUpdate := func() {
2377                 for ; time.Now().Before(deadline); time.Sleep(time.Second / 10) {
2378                         for apiIndex < len(s.api.Content) {
2379                                 update := s.api.Content[apiIndex]
2380                                 apiIndex++
2381                                 var ok bool
2382                                 var cost float64
2383                                 if update, ok = update["container"].(arvadosclient.Dict); !ok {
2384                                         continue
2385                                 }
2386                                 if cost, ok = update["cost"].(float64); !ok {
2387                                         continue
2388                                 }
2389                                 c.Logf("API call #%d updates cost to %v", apiIndex-1, cost)
2390                                 costUpdates[costIndex] = cost
2391                                 costIndex++
2392                                 return
2393                         }
2394                 }
2395         }
2396
2397         s.fullRunHelper(c, `{
2398                 "command": ["true"],
2399                 "container_image": "`+arvadostest.DockerImage112PDH+`",
2400                 "cwd": ".",
2401                 "environment": {},
2402                 "mounts": {"/tmp": {"kind": "tmp"} },
2403                 "output_path": "/tmp",
2404                 "priority": 1,
2405                 "runtime_constraints": {},
2406                 "state": "Locked",
2407                 "uuid": "zzzzz-dz642-20230320101530a"
2408         }`, nil, func() int {
2409                 s.runner.costStartTime = now.Add(-3 * time.Hour)
2410                 err := syscall.Kill(pid, syscall.SIGUSR2)
2411                 c.Check(err, IsNil, Commentf("error sending first SIGUSR2 to runner"))
2412                 getCostUpdate()
2413
2414                 err = os.WriteFile(path.Join(lockdir, pricesfile), pricesJSON, 0o700)
2415                 c.Check(err, IsNil, Commentf("error writing JSON prices file"))
2416                 err = syscall.Kill(pid, syscall.SIGUSR2)
2417                 c.Check(err, IsNil, Commentf("error sending second SIGUSR2 to runner"))
2418                 getCostUpdate()
2419
2420                 return 0
2421         })
2422         // Comparing with format strings makes it easy to ignore minor variations
2423         // in cost across runs while keeping diagnostics pretty.
2424         c.Check(fmt.Sprintf("%.3f", costUpdates[0]), Equals, "6.600")
2425         c.Check(fmt.Sprintf("%.3f", costUpdates[1]), Equals, "7.600")
2426 }
2427
2428 type FakeProcess struct {
2429         cmdLine []string
2430 }
2431
2432 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2433         return fp.cmdLine, nil
2434 }