lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/lib/cmd"
  26         "git.arvados.org/arvados.git/sdk/go/arvados"
  27         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  28         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  29         "git.arvados.org/arvados.git/sdk/go/manifest"
  30         "golang.org/x/net/context"
  31
  32         . "gopkg.in/check.v1"
  33 )
  34
  35 // Gocheck boilerplate
  36 func TestCrunchExec(t *testing.T) {
  37         TestingT(t)
  38 }
  39
  40 var _ = Suite(&TestSuite{})
  41
  42 type TestSuite struct {
  43         client                   *arvados.Client
  44         api                      *ArvTestClient
  45         runner                   *ContainerRunner
  46         executor                 *stubExecutor
  47         keepmount                string
  48         keepmountTmp             []string
  49         testDispatcherKeepClient KeepTestClient
  50         testContainerKeepClient  KeepTestClient
  51 }
  52
  53 func (s *TestSuite) SetUpTest(c *C) {
  54         s.client = arvados.NewClientFromEnv()
  55         s.executor = &stubExecutor{}
  56         var err error
  57         s.api = &ArvTestClient{}
  58         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  59         c.Assert(err, IsNil)
  60         s.runner.executor = s.executor
  61         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  62                 return s.api, &s.testContainerKeepClient, s.client, nil
  63         }
  64         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  65                 s.runner.ArvMountPoint = s.keepmount
  66                 for i, opt := range cmd {
  67                         if opt == "--mount-tmp" {
  68                                 err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700)
  69                                 if err != nil {
  70                                         return nil, err
  71                                 }
  72                                 s.keepmountTmp = append(s.keepmountTmp, cmd[i+1])
  73                         }
  74                 }
  75                 return nil, nil
  76         }
  77         s.keepmount = c.MkDir()
  78         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  79         s.keepmountTmp = nil
  80         c.Assert(err, IsNil)
  81         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  82         c.Assert(err, IsNil)
  83         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  84         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  85         c.Assert(err, IsNil)
  86         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  87         c.Assert(err, IsNil)
  88         s.runner.ArvMountPoint = s.keepmount
  89 }
  90
  91 type ArvTestClient struct {
  92         Total   int64
  93         Calls   int
  94         Content []arvadosclient.Dict
  95         arvados.Container
  96         secretMounts []byte
  97         Logs         map[string]*bytes.Buffer
  98         sync.Mutex
  99         WasSetRunning bool
 100         callraw       bool
 101 }
 102
 103 type KeepTestClient struct {
 104         Called         bool
 105         Content        []byte
 106         StorageClasses []string
 107 }
 108
 109 type stubExecutor struct {
 110         imageLoaded bool
 111         loaded      string
 112         loadErr     error
 113         exitCode    int
 114         createErr   error
 115         created     containerSpec
 116         startErr    error
 117         waitSleep   time.Duration
 118         waitErr     error
 119         stopErr     error
 120         stopped     bool
 121         closed      bool
 122         runFunc     func()
 123         exit        chan int
 124 }
 125
 126 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 127         containerClient *arvados.Client) error {
 128         e.loaded = tarball
 129         return e.loadErr
 130 }
 131 func (e *stubExecutor) Runtime() string                 { return "stub" }
 132 func (e *stubExecutor) Version() string                 { return "stub " + cmd.Version.String() }
 133 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 134 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 135 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 136 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 137 func (e *stubExecutor) Close()                          { e.closed = true }
 138 func (e *stubExecutor) Wait(context.Context) (int, error) {
 139         return <-e.exit, e.waitErr
 140 }
 141 func (e *stubExecutor) InjectCommand(ctx context.Context, _, _ string, _ bool, _ []string) (*exec.Cmd, error) {
 142         return nil, errors.New("unimplemented")
 143 }
 144 func (e *stubExecutor) IPAddress() (string, error) { return "", errors.New("unimplemented") }
 145
 146 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 147
 148 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 149 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 150 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 151
 152 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 153 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 154
 155 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 156 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 157 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 158 `
 159
 160 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 161
 162 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 163 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 164
 165 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 166 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 167
 168 func (client *ArvTestClient) Create(resourceType string,
 169         parameters arvadosclient.Dict,
 170         output interface{}) error {
 171
 172         client.Mutex.Lock()
 173         defer client.Mutex.Unlock()
 174
 175         client.Calls++
 176         client.Content = append(client.Content, parameters)
 177
 178         if resourceType == "logs" {
 179                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 180                 if client.Logs == nil {
 181                         client.Logs = make(map[string]*bytes.Buffer)
 182                 }
 183                 if client.Logs[et] == nil {
 184                         client.Logs[et] = &bytes.Buffer{}
 185                 }
 186                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 187         }
 188
 189         if resourceType == "collections" && output != nil {
 190                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 191                 outmap := output.(*arvados.Collection)
 192                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 193                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 194         }
 195
 196         return nil
 197 }
 198
 199 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 200         switch {
 201         case method == "GET" && resourceType == "containers" && action == "auth":
 202                 return json.Unmarshal([]byte(`{
 203                         "kind": "arvados#api_client_authorization",
 204                         "uuid": "`+fakeAuthUUID+`",
 205                         "api_token": "`+fakeAuthToken+`"
 206                         }`), output)
 207         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 208                 if client.secretMounts != nil {
 209                         return json.Unmarshal(client.secretMounts, output)
 210                 }
 211                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 212         default:
 213                 return fmt.Errorf("Not found")
 214         }
 215 }
 216
 217 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 218         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 219         var j []byte
 220         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 221                 j = []byte(`{
 222                         "kind": "arvados#nodeList",
 223                         "items": [{
 224                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 225                                 "hostname": "compute2",
 226                                 "properties": {"total_cpu_cores": 16}
 227                         }]}`)
 228         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 229                 if uuid == "" {
 230                         j, err = json.Marshal(map[string]interface{}{
 231                                 "items": []interface{}{client.Container},
 232                                 "kind":  "arvados#nodeList",
 233                         })
 234                 } else {
 235                         j, err = json.Marshal(client.Container)
 236                 }
 237         } else {
 238                 j = []byte(`{
 239                         "command": ["sleep", "1"],
 240                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 241                         "cwd": ".",
 242                         "environment": {},
 243                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 244                         "output_path": "/tmp",
 245                         "priority": 1,
 246                         "runtime_constraints": {}
 247                 }`)
 248         }
 249         return ioutil.NopCloser(bytes.NewReader(j)), err
 250 }
 251
 252 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 253         if resourceType == "collections" {
 254                 if uuid == hwPDH {
 255                         output.(*arvados.Collection).ManifestText = hwManifest
 256                 } else if uuid == otherPDH {
 257                         output.(*arvados.Collection).ManifestText = otherManifest
 258                 } else if uuid == normalizedWithSubdirsPDH {
 259                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 260                 } else if uuid == denormalizedWithSubdirsPDH {
 261                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 262                 }
 263         }
 264         if resourceType == "containers" {
 265                 (*output.(*arvados.Container)) = client.Container
 266         }
 267         return nil
 268 }
 269
 270 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 271         client.Mutex.Lock()
 272         defer client.Mutex.Unlock()
 273         client.Calls++
 274         client.Content = append(client.Content, parameters)
 275         if resourceType == "containers" {
 276                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 277                         client.WasSetRunning = true
 278                 }
 279         } else if resourceType == "collections" && output != nil {
 280                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 281                 output.(*arvados.Collection).UUID = uuid
 282                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 283         }
 284         return nil
 285 }
 286
 287 var discoveryMap = map[string]interface{}{
 288         "defaultTrashLifetime":               float64(1209600),
 289         "crunchLimitLogBytesPerJob":          float64(67108864),
 290         "crunchLogThrottleBytes":             float64(65536),
 291         "crunchLogThrottlePeriod":            float64(60),
 292         "crunchLogThrottleLines":             float64(1024),
 293         "crunchLogPartialLineThrottlePeriod": float64(5),
 294         "crunchLogBytesPerEvent":             float64(4096),
 295         "crunchLogSecondsBetweenEvents":      float64(1),
 296 }
 297
 298 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 299         return discoveryMap[key], nil
 300 }
 301
 302 // CalledWith returns the parameters from the first API call whose
 303 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 304 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 305 // no call matches, it returns nil.
 306 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 307 call:
 308         for _, content := range client.Content {
 309                 var v interface{} = content
 310                 for _, k := range strings.Split(jpath, ".") {
 311                         if dict, ok := v.(arvadosclient.Dict); !ok {
 312                                 continue call
 313                         } else {
 314                                 v = dict[k]
 315                         }
 316                 }
 317                 if v == expect {
 318                         return content
 319                 }
 320         }
 321         return nil
 322 }
 323
 324 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 325         return locator, nil
 326 }
 327
 328 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 329         client.Content = opts.Data
 330         return arvados.BlockWriteResponse{
 331                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 332         }, nil
 333 }
 334
 335 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 336         return 0, errors.New("not implemented")
 337 }
 338
 339 func (client *KeepTestClient) ClearBlockCache() {
 340 }
 341
 342 func (client *KeepTestClient) Close() {
 343         client.Content = nil
 344 }
 345
 346 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 347         client.StorageClasses = sc
 348 }
 349
 350 type FileWrapper struct {
 351         io.ReadCloser
 352         len int64
 353 }
 354
 355 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 356         return nil, errors.New("not implemented")
 357 }
 358
 359 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 360         return 0, errors.New("not implemented")
 361 }
 362
 363 func (fw FileWrapper) Size() int64 {
 364         return fw.len
 365 }
 366
 367 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 368         return nil, errors.New("not implemented")
 369 }
 370
 371 func (fw FileWrapper) Truncate(int64) error {
 372         return errors.New("not implemented")
 373 }
 374
 375 func (fw FileWrapper) Write([]byte) (int, error) {
 376         return 0, errors.New("not implemented")
 377 }
 378
 379 func (fw FileWrapper) Sync() error {
 380         return errors.New("not implemented")
 381 }
 382
 383 func (fw FileWrapper) Snapshot() (*arvados.Subtree, error) {
 384         return nil, errors.New("not implemented")
 385 }
 386
 387 func (fw FileWrapper) Splice(*arvados.Subtree) error {
 388         return errors.New("not implemented")
 389 }
 390
 391 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 392         if filename == hwImageID+".tar" {
 393                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 394                 client.Called = true
 395                 return FileWrapper{rdr, 1321984}, nil
 396         } else if filename == "/file1_in_main.txt" {
 397                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 398                 client.Called = true
 399                 return FileWrapper{rdr, 3}, nil
 400         }
 401         return nil, nil
 402 }
 403
 404 func (s *TestSuite) TestLoadImage(c *C) {
 405         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 406         s.runner.Container.Mounts = map[string]arvados.Mount{
 407                 "/out": {Kind: "tmp", Writable: true},
 408         }
 409         s.runner.Container.OutputPath = "/out"
 410
 411         _, err := s.runner.SetupMounts()
 412         c.Assert(err, IsNil)
 413
 414         imageID, err := s.runner.LoadImage()
 415         c.Check(err, IsNil)
 416         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 417         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 418
 419         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 420         s.executor.imageLoaded = false
 421         s.executor.loaded = ""
 422         s.executor.loadErr = errors.New("bork")
 423         imageID, err = s.runner.LoadImage()
 424         c.Check(err, ErrorMatches, ".*bork")
 425         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 426
 427         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 428         s.executor.imageLoaded = false
 429         s.executor.loaded = ""
 430         s.executor.loadErr = nil
 431         imageID, err = s.runner.LoadImage()
 432         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 433         c.Check(s.executor.loaded, Equals, "")
 434 }
 435
 436 type ArvErrorTestClient struct{}
 437
 438 func (ArvErrorTestClient) Create(resourceType string,
 439         parameters arvadosclient.Dict,
 440         output interface{}) error {
 441         return nil
 442 }
 443
 444 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 445         if method == "GET" && resourceType == "containers" && action == "auth" {
 446                 return nil
 447         }
 448         return errors.New("ArvError")
 449 }
 450
 451 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 452         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 453         return nil, errors.New("ArvError")
 454 }
 455
 456 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 457         return errors.New("ArvError")
 458 }
 459
 460 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 461         return nil
 462 }
 463
 464 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 465         return discoveryMap[key], nil
 466 }
 467
 468 type KeepErrorTestClient struct {
 469         KeepTestClient
 470 }
 471
 472 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 473         return nil, errors.New("KeepError")
 474 }
 475
 476 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 477         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 478 }
 479
 480 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 481         return "", errors.New("KeepError")
 482 }
 483
 484 type KeepReadErrorTestClient struct {
 485         KeepTestClient
 486 }
 487
 488 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 489         return 0, errors.New("KeepError")
 490 }
 491
 492 type ErrorReader struct {
 493         FileWrapper
 494 }
 495
 496 func (ErrorReader) Read(p []byte) (n int, err error) {
 497         return 0, errors.New("ErrorReader")
 498 }
 499
 500 func (ErrorReader) Seek(int64, int) (int64, error) {
 501         return 0, errors.New("ErrorReader")
 502 }
 503
 504 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 505         return ErrorReader{}, nil
 506 }
 507
 508 type ClosableBuffer struct {
 509         bytes.Buffer
 510 }
 511
 512 func (*ClosableBuffer) Close() error {
 513         return nil
 514 }
 515
 516 type TestLogs struct {
 517         Stdout ClosableBuffer
 518         Stderr ClosableBuffer
 519 }
 520
 521 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 522         if logstr == "stdout" {
 523                 return &tl.Stdout, nil
 524         }
 525         if logstr == "stderr" {
 526                 return &tl.Stderr, nil
 527         }
 528         return nil, errors.New("???")
 529 }
 530
 531 func dockerLog(fd byte, msg string) []byte {
 532         by := []byte(msg)
 533         header := make([]byte, 8+len(by))
 534         header[0] = fd
 535         header[7] = byte(len(by))
 536         copy(header[8:], by)
 537         return header
 538 }
 539
 540 func (s *TestSuite) TestRunContainer(c *C) {
 541         s.executor.runFunc = func() {
 542                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 543                 s.executor.exit <- 0
 544         }
 545
 546         var logs TestLogs
 547         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 548         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 549         s.runner.Container.Command = []string{"./hw"}
 550         s.runner.Container.OutputStorageClasses = []string{"default"}
 551
 552         imageID, err := s.runner.LoadImage()
 553         c.Assert(err, IsNil)
 554
 555         err = s.runner.CreateContainer(imageID, nil)
 556         c.Assert(err, IsNil)
 557
 558         err = s.runner.StartContainer()
 559         c.Assert(err, IsNil)
 560
 561         err = s.runner.WaitFinish()
 562         c.Assert(err, IsNil)
 563
 564         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 565         c.Check(logs.Stderr.String(), Equals, "")
 566 }
 567
 568 func (s *TestSuite) TestCommitLogs(c *C) {
 569         api := &ArvTestClient{}
 570         kc := &KeepTestClient{}
 571         defer kc.Close()
 572         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 573         c.Assert(err, IsNil)
 574         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 575
 576         cr.CrunchLog.Print("Hello world!")
 577         cr.CrunchLog.Print("Goodbye")
 578         cr.finalState = "Complete"
 579
 580         err = cr.CommitLogs()
 581         c.Check(err, IsNil)
 582
 583         c.Check(api.Calls, Equals, 2)
 584         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 585         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 586         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 587         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 588 }
 589
 590 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 591         api := &ArvTestClient{}
 592         kc := &KeepTestClient{}
 593         defer kc.Close()
 594         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 595         c.Assert(err, IsNil)
 596
 597         err = cr.UpdateContainerRunning()
 598         c.Check(err, IsNil)
 599
 600         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 601 }
 602
 603 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 604         api := &ArvTestClient{}
 605         kc := &KeepTestClient{}
 606         defer kc.Close()
 607         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 608         c.Assert(err, IsNil)
 609
 610         cr.LogsPDH = new(string)
 611         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 612
 613         cr.ExitCode = new(int)
 614         *cr.ExitCode = 42
 615         cr.finalState = "Complete"
 616
 617         err = cr.UpdateContainerFinal()
 618         c.Check(err, IsNil)
 619
 620         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 621         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 622         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 623 }
 624
 625 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 626         api := &ArvTestClient{}
 627         kc := &KeepTestClient{}
 628         defer kc.Close()
 629         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 630         c.Assert(err, IsNil)
 631         cr.cCancelled = true
 632         cr.finalState = "Cancelled"
 633
 634         err = cr.UpdateContainerFinal()
 635         c.Check(err, IsNil)
 636
 637         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 638         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 639         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 640 }
 641
 642 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 643 // dress rehearsal of the Run() function, starting from a JSON container record.
 644 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 645         err := json.Unmarshal([]byte(record), &s.api.Container)
 646         c.Assert(err, IsNil)
 647         initialState := s.api.Container.State
 648
 649         var sm struct {
 650                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 651         }
 652         err = json.Unmarshal([]byte(record), &sm)
 653         c.Check(err, IsNil)
 654         secretMounts, err := json.Marshal(sm)
 655         c.Assert(err, IsNil)
 656         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 657
 658         s.executor.runFunc = func() {
 659                 fn()
 660                 s.executor.exit <- exitCode
 661         }
 662
 663         s.runner.statInterval = 100 * time.Millisecond
 664         s.runner.containerWatchdogInterval = time.Second
 665
 666         realTemp := c.MkDir()
 667         tempcount := 0
 668         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 669                 tempcount++
 670                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 671                 err := os.Mkdir(d, os.ModePerm)
 672                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 673                         // Test case must have pre-populated the tempdir
 674                         err = nil
 675                 }
 676                 return d, err
 677         }
 678         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 679                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 680         }
 681
 682         if extraMounts != nil && len(extraMounts) > 0 {
 683                 err := s.runner.SetupArvMountPoint("keep")
 684                 c.Check(err, IsNil)
 685
 686                 for _, m := range extraMounts {
 687                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 688                 }
 689         }
 690
 691         err = s.runner.Run()
 692         if s.api.CalledWith("container.state", "Complete") != nil {
 693                 c.Check(err, IsNil)
 694         }
 695         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 696                 c.Check(s.api.WasSetRunning, Equals, true)
 697                 var lastupdate arvadosclient.Dict
 698                 for _, content := range s.api.Content {
 699                         if content["container"] != nil {
 700                                 lastupdate = content["container"].(arvadosclient.Dict)
 701                         }
 702                 }
 703                 if lastupdate["log"] == nil {
 704                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 705                 }
 706         }
 707
 708         if err != nil {
 709                 for k, v := range s.api.Logs {
 710                         c.Log(k)
 711                         c.Log(v.String())
 712                 }
 713         }
 714
 715         return s.api, s.runner, realTemp
 716 }
 717
 718 func (s *TestSuite) TestFullRunHello(c *C) {
 719         s.runner.enableMemoryLimit = true
 720         s.runner.networkMode = "default"
 721         s.fullRunHelper(c, `{
 722     "command": ["echo", "hello world"],
 723     "container_image": "`+arvadostest.DockerImage112PDH+`",
 724     "cwd": ".",
 725     "environment": {"foo":"bar","baz":"waz"},
 726     "mounts": {"/tmp": {"kind": "tmp"} },
 727     "output_path": "/tmp",
 728     "priority": 1,
 729     "runtime_constraints": {"vcpus":1,"ram":1000000},
 730     "state": "Locked",
 731     "output_storage_classes": ["default"]
 732 }`, nil, 0, func() {
 733                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 734                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 735                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 736                 c.Check(s.executor.created.VCPUs, Equals, 1)
 737                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 738                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 739                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 740                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
 741                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 742         })
 743
 744         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 745         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 746         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 747         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 748         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 749 }
 750
 751 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 752         var ran bool
 753         s.fullRunHelper(c, `{
 754     "command": ["sleep", "3"],
 755     "container_image": "`+arvadostest.DockerImage112PDH+`",
 756     "cwd": ".",
 757     "environment": {},
 758     "mounts": {"/tmp": {"kind": "tmp"} },
 759     "output_path": "/tmp",
 760     "priority": 1,
 761     "runtime_constraints": {},
 762     "scheduling_parameters":{"max_run_time": 1},
 763     "state": "Running"
 764 }`, nil, 2, func() {
 765                 ran = true
 766         })
 767         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 768         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 769         c.Check(ran, Equals, false)
 770 }
 771
 772 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 773         s.fullRunHelper(c, `{
 774     "command": ["sleep", "3"],
 775     "container_image": "`+arvadostest.DockerImage112PDH+`",
 776     "cwd": ".",
 777     "environment": {},
 778     "mounts": {"/tmp": {"kind": "tmp"} },
 779     "output_path": "/tmp",
 780     "priority": 1,
 781     "runtime_constraints": {},
 782     "scheduling_parameters":{"max_run_time": 1},
 783     "state": "Locked"
 784 }`, nil, 0, func() {
 785                 time.Sleep(3 * time.Second)
 786         })
 787
 788         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 789         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 790 }
 791
 792 func (s *TestSuite) TestContainerWaitFails(c *C) {
 793         s.fullRunHelper(c, `{
 794     "command": ["sleep", "3"],
 795     "container_image": "`+arvadostest.DockerImage112PDH+`",
 796     "cwd": ".",
 797     "mounts": {"/tmp": {"kind": "tmp"} },
 798     "output_path": "/tmp",
 799     "priority": 1,
 800     "state": "Locked"
 801 }`, nil, 0, func() {
 802                 s.executor.waitErr = errors.New("Container is not running")
 803         })
 804
 805         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 806         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 807 }
 808
 809 func (s *TestSuite) TestCrunchstat(c *C) {
 810         s.fullRunHelper(c, `{
 811                 "command": ["sleep", "1"],
 812                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 813                 "cwd": ".",
 814                 "environment": {},
 815                 "mounts": {"/tmp": {"kind": "tmp"} },
 816                 "output_path": "/tmp",
 817                 "priority": 1,
 818                 "runtime_constraints": {},
 819                 "state": "Locked"
 820         }`, nil, 0, func() {
 821                 time.Sleep(time.Second)
 822         })
 823
 824         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 825         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 826
 827         // We didn't actually start a container, so crunchstat didn't
 828         // find accounting files and therefore didn't log any stats.
 829         // It should have logged a "can't find accounting files"
 830         // message after one poll interval, though, so we can confirm
 831         // it's alive:
 832         c.Assert(s.api.Logs["crunchstat"], NotNil)
 833         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 834
 835         // The "files never appeared" log assures us that we called
 836         // (*crunchstat.Reporter)Stop(), and that we set it up with
 837         // the correct container ID "abcde":
 838         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 839 }
 840
 841 func (s *TestSuite) TestNodeInfoLog(c *C) {
 842         os.Setenv("SLURMD_NODENAME", "compute2")
 843         s.fullRunHelper(c, `{
 844                 "command": ["sleep", "1"],
 845                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 846                 "cwd": ".",
 847                 "environment": {},
 848                 "mounts": {"/tmp": {"kind": "tmp"} },
 849                 "output_path": "/tmp",
 850                 "priority": 1,
 851                 "runtime_constraints": {},
 852                 "state": "Locked"
 853         }`, nil, 0,
 854                 func() {
 855                         time.Sleep(time.Second)
 856                 })
 857
 858         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 859         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 860
 861         c.Assert(s.api.Logs["node"], NotNil)
 862         json := s.api.Logs["node"].String()
 863         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 864         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 865         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 866
 867         c.Assert(s.api.Logs["node-info"], NotNil)
 868         json = s.api.Logs["node-info"].String()
 869         c.Check(json, Matches, `(?ms).*Host Information.*`)
 870         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 871         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 872         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 873         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 874 }
 875
 876 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 877         s.fullRunHelper(c, `{
 878                 "command": ["sleep", "1"],
 879                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 880                 "cwd": ".",
 881                 "environment": {},
 882                 "mounts": {"/tmp": {"kind": "tmp"} },
 883                 "output_path": "/tmp",
 884                 "priority": 1,
 885                 "runtime_constraints": {},
 886                 "state": "Locked"
 887         }`, nil, 0,
 888                 func() {
 889                 })
 890
 891         c.Assert(s.api.Logs["crunch-run"], NotNil)
 892         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 893         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`)
 894         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container: zzzzz-zzzzz-zzzzzzzzzzzzzzz.*`)
 895         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Using container runtime: stub.*`)
 896 }
 897
 898 func (s *TestSuite) TestContainerRecordLog(c *C) {
 899         s.fullRunHelper(c, `{
 900                 "command": ["sleep", "1"],
 901                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 902                 "cwd": ".",
 903                 "environment": {},
 904                 "mounts": {"/tmp": {"kind": "tmp"} },
 905                 "output_path": "/tmp",
 906                 "priority": 1,
 907                 "runtime_constraints": {},
 908                 "state": "Locked"
 909         }`, nil, 0,
 910                 func() {
 911                         time.Sleep(time.Second)
 912                 })
 913
 914         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 915         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 916
 917         c.Assert(s.api.Logs["container"], NotNil)
 918         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 919 }
 920
 921 func (s *TestSuite) TestFullRunStderr(c *C) {
 922         s.fullRunHelper(c, `{
 923     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 924     "container_image": "`+arvadostest.DockerImage112PDH+`",
 925     "cwd": ".",
 926     "environment": {},
 927     "mounts": {"/tmp": {"kind": "tmp"} },
 928     "output_path": "/tmp",
 929     "priority": 1,
 930     "runtime_constraints": {},
 931     "state": "Locked"
 932 }`, nil, 1, func() {
 933                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 934                 fmt.Fprintln(s.executor.created.Stderr, "world")
 935         })
 936
 937         final := s.api.CalledWith("container.state", "Complete")
 938         c.Assert(final, NotNil)
 939         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 940         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 941
 942         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 943         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 944 }
 945
 946 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 947         s.fullRunHelper(c, `{
 948     "command": ["pwd"],
 949     "container_image": "`+arvadostest.DockerImage112PDH+`",
 950     "cwd": ".",
 951     "environment": {},
 952     "mounts": {"/tmp": {"kind": "tmp"} },
 953     "output_path": "/tmp",
 954     "priority": 1,
 955     "runtime_constraints": {},
 956     "state": "Locked"
 957 }`, nil, 0, func() {
 958                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 959         })
 960
 961         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 962         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 963         c.Log(s.api.Logs["stdout"])
 964         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 965 }
 966
 967 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 968         s.fullRunHelper(c, `{
 969     "command": ["pwd"],
 970     "container_image": "`+arvadostest.DockerImage112PDH+`",
 971     "cwd": "/bin",
 972     "environment": {},
 973     "mounts": {"/tmp": {"kind": "tmp"} },
 974     "output_path": "/tmp",
 975     "priority": 1,
 976     "runtime_constraints": {},
 977     "state": "Locked"
 978 }`, nil, 0, func() {
 979                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 980         })
 981
 982         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 983         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 984         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 985 }
 986
 987 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
 988         s.fullRunHelper(c, `{
 989     "command": ["pwd"],
 990     "container_image": "`+arvadostest.DockerImage112PDH+`",
 991     "cwd": "/bin",
 992     "environment": {},
 993     "mounts": {"/tmp": {"kind": "tmp"} },
 994     "output_path": "/tmp",
 995     "priority": 1,
 996     "runtime_constraints": {},
 997     "state": "Locked",
 998     "output_storage_classes": ["foo", "bar"]
 999 }`, nil, 0, func() {
1000                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1001         })
1002
1003         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1004         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1005         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1006         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1007         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1008 }
1009
1010 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
1011         s.fullRunHelper(c, `{
1012     "command": ["pwd"],
1013     "container_image": "`+arvadostest.DockerImage112PDH+`",
1014     "cwd": "/bin",
1015     "environment": {},
1016     "mounts": {"/tmp": {"kind": "tmp"} },
1017     "output_path": "/tmp",
1018     "priority": 1,
1019     "runtime_constraints": {"cuda": {"device_count": 2}},
1020     "state": "Locked",
1021     "output_storage_classes": ["foo", "bar"]
1022 }`, nil, 0, func() {
1023                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1024         })
1025         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1026 }
1027
1028 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1029         s.fullRunHelper(c, `{
1030     "command": ["pwd"],
1031     "container_image": "`+arvadostest.DockerImage112PDH+`",
1032     "cwd": "/bin",
1033     "environment": {},
1034     "mounts": {"/tmp": {"kind": "tmp"} },
1035     "output_path": "/tmp",
1036     "priority": 1,
1037     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1038     "state": "Locked",
1039     "output_storage_classes": ["foo", "bar"]
1040 }`, nil, 0, func() {
1041                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1042         })
1043         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1044 }
1045
1046 func (s *TestSuite) TestStopOnSignal(c *C) {
1047         s.executor.runFunc = func() {
1048                 s.executor.created.Stdout.Write([]byte("foo\n"))
1049                 s.runner.SigChan <- syscall.SIGINT
1050         }
1051         s.testStopContainer(c)
1052 }
1053
1054 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1055         s.executor.runFunc = func() {
1056                 s.executor.created.Stdout.Write([]byte("foo\n"))
1057                 s.runner.ArvMountExit <- nil
1058                 close(s.runner.ArvMountExit)
1059         }
1060         s.runner.ArvMountExit = make(chan error)
1061         s.testStopContainer(c)
1062 }
1063
1064 func (s *TestSuite) testStopContainer(c *C) {
1065         record := `{
1066     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1067     "container_image": "` + arvadostest.DockerImage112PDH + `",
1068     "cwd": ".",
1069     "environment": {},
1070     "mounts": {"/tmp": {"kind": "tmp"} },
1071     "output_path": "/tmp",
1072     "priority": 1,
1073     "runtime_constraints": {},
1074     "state": "Locked"
1075 }`
1076
1077         err := json.Unmarshal([]byte(record), &s.api.Container)
1078         c.Assert(err, IsNil)
1079
1080         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1081         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1082                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1083         }
1084
1085         done := make(chan error)
1086         go func() {
1087                 done <- s.runner.Run()
1088         }()
1089         select {
1090         case <-time.After(20 * time.Second):
1091                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1092                 c.Fatal("timed out")
1093         case err = <-done:
1094                 c.Check(err, IsNil)
1095         }
1096         for k, v := range s.api.Logs {
1097                 c.Log(k)
1098                 c.Log(v.String(), "\n")
1099         }
1100
1101         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1102         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1103         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1104 }
1105
1106 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1107         s.fullRunHelper(c, `{
1108     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1109     "container_image": "`+arvadostest.DockerImage112PDH+`",
1110     "cwd": "/bin",
1111     "environment": {"FROBIZ": "bilbo"},
1112     "mounts": {"/tmp": {"kind": "tmp"} },
1113     "output_path": "/tmp",
1114     "priority": 1,
1115     "runtime_constraints": {},
1116     "state": "Locked"
1117 }`, nil, 0, func() {
1118                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1119         })
1120
1121         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1122         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1123         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1124 }
1125
1126 type ArvMountCmdLine struct {
1127         Cmd   []string
1128         token string
1129 }
1130
1131 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1132         am.Cmd = c
1133         am.token = token
1134         return nil, nil
1135 }
1136
1137 func stubCert(temp string) string {
1138         path := temp + "/ca-certificates.crt"
1139         crt, _ := os.Create(path)
1140         crt.Close()
1141         arvadosclient.CertFiles = []string{path}
1142         return path
1143 }
1144
1145 func (s *TestSuite) TestSetupMounts(c *C) {
1146         cr := s.runner
1147         am := &ArvMountCmdLine{}
1148         cr.RunArvMount = am.ArvMountTest
1149         cr.ContainerArvClient = &ArvTestClient{}
1150         cr.ContainerKeepClient = &KeepTestClient{}
1151         cr.Container.OutputStorageClasses = []string{"default"}
1152
1153         realTemp := c.MkDir()
1154         certTemp := c.MkDir()
1155         stubCertPath := stubCert(certTemp)
1156         cr.parentTemp = realTemp
1157
1158         i := 0
1159         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1160                 i++
1161                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1162                 err := os.Mkdir(d, os.ModePerm)
1163                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1164                         // Test case must have pre-populated the tempdir
1165                         err = nil
1166                 }
1167                 return d, err
1168         }
1169
1170         checkEmpty := func() {
1171                 // Should be deleted.
1172                 _, err := os.Stat(realTemp)
1173                 c.Assert(os.IsNotExist(err), Equals, true)
1174
1175                 // Now recreate it for the next test.
1176                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1177         }
1178
1179         {
1180                 i = 0
1181                 cr.ArvMountPoint = ""
1182                 cr.Container.Mounts = make(map[string]arvados.Mount)
1183                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1184                 cr.Container.OutputPath = "/tmp"
1185                 cr.statInterval = 5 * time.Second
1186                 bindmounts, err := cr.SetupMounts()
1187                 c.Check(err, IsNil)
1188                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1189                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1190                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1191                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1192                 os.RemoveAll(cr.ArvMountPoint)
1193                 cr.CleanupDirs()
1194                 checkEmpty()
1195         }
1196
1197         {
1198                 i = 0
1199                 cr.ArvMountPoint = ""
1200                 cr.Container.Mounts = make(map[string]arvados.Mount)
1201                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1202                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1203                 cr.Container.OutputPath = "/out"
1204                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1205
1206                 bindmounts, err := cr.SetupMounts()
1207                 c.Check(err, IsNil)
1208                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1209                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1210                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1211                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1212                 os.RemoveAll(cr.ArvMountPoint)
1213                 cr.CleanupDirs()
1214                 checkEmpty()
1215         }
1216
1217         {
1218                 i = 0
1219                 cr.ArvMountPoint = ""
1220                 cr.Container.Mounts = make(map[string]arvados.Mount)
1221                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1222                 cr.Container.OutputPath = "/tmp"
1223                 cr.Container.RuntimeConstraints.API = true
1224                 cr.Container.OutputStorageClasses = []string{"default"}
1225
1226                 bindmounts, err := cr.SetupMounts()
1227                 c.Check(err, IsNil)
1228                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1229                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1230                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1231                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1232                 os.RemoveAll(cr.ArvMountPoint)
1233                 cr.CleanupDirs()
1234                 checkEmpty()
1235
1236                 cr.Container.RuntimeConstraints.API = false
1237         }
1238
1239         {
1240                 i = 0
1241                 cr.ArvMountPoint = ""
1242                 cr.Container.Mounts = map[string]arvados.Mount{
1243                         "/keeptmp": {Kind: "collection", Writable: true},
1244                 }
1245                 cr.Container.OutputPath = "/keeptmp"
1246
1247                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1248
1249                 bindmounts, err := cr.SetupMounts()
1250                 c.Check(err, IsNil)
1251                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1252                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1253                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1254                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1255                 os.RemoveAll(cr.ArvMountPoint)
1256                 cr.CleanupDirs()
1257                 checkEmpty()
1258         }
1259
1260         {
1261                 i = 0
1262                 cr.ArvMountPoint = ""
1263                 cr.Container.Mounts = map[string]arvados.Mount{
1264                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1265                         "/keepout": {Kind: "collection", Writable: true},
1266                 }
1267                 cr.Container.OutputPath = "/keepout"
1268
1269                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1270                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1271
1272                 bindmounts, err := cr.SetupMounts()
1273                 c.Check(err, IsNil)
1274                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1275                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1276                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1277                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1278                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1279                         "/keepout": {realTemp + "/keep1/tmp0", false},
1280                 })
1281                 os.RemoveAll(cr.ArvMountPoint)
1282                 cr.CleanupDirs()
1283                 checkEmpty()
1284         }
1285
1286         {
1287                 i = 0
1288                 cr.ArvMountPoint = ""
1289                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1290                 cr.Container.Mounts = map[string]arvados.Mount{
1291                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1292                         "/keepout": {Kind: "collection", Writable: true},
1293                 }
1294                 cr.Container.OutputPath = "/keepout"
1295
1296                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1297                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1298
1299                 bindmounts, err := cr.SetupMounts()
1300                 c.Check(err, IsNil)
1301                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1302                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1303                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1304                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1305                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1306                         "/keepout": {realTemp + "/keep1/tmp0", false},
1307                 })
1308                 os.RemoveAll(cr.ArvMountPoint)
1309                 cr.CleanupDirs()
1310                 checkEmpty()
1311         }
1312
1313         for _, test := range []struct {
1314                 in  interface{}
1315                 out string
1316         }{
1317                 {in: "foo", out: `"foo"`},
1318                 {in: nil, out: `null`},
1319                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1320         } {
1321                 i = 0
1322                 cr.ArvMountPoint = ""
1323                 cr.Container.Mounts = map[string]arvados.Mount{
1324                         "/mnt/test.json": {Kind: "json", Content: test.in},
1325                 }
1326                 bindmounts, err := cr.SetupMounts()
1327                 c.Check(err, IsNil)
1328                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1329                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1330                 })
1331                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1332                 c.Check(err, IsNil)
1333                 c.Check(content, DeepEquals, []byte(test.out))
1334                 os.RemoveAll(cr.ArvMountPoint)
1335                 cr.CleanupDirs()
1336                 checkEmpty()
1337         }
1338
1339         for _, test := range []struct {
1340                 in  interface{}
1341                 out string
1342         }{
1343                 {in: "foo", out: `foo`},
1344                 {in: nil, out: "error"},
1345                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1346         } {
1347                 i = 0
1348                 cr.ArvMountPoint = ""
1349                 cr.Container.Mounts = map[string]arvados.Mount{
1350                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1351                 }
1352                 bindmounts, err := cr.SetupMounts()
1353                 if test.out == "error" {
1354                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1355                 } else {
1356                         c.Check(err, IsNil)
1357                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1358                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1359                         })
1360                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1361                         c.Check(err, IsNil)
1362                         c.Check(content, DeepEquals, []byte(test.out))
1363                 }
1364                 os.RemoveAll(cr.ArvMountPoint)
1365                 cr.CleanupDirs()
1366                 checkEmpty()
1367         }
1368
1369         // Read-only mount points are allowed underneath output_dir mount point
1370         {
1371                 i = 0
1372                 cr.ArvMountPoint = ""
1373                 cr.Container.Mounts = make(map[string]arvados.Mount)
1374                 cr.Container.Mounts = map[string]arvados.Mount{
1375                         "/tmp":     {Kind: "tmp"},
1376                         "/tmp/foo": {Kind: "collection"},
1377                 }
1378                 cr.Container.OutputPath = "/tmp"
1379
1380                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1381
1382                 bindmounts, err := cr.SetupMounts()
1383                 c.Check(err, IsNil)
1384                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1385                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1386                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1387                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1388                         "/tmp":     {realTemp + "/tmp2", false},
1389                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1390                 })
1391                 os.RemoveAll(cr.ArvMountPoint)
1392                 cr.CleanupDirs()
1393                 checkEmpty()
1394         }
1395
1396         // Writable mount points copied to output_dir mount point
1397         {
1398                 i = 0
1399                 cr.ArvMountPoint = ""
1400                 cr.Container.Mounts = make(map[string]arvados.Mount)
1401                 cr.Container.Mounts = map[string]arvados.Mount{
1402                         "/tmp": {Kind: "tmp"},
1403                         "/tmp/foo": {Kind: "collection",
1404                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1405                                 Writable:         true},
1406                         "/tmp/bar": {Kind: "collection",
1407                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1408                                 Path:             "baz",
1409                                 Writable:         true},
1410                 }
1411                 cr.Container.OutputPath = "/tmp"
1412
1413                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1414                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1415
1416                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1417                 rf.Write([]byte("bar"))
1418                 rf.Close()
1419
1420                 _, err := cr.SetupMounts()
1421                 c.Check(err, IsNil)
1422                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1423                 c.Check(err, IsNil)
1424                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1425                 c.Check(err, IsNil)
1426                 os.RemoveAll(cr.ArvMountPoint)
1427                 cr.CleanupDirs()
1428                 checkEmpty()
1429         }
1430
1431         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1432         {
1433                 i = 0
1434                 cr.ArvMountPoint = ""
1435                 cr.Container.Mounts = make(map[string]arvados.Mount)
1436                 cr.Container.Mounts = map[string]arvados.Mount{
1437                         "/tmp":     {Kind: "tmp"},
1438                         "/tmp/foo": {Kind: "tmp"},
1439                 }
1440                 cr.Container.OutputPath = "/tmp"
1441
1442                 _, err := cr.SetupMounts()
1443                 c.Check(err, NotNil)
1444                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1445                 os.RemoveAll(cr.ArvMountPoint)
1446                 cr.CleanupDirs()
1447                 checkEmpty()
1448         }
1449
1450         // Only mount point of kind 'collection' is allowed for stdin
1451         {
1452                 i = 0
1453                 cr.ArvMountPoint = ""
1454                 cr.Container.Mounts = make(map[string]arvados.Mount)
1455                 cr.Container.Mounts = map[string]arvados.Mount{
1456                         "stdin": {Kind: "tmp"},
1457                 }
1458
1459                 _, err := cr.SetupMounts()
1460                 c.Check(err, NotNil)
1461                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1462                 os.RemoveAll(cr.ArvMountPoint)
1463                 cr.CleanupDirs()
1464                 checkEmpty()
1465         }
1466
1467         // git_tree mounts
1468         {
1469                 i = 0
1470                 cr.ArvMountPoint = ""
1471                 (*GitMountSuite)(nil).useTestGitServer(c)
1472                 cr.token = arvadostest.ActiveToken
1473                 cr.Container.Mounts = make(map[string]arvados.Mount)
1474                 cr.Container.Mounts = map[string]arvados.Mount{
1475                         "/tip": {
1476                                 Kind:   "git_tree",
1477                                 UUID:   arvadostest.Repository2UUID,
1478                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1479                                 Path:   "/",
1480                         },
1481                         "/non-tip": {
1482                                 Kind:   "git_tree",
1483                                 UUID:   arvadostest.Repository2UUID,
1484                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1485                                 Path:   "/",
1486                         },
1487                 }
1488                 cr.Container.OutputPath = "/tmp"
1489
1490                 bindmounts, err := cr.SetupMounts()
1491                 c.Check(err, IsNil)
1492
1493                 for path, mount := range bindmounts {
1494                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1495                 }
1496
1497                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1498                 c.Check(err, IsNil)
1499                 c.Check(string(data), Equals, "\000\001\002\003")
1500                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1501                 c.Check(err, FitsTypeOf, &os.PathError{})
1502                 c.Check(os.IsNotExist(err), Equals, true)
1503
1504                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1505                 c.Check(err, IsNil)
1506                 c.Check(string(data), Equals, "\000\001\002\003")
1507                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1508                 c.Check(err, IsNil)
1509                 c.Check(string(data), Equals, "testfile\n")
1510
1511                 cr.CleanupDirs()
1512                 checkEmpty()
1513         }
1514 }
1515
1516 func (s *TestSuite) TestStdout(c *C) {
1517         helperRecord := `{
1518                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1519                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1520                 "cwd": "/bin",
1521                 "environment": {"FROBIZ": "bilbo"},
1522                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1523                 "output_path": "/tmp",
1524                 "priority": 1,
1525                 "runtime_constraints": {},
1526                 "state": "Locked"
1527         }`
1528
1529         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1530                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1531         })
1532
1533         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1534         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1535         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1536 }
1537
1538 // Used by the TestStdoutWithWrongPath*()
1539 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1540         err := json.Unmarshal([]byte(record), &s.api.Container)
1541         c.Assert(err, IsNil)
1542         s.executor.runFunc = fn
1543         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1544         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1545                 return s.api, &KeepTestClient{}, nil, nil
1546         }
1547         return s.api, s.runner, s.runner.Run()
1548 }
1549
1550 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1551         _, _, err := s.stdoutErrorRunHelper(c, `{
1552     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1553     "output_path": "/tmp",
1554     "state": "Locked"
1555 }`, func() {})
1556         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1557 }
1558
1559 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1560         _, _, err := s.stdoutErrorRunHelper(c, `{
1561     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1562     "output_path": "/tmp",
1563     "state": "Locked"
1564 }`, func() {})
1565         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1566 }
1567
1568 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1569         _, _, err := s.stdoutErrorRunHelper(c, `{
1570     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1571     "output_path": "/tmp",
1572     "state": "Locked"
1573 }`, func() {})
1574         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1575 }
1576
1577 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1578         s.fullRunHelper(c, `{
1579     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1580     "container_image": "`+arvadostest.DockerImage112PDH+`",
1581     "cwd": "/bin",
1582     "environment": {},
1583     "mounts": {"/tmp": {"kind": "tmp"} },
1584     "output_path": "/tmp",
1585     "priority": 1,
1586     "runtime_constraints": {"API": true},
1587     "state": "Locked"
1588 }`, nil, 0, func() {
1589                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1590                 s.executor.exit <- 3
1591         })
1592         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1593         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1594         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1595 }
1596
1597 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1598         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1599         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1600         s.fullRunHelper(c, `{
1601     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1602     "container_image": "`+arvadostest.DockerImage112PDH+`",
1603     "cwd": "/bin",
1604     "environment": {},
1605     "mounts": {"/tmp": {"kind": "tmp"} },
1606     "output_path": "/tmp",
1607     "priority": 1,
1608     "runtime_constraints": {"API": true},
1609     "state": "Locked"
1610 }`, nil, 0, func() {
1611                 s.api.Container.Output = arvadostest.DockerImage112PDH
1612         })
1613
1614         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1615         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1616         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1617 }
1618
1619 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1620         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1621                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1622                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1623                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1624         }
1625         s.executor.runFunc = func() {
1626                 time.Sleep(time.Second)
1627                 s.executor.exit <- 137
1628         }
1629         record := `{
1630     "command": ["sleep", "1"],
1631     "container_image": "` + arvadostest.DockerImage112PDH + `",
1632     "cwd": "/bin",
1633     "environment": {},
1634     "mounts": {"/tmp": {"kind": "tmp"} },
1635     "output_path": "/tmp",
1636     "priority": 1,
1637     "runtime_constraints": {"API": true},
1638     "state": "Locked"
1639 }`
1640         err := json.Unmarshal([]byte(record), &s.api.Container)
1641         c.Assert(err, IsNil)
1642         err = s.runner.Run()
1643         c.Assert(err, IsNil)
1644         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1645         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1646         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1647         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1648         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1649 }
1650
1651 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1652         helperRecord := `{
1653                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1654                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1655                 "cwd": "/bin",
1656                 "environment": {"FROBIZ": "bilbo"},
1657                 "mounts": {
1658         "/tmp": {"kind": "tmp"},
1659         "/tmp/foo": {"kind": "collection",
1660                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1661                      "exclude_from_output": true
1662         },
1663         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1664     },
1665                 "output_path": "/tmp",
1666                 "priority": 1,
1667                 "runtime_constraints": {},
1668                 "state": "Locked"
1669         }`
1670
1671         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1672
1673         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1674                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1675         })
1676
1677         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1678         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1679         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1680 }
1681
1682 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1683         helperRecord := `{
1684                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1685                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1686                 "cwd": "/bin",
1687                 "environment": {"FROBIZ": "bilbo"},
1688                 "mounts": {
1689         "/tmp": {"kind": "tmp"},
1690         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1691         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1692         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1693         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1694         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1695     },
1696                 "output_path": "/tmp",
1697                 "priority": 1,
1698                 "runtime_constraints": {},
1699                 "state": "Locked"
1700         }`
1701
1702         extraMounts := []string{
1703                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1704                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1705                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1706         }
1707
1708         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1709                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1710         })
1711
1712         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1713                 "/tmp":                   {realtemp + "/tmp1", false},
1714                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1715                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1716                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1717                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1718         })
1719
1720         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1721         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1722         for _, v := range api.Content {
1723                 if v["collection"] != nil {
1724                         c.Check(v["ensure_unique_name"], Equals, true)
1725                         collection := v["collection"].(arvadosclient.Dict)
1726                         if strings.Index(collection["name"].(string), "output") == 0 {
1727                                 manifest := collection["manifest_text"].(string)
1728
1729                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1730 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1731 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1732 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1733 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1734 `)
1735                         }
1736                 }
1737         }
1738 }
1739
1740 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1741         helperRecord := `{
1742                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1743                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1744                 "cwd": "/bin",
1745                 "environment": {"FROBIZ": "bilbo"},
1746                 "mounts": {
1747         "/tmp": {"kind": "tmp"},
1748         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1749         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1750     },
1751                 "output_path": "/tmp",
1752                 "priority": 1,
1753                 "runtime_constraints": {},
1754                 "state": "Locked"
1755         }`
1756
1757         extraMounts := []string{
1758                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1759         }
1760
1761         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1762                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1763         })
1764
1765         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1766         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1767         for _, v := range s.api.Content {
1768                 if v["collection"] != nil {
1769                         collection := v["collection"].(arvadosclient.Dict)
1770                         if strings.Index(collection["name"].(string), "output") == 0 {
1771                                 manifest := collection["manifest_text"].(string)
1772
1773                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1774 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1775 `)
1776                         }
1777                 }
1778         }
1779 }
1780
1781 func (s *TestSuite) TestOutputError(c *C) {
1782         helperRecord := `{
1783                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1784                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1785                 "cwd": "/bin",
1786                 "environment": {"FROBIZ": "bilbo"},
1787                 "mounts": {
1788                         "/tmp": {"kind": "tmp"}
1789                 },
1790                 "output_path": "/tmp",
1791                 "priority": 1,
1792                 "runtime_constraints": {},
1793                 "state": "Locked"
1794         }`
1795         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1796                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1797         })
1798
1799         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1800 }
1801
1802 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1803         helperRecord := `{
1804                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1805                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1806                 "cwd": "/bin",
1807                 "environment": {"FROBIZ": "bilbo"},
1808                 "mounts": {
1809         "/tmp": {"kind": "tmp"},
1810         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1811         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1812     },
1813                 "output_path": "/tmp",
1814                 "priority": 1,
1815                 "runtime_constraints": {},
1816                 "state": "Locked"
1817         }`
1818
1819         extraMounts := []string{
1820                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1821         }
1822
1823         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1824                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1825         })
1826
1827         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1828         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1829         for _, v := range api.Content {
1830                 if v["collection"] != nil {
1831                         collection := v["collection"].(arvadosclient.Dict)
1832                         if strings.Index(collection["name"].(string), "output") == 0 {
1833                                 manifest := collection["manifest_text"].(string)
1834                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1835 `)
1836                         }
1837                 }
1838         }
1839 }
1840
1841 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1842         helperRecord := `{
1843                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1844                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1845                 "cwd": "/bin",
1846                 "environment": {"FROBIZ": "bilbo"},
1847                 "mounts": {
1848         "/tmp": {"kind": "tmp"},
1849         "stdin": {"kind": "json", "content": "foo"},
1850         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1851     },
1852                 "output_path": "/tmp",
1853                 "priority": 1,
1854                 "runtime_constraints": {},
1855                 "state": "Locked"
1856         }`
1857
1858         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1859                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1860         })
1861
1862         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1863         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1864         for _, v := range api.Content {
1865                 if v["collection"] != nil {
1866                         collection := v["collection"].(arvadosclient.Dict)
1867                         if strings.Index(collection["name"].(string), "output") == 0 {
1868                                 manifest := collection["manifest_text"].(string)
1869                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1870 `)
1871                         }
1872                 }
1873         }
1874 }
1875
1876 func (s *TestSuite) TestStderrMount(c *C) {
1877         api, cr, _ := s.fullRunHelper(c, `{
1878     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1879     "container_image": "`+arvadostest.DockerImage112PDH+`",
1880     "cwd": ".",
1881     "environment": {},
1882     "mounts": {"/tmp": {"kind": "tmp"},
1883                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1884                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1885     "output_path": "/tmp",
1886     "priority": 1,
1887     "runtime_constraints": {},
1888     "state": "Locked"
1889 }`, nil, 1, func() {
1890                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1891                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1892         })
1893
1894         final := api.CalledWith("container.state", "Complete")
1895         c.Assert(final, NotNil)
1896         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1897         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1898
1899         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1900 }
1901
1902 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1903         s.api.callraw = true
1904         err := s.runner.fetchContainerRecord()
1905         c.Assert(err, IsNil)
1906         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1907         c.Logf("%#v", s.runner.Container)
1908         c.Check(err, IsNil)
1909         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1910 }
1911
1912 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1913         nextState := ""
1914         for _, setup := range []func(){
1915                 func() {
1916                         c.Log("// waitErr = ocl runtime error")
1917                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1918                         nextState = "Cancelled"
1919                 },
1920                 func() {
1921                         c.Log("// loadErr = cannot connect")
1922                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1923                         s.runner.brokenNodeHook = c.MkDir() + "/broken-node-hook"
1924                         err := ioutil.WriteFile(s.runner.brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1925                         c.Assert(err, IsNil)
1926                         nextState = "Queued"
1927                 },
1928         } {
1929                 s.SetUpTest(c)
1930                 setup()
1931                 s.fullRunHelper(c, `{
1932     "command": ["echo", "hello world"],
1933     "container_image": "`+arvadostest.DockerImage112PDH+`",
1934     "cwd": ".",
1935     "environment": {},
1936     "mounts": {"/tmp": {"kind": "tmp"} },
1937     "output_path": "/tmp",
1938     "priority": 1,
1939     "runtime_constraints": {},
1940     "state": "Locked"
1941 }`, nil, 0, func() {})
1942                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1943                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1944                 if s.runner.brokenNodeHook != "" {
1945                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1946                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1947                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1948                 } else {
1949                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1950                 }
1951         }
1952 }
1953
1954 func (s *TestSuite) TestBadCommand(c *C) {
1955         for _, startError := range []string{
1956                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1957                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1958                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1959         } {
1960                 s.SetUpTest(c)
1961                 s.executor.startErr = errors.New(startError)
1962                 s.fullRunHelper(c, `{
1963     "command": ["echo", "hello world"],
1964     "container_image": "`+arvadostest.DockerImage112PDH+`",
1965     "cwd": ".",
1966     "environment": {},
1967     "mounts": {"/tmp": {"kind": "tmp"} },
1968     "output_path": "/tmp",
1969     "priority": 1,
1970     "runtime_constraints": {},
1971     "state": "Locked"
1972 }`, nil, 0, func() {})
1973                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1974                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1975         }
1976 }
1977
1978 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1979         helperRecord := `{
1980                 "command": ["true"],
1981                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1982                 "cwd": "/bin",
1983                 "mounts": {
1984                     "/tmp": {"kind": "tmp"},
1985                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1986                 },
1987                 "secret_mounts": {
1988                 },
1989                 "output_path": "/tmp",
1990                 "priority": 1,
1991                 "runtime_constraints": {},
1992                 "state": "Locked"
1993         }`
1994
1995         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1996                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1997                 c.Check(err, IsNil)
1998                 c.Check(string(content), Equals, "mypassword")
1999         })
2000
2001         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2002         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2003         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
2004         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
2005
2006         // under secret mounts, not captured in output
2007         helperRecord = `{
2008                 "command": ["true"],
2009                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2010                 "cwd": "/bin",
2011                 "mounts": {
2012                     "/tmp": {"kind": "tmp"}
2013                 },
2014                 "secret_mounts": {
2015                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2016                 },
2017                 "output_path": "/tmp",
2018                 "priority": 1,
2019                 "runtime_constraints": {},
2020                 "state": "Locked"
2021         }`
2022
2023         s.SetUpTest(c)
2024         s.fullRunHelper(c, helperRecord, nil, 0, func() {
2025                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2026                 c.Check(err, IsNil)
2027                 c.Check(string(content), Equals, "mypassword")
2028         })
2029
2030         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2031         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2032         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2033         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2034
2035         // under secret mounts, output dir is a collection, not captured in output
2036         helperRecord = `{
2037                 "command": ["true"],
2038                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2039                 "cwd": "/bin",
2040                 "mounts": {
2041                     "/tmp": {"kind": "collection", "writable": true}
2042                 },
2043                 "secret_mounts": {
2044                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2045                 },
2046                 "output_path": "/tmp",
2047                 "priority": 1,
2048                 "runtime_constraints": {},
2049                 "state": "Locked"
2050         }`
2051
2052         s.SetUpTest(c)
2053         _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, 0, func() {
2054                 // secret.conf should be provisioned as a separate
2055                 // bind mount, i.e., it should not appear in the
2056                 // (fake) fuse filesystem as viewed from the host.
2057                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2058                 if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) {
2059                         c.Logf("secret.conf: content %q, err %#v", content, err)
2060                 }
2061                 err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700)
2062                 c.Check(err, IsNil)
2063         })
2064
2065         content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text")
2066         c.Check(err, IsNil)
2067         c.Check(string(content), Equals, "mypassword")
2068         c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true})
2069         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2070         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2071         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil)
2072 }
2073
2074 type FakeProcess struct {
2075         cmdLine []string
2076 }
2077
2078 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2079         return fp.cmdLine, nil
2080 }