lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/manifest"
  29         "golang.org/x/net/context"
  30
  31         . "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func TestCrunchExec(t *testing.T) {
  36         TestingT(t)
  37 }
  38
  39 var _ = Suite(&TestSuite{})
  40
  41 type TestSuite struct {
  42         client                   *arvados.Client
  43         api                      *ArvTestClient
  44         runner                   *ContainerRunner
  45         executor                 *stubExecutor
  46         keepmount                string
  47         keepmountTmp             []string
  48         testDispatcherKeepClient KeepTestClient
  49         testContainerKeepClient  KeepTestClient
  50 }
  51
  52 func (s *TestSuite) SetUpTest(c *C) {
  53         *brokenNodeHook = ""
  54         s.client = arvados.NewClientFromEnv()
  55         s.executor = &stubExecutor{}
  56         var err error
  57         s.api = &ArvTestClient{}
  58         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  59         c.Assert(err, IsNil)
  60         s.runner.executor = s.executor
  61         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  62                 return s.api, &s.testContainerKeepClient, s.client, nil
  63         }
  64         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  65                 s.runner.ArvMountPoint = s.keepmount
  66                 for i, opt := range cmd {
  67                         if opt == "--mount-tmp" {
  68                                 err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700)
  69                                 if err != nil {
  70                                         return nil, err
  71                                 }
  72                                 s.keepmountTmp = append(s.keepmountTmp, cmd[i+1])
  73                         }
  74                 }
  75                 return nil, nil
  76         }
  77         s.keepmount = c.MkDir()
  78         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  79         s.keepmountTmp = nil
  80         c.Assert(err, IsNil)
  81         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  82         c.Assert(err, IsNil)
  83         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  84         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  85         c.Assert(err, IsNil)
  86         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  87         c.Assert(err, IsNil)
  88         s.runner.ArvMountPoint = s.keepmount
  89 }
  90
  91 type ArvTestClient struct {
  92         Total   int64
  93         Calls   int
  94         Content []arvadosclient.Dict
  95         arvados.Container
  96         secretMounts []byte
  97         Logs         map[string]*bytes.Buffer
  98         sync.Mutex
  99         WasSetRunning bool
 100         callraw       bool
 101 }
 102
 103 type KeepTestClient struct {
 104         Called         bool
 105         Content        []byte
 106         StorageClasses []string
 107 }
 108
 109 type stubExecutor struct {
 110         imageLoaded bool
 111         loaded      string
 112         loadErr     error
 113         exitCode    int
 114         createErr   error
 115         created     containerSpec
 116         startErr    error
 117         waitSleep   time.Duration
 118         waitErr     error
 119         stopErr     error
 120         stopped     bool
 121         closed      bool
 122         runFunc     func()
 123         exit        chan int
 124 }
 125
 126 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 127         containerClient *arvados.Client) error {
 128         e.loaded = tarball
 129         return e.loadErr
 130 }
 131 func (e *stubExecutor) Runtime() string                 { return "stub" }
 132 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 133 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 134 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 135 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 136 func (e *stubExecutor) Close()                          { e.closed = true }
 137 func (e *stubExecutor) Wait(context.Context) (int, error) {
 138         return <-e.exit, e.waitErr
 139 }
 140
 141 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 142
 143 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 144 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 145 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 146
 147 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 148 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 149
 150 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 151 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 152 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 153 `
 154
 155 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 156
 157 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 158 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 159
 160 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 161 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 162
 163 func (client *ArvTestClient) Create(resourceType string,
 164         parameters arvadosclient.Dict,
 165         output interface{}) error {
 166
 167         client.Mutex.Lock()
 168         defer client.Mutex.Unlock()
 169
 170         client.Calls++
 171         client.Content = append(client.Content, parameters)
 172
 173         if resourceType == "logs" {
 174                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 175                 if client.Logs == nil {
 176                         client.Logs = make(map[string]*bytes.Buffer)
 177                 }
 178                 if client.Logs[et] == nil {
 179                         client.Logs[et] = &bytes.Buffer{}
 180                 }
 181                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 182         }
 183
 184         if resourceType == "collections" && output != nil {
 185                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 186                 outmap := output.(*arvados.Collection)
 187                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 188                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 189         }
 190
 191         return nil
 192 }
 193
 194 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 195         switch {
 196         case method == "GET" && resourceType == "containers" && action == "auth":
 197                 return json.Unmarshal([]byte(`{
 198                         "kind": "arvados#api_client_authorization",
 199                         "uuid": "`+fakeAuthUUID+`",
 200                         "api_token": "`+fakeAuthToken+`"
 201                         }`), output)
 202         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 203                 if client.secretMounts != nil {
 204                         return json.Unmarshal(client.secretMounts, output)
 205                 }
 206                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 207         default:
 208                 return fmt.Errorf("Not found")
 209         }
 210 }
 211
 212 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 213         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 214         var j []byte
 215         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 216                 j = []byte(`{
 217                         "kind": "arvados#nodeList",
 218                         "items": [{
 219                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 220                                 "hostname": "compute2",
 221                                 "properties": {"total_cpu_cores": 16}
 222                         }]}`)
 223         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 224                 if uuid == "" {
 225                         j, err = json.Marshal(map[string]interface{}{
 226                                 "items": []interface{}{client.Container},
 227                                 "kind":  "arvados#nodeList",
 228                         })
 229                 } else {
 230                         j, err = json.Marshal(client.Container)
 231                 }
 232         } else {
 233                 j = []byte(`{
 234                         "command": ["sleep", "1"],
 235                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 236                         "cwd": ".",
 237                         "environment": {},
 238                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 239                         "output_path": "/tmp",
 240                         "priority": 1,
 241                         "runtime_constraints": {}
 242                 }`)
 243         }
 244         return ioutil.NopCloser(bytes.NewReader(j)), err
 245 }
 246
 247 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 248         if resourceType == "collections" {
 249                 if uuid == hwPDH {
 250                         output.(*arvados.Collection).ManifestText = hwManifest
 251                 } else if uuid == otherPDH {
 252                         output.(*arvados.Collection).ManifestText = otherManifest
 253                 } else if uuid == normalizedWithSubdirsPDH {
 254                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 255                 } else if uuid == denormalizedWithSubdirsPDH {
 256                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 257                 }
 258         }
 259         if resourceType == "containers" {
 260                 (*output.(*arvados.Container)) = client.Container
 261         }
 262         return nil
 263 }
 264
 265 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 266         client.Mutex.Lock()
 267         defer client.Mutex.Unlock()
 268         client.Calls++
 269         client.Content = append(client.Content, parameters)
 270         if resourceType == "containers" {
 271                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 272                         client.WasSetRunning = true
 273                 }
 274         } else if resourceType == "collections" {
 275                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 276                 output.(*arvados.Collection).UUID = uuid
 277                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 278         }
 279         return nil
 280 }
 281
 282 var discoveryMap = map[string]interface{}{
 283         "defaultTrashLifetime":               float64(1209600),
 284         "crunchLimitLogBytesPerJob":          float64(67108864),
 285         "crunchLogThrottleBytes":             float64(65536),
 286         "crunchLogThrottlePeriod":            float64(60),
 287         "crunchLogThrottleLines":             float64(1024),
 288         "crunchLogPartialLineThrottlePeriod": float64(5),
 289         "crunchLogBytesPerEvent":             float64(4096),
 290         "crunchLogSecondsBetweenEvents":      float64(1),
 291 }
 292
 293 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 294         return discoveryMap[key], nil
 295 }
 296
 297 // CalledWith returns the parameters from the first API call whose
 298 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 299 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 300 // no call matches, it returns nil.
 301 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 302 call:
 303         for _, content := range client.Content {
 304                 var v interface{} = content
 305                 for _, k := range strings.Split(jpath, ".") {
 306                         if dict, ok := v.(arvadosclient.Dict); !ok {
 307                                 continue call
 308                         } else {
 309                                 v = dict[k]
 310                         }
 311                 }
 312                 if v == expect {
 313                         return content
 314                 }
 315         }
 316         return nil
 317 }
 318
 319 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 320         return locator, nil
 321 }
 322
 323 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 324         client.Content = opts.Data
 325         return arvados.BlockWriteResponse{
 326                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 327         }, nil
 328 }
 329
 330 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 331         return 0, errors.New("not implemented")
 332 }
 333
 334 func (client *KeepTestClient) ClearBlockCache() {
 335 }
 336
 337 func (client *KeepTestClient) Close() {
 338         client.Content = nil
 339 }
 340
 341 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 342         client.StorageClasses = sc
 343 }
 344
 345 type FileWrapper struct {
 346         io.ReadCloser
 347         len int64
 348 }
 349
 350 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 351         return nil, errors.New("not implemented")
 352 }
 353
 354 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 355         return 0, errors.New("not implemented")
 356 }
 357
 358 func (fw FileWrapper) Size() int64 {
 359         return fw.len
 360 }
 361
 362 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 363         return nil, errors.New("not implemented")
 364 }
 365
 366 func (fw FileWrapper) Truncate(int64) error {
 367         return errors.New("not implemented")
 368 }
 369
 370 func (fw FileWrapper) Write([]byte) (int, error) {
 371         return 0, errors.New("not implemented")
 372 }
 373
 374 func (fw FileWrapper) Sync() error {
 375         return errors.New("not implemented")
 376 }
 377
 378 func (fw FileWrapper) Snapshot() (*arvados.Subtree, error) {
 379         return nil, errors.New("not implemented")
 380 }
 381
 382 func (fw FileWrapper) Splice(*arvados.Subtree) error {
 383         return errors.New("not implemented")
 384 }
 385
 386 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 387         if filename == hwImageID+".tar" {
 388                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 389                 client.Called = true
 390                 return FileWrapper{rdr, 1321984}, nil
 391         } else if filename == "/file1_in_main.txt" {
 392                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 393                 client.Called = true
 394                 return FileWrapper{rdr, 3}, nil
 395         }
 396         return nil, nil
 397 }
 398
 399 func (s *TestSuite) TestLoadImage(c *C) {
 400         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 401         s.runner.Container.Mounts = map[string]arvados.Mount{
 402                 "/out": {Kind: "tmp", Writable: true},
 403         }
 404         s.runner.Container.OutputPath = "/out"
 405
 406         _, err := s.runner.SetupMounts()
 407         c.Assert(err, IsNil)
 408
 409         imageID, err := s.runner.LoadImage()
 410         c.Check(err, IsNil)
 411         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 412         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 413
 414         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 415         s.executor.imageLoaded = false
 416         s.executor.loaded = ""
 417         s.executor.loadErr = errors.New("bork")
 418         imageID, err = s.runner.LoadImage()
 419         c.Check(err, ErrorMatches, ".*bork")
 420         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 421
 422         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 423         s.executor.imageLoaded = false
 424         s.executor.loaded = ""
 425         s.executor.loadErr = nil
 426         imageID, err = s.runner.LoadImage()
 427         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 428         c.Check(s.executor.loaded, Equals, "")
 429 }
 430
 431 type ArvErrorTestClient struct{}
 432
 433 func (ArvErrorTestClient) Create(resourceType string,
 434         parameters arvadosclient.Dict,
 435         output interface{}) error {
 436         return nil
 437 }
 438
 439 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 440         if method == "GET" && resourceType == "containers" && action == "auth" {
 441                 return nil
 442         }
 443         return errors.New("ArvError")
 444 }
 445
 446 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 447         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 448         return nil, errors.New("ArvError")
 449 }
 450
 451 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 452         return errors.New("ArvError")
 453 }
 454
 455 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 456         return nil
 457 }
 458
 459 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 460         return discoveryMap[key], nil
 461 }
 462
 463 type KeepErrorTestClient struct {
 464         KeepTestClient
 465 }
 466
 467 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 468         return nil, errors.New("KeepError")
 469 }
 470
 471 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 472         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 473 }
 474
 475 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 476         return "", errors.New("KeepError")
 477 }
 478
 479 type KeepReadErrorTestClient struct {
 480         KeepTestClient
 481 }
 482
 483 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 484         return 0, errors.New("KeepError")
 485 }
 486
 487 type ErrorReader struct {
 488         FileWrapper
 489 }
 490
 491 func (ErrorReader) Read(p []byte) (n int, err error) {
 492         return 0, errors.New("ErrorReader")
 493 }
 494
 495 func (ErrorReader) Seek(int64, int) (int64, error) {
 496         return 0, errors.New("ErrorReader")
 497 }
 498
 499 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 500         return ErrorReader{}, nil
 501 }
 502
 503 type ClosableBuffer struct {
 504         bytes.Buffer
 505 }
 506
 507 func (*ClosableBuffer) Close() error {
 508         return nil
 509 }
 510
 511 type TestLogs struct {
 512         Stdout ClosableBuffer
 513         Stderr ClosableBuffer
 514 }
 515
 516 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 517         if logstr == "stdout" {
 518                 return &tl.Stdout, nil
 519         }
 520         if logstr == "stderr" {
 521                 return &tl.Stderr, nil
 522         }
 523         return nil, errors.New("???")
 524 }
 525
 526 func dockerLog(fd byte, msg string) []byte {
 527         by := []byte(msg)
 528         header := make([]byte, 8+len(by))
 529         header[0] = fd
 530         header[7] = byte(len(by))
 531         copy(header[8:], by)
 532         return header
 533 }
 534
 535 func (s *TestSuite) TestRunContainer(c *C) {
 536         s.executor.runFunc = func() {
 537                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 538                 s.executor.exit <- 0
 539         }
 540
 541         var logs TestLogs
 542         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 543         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 544         s.runner.Container.Command = []string{"./hw"}
 545         s.runner.Container.OutputStorageClasses = []string{"default"}
 546
 547         imageID, err := s.runner.LoadImage()
 548         c.Assert(err, IsNil)
 549
 550         err = s.runner.CreateContainer(imageID, nil)
 551         c.Assert(err, IsNil)
 552
 553         err = s.runner.StartContainer()
 554         c.Assert(err, IsNil)
 555
 556         err = s.runner.WaitFinish()
 557         c.Assert(err, IsNil)
 558
 559         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 560         c.Check(logs.Stderr.String(), Equals, "")
 561 }
 562
 563 func (s *TestSuite) TestCommitLogs(c *C) {
 564         api := &ArvTestClient{}
 565         kc := &KeepTestClient{}
 566         defer kc.Close()
 567         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 568         c.Assert(err, IsNil)
 569         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 570
 571         cr.CrunchLog.Print("Hello world!")
 572         cr.CrunchLog.Print("Goodbye")
 573         cr.finalState = "Complete"
 574
 575         err = cr.CommitLogs()
 576         c.Check(err, IsNil)
 577
 578         c.Check(api.Calls, Equals, 2)
 579         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 580         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 581         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 582         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 583 }
 584
 585 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 586         api := &ArvTestClient{}
 587         kc := &KeepTestClient{}
 588         defer kc.Close()
 589         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 590         c.Assert(err, IsNil)
 591
 592         err = cr.UpdateContainerRunning()
 593         c.Check(err, IsNil)
 594
 595         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 596 }
 597
 598 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 599         api := &ArvTestClient{}
 600         kc := &KeepTestClient{}
 601         defer kc.Close()
 602         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 603         c.Assert(err, IsNil)
 604
 605         cr.LogsPDH = new(string)
 606         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 607
 608         cr.ExitCode = new(int)
 609         *cr.ExitCode = 42
 610         cr.finalState = "Complete"
 611
 612         err = cr.UpdateContainerFinal()
 613         c.Check(err, IsNil)
 614
 615         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 616         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 617         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 618 }
 619
 620 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 621         api := &ArvTestClient{}
 622         kc := &KeepTestClient{}
 623         defer kc.Close()
 624         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 625         c.Assert(err, IsNil)
 626         cr.cCancelled = true
 627         cr.finalState = "Cancelled"
 628
 629         err = cr.UpdateContainerFinal()
 630         c.Check(err, IsNil)
 631
 632         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 633         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 634         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 635 }
 636
 637 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 638 // dress rehearsal of the Run() function, starting from a JSON container record.
 639 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 640         err := json.Unmarshal([]byte(record), &s.api.Container)
 641         c.Assert(err, IsNil)
 642         initialState := s.api.Container.State
 643
 644         var sm struct {
 645                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 646         }
 647         err = json.Unmarshal([]byte(record), &sm)
 648         c.Check(err, IsNil)
 649         secretMounts, err := json.Marshal(sm)
 650         c.Assert(err, IsNil)
 651         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 652
 653         s.executor.runFunc = func() {
 654                 fn()
 655                 s.executor.exit <- exitCode
 656         }
 657
 658         s.runner.statInterval = 100 * time.Millisecond
 659         s.runner.containerWatchdogInterval = time.Second
 660
 661         realTemp := c.MkDir()
 662         tempcount := 0
 663         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 664                 tempcount++
 665                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 666                 err := os.Mkdir(d, os.ModePerm)
 667                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 668                         // Test case must have pre-populated the tempdir
 669                         err = nil
 670                 }
 671                 return d, err
 672         }
 673         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 674                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 675         }
 676
 677         if extraMounts != nil && len(extraMounts) > 0 {
 678                 err := s.runner.SetupArvMountPoint("keep")
 679                 c.Check(err, IsNil)
 680
 681                 for _, m := range extraMounts {
 682                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 683                 }
 684         }
 685
 686         err = s.runner.Run()
 687         if s.api.CalledWith("container.state", "Complete") != nil {
 688                 c.Check(err, IsNil)
 689         }
 690         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 691                 c.Check(s.api.WasSetRunning, Equals, true)
 692                 var lastupdate arvadosclient.Dict
 693                 for _, content := range s.api.Content {
 694                         if content["container"] != nil {
 695                                 lastupdate = content["container"].(arvadosclient.Dict)
 696                         }
 697                 }
 698                 if lastupdate["log"] == nil {
 699                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 700                 }
 701         }
 702
 703         if err != nil {
 704                 for k, v := range s.api.Logs {
 705                         c.Log(k)
 706                         c.Log(v.String())
 707                 }
 708         }
 709
 710         return s.api, s.runner, realTemp
 711 }
 712
 713 func (s *TestSuite) TestFullRunHello(c *C) {
 714         s.runner.enableMemoryLimit = true
 715         s.runner.networkMode = "default"
 716         s.fullRunHelper(c, `{
 717     "command": ["echo", "hello world"],
 718     "container_image": "`+arvadostest.DockerImage112PDH+`",
 719     "cwd": ".",
 720     "environment": {"foo":"bar","baz":"waz"},
 721     "mounts": {"/tmp": {"kind": "tmp"} },
 722     "output_path": "/tmp",
 723     "priority": 1,
 724     "runtime_constraints": {"vcpus":1,"ram":1000000},
 725     "state": "Locked",
 726     "output_storage_classes": ["default"]
 727 }`, nil, 0, func() {
 728                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 729                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 730                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 731                 c.Check(s.executor.created.VCPUs, Equals, 1)
 732                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 733                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 734                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 735                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
 736                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 737         })
 738
 739         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 740         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 741         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 742         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 743         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 744 }
 745
 746 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 747         var ran bool
 748         s.fullRunHelper(c, `{
 749     "command": ["sleep", "3"],
 750     "container_image": "`+arvadostest.DockerImage112PDH+`",
 751     "cwd": ".",
 752     "environment": {},
 753     "mounts": {"/tmp": {"kind": "tmp"} },
 754     "output_path": "/tmp",
 755     "priority": 1,
 756     "runtime_constraints": {},
 757     "scheduling_parameters":{"max_run_time": 1},
 758     "state": "Running"
 759 }`, nil, 2, func() {
 760                 ran = true
 761         })
 762         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 763         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 764         c.Check(ran, Equals, false)
 765 }
 766
 767 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 768         s.fullRunHelper(c, `{
 769     "command": ["sleep", "3"],
 770     "container_image": "`+arvadostest.DockerImage112PDH+`",
 771     "cwd": ".",
 772     "environment": {},
 773     "mounts": {"/tmp": {"kind": "tmp"} },
 774     "output_path": "/tmp",
 775     "priority": 1,
 776     "runtime_constraints": {},
 777     "scheduling_parameters":{"max_run_time": 1},
 778     "state": "Locked"
 779 }`, nil, 0, func() {
 780                 time.Sleep(3 * time.Second)
 781         })
 782
 783         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 784         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 785 }
 786
 787 func (s *TestSuite) TestContainerWaitFails(c *C) {
 788         s.fullRunHelper(c, `{
 789     "command": ["sleep", "3"],
 790     "container_image": "`+arvadostest.DockerImage112PDH+`",
 791     "cwd": ".",
 792     "mounts": {"/tmp": {"kind": "tmp"} },
 793     "output_path": "/tmp",
 794     "priority": 1,
 795     "state": "Locked"
 796 }`, nil, 0, func() {
 797                 s.executor.waitErr = errors.New("Container is not running")
 798         })
 799
 800         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 801         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 802 }
 803
 804 func (s *TestSuite) TestCrunchstat(c *C) {
 805         s.fullRunHelper(c, `{
 806                 "command": ["sleep", "1"],
 807                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 808                 "cwd": ".",
 809                 "environment": {},
 810                 "mounts": {"/tmp": {"kind": "tmp"} },
 811                 "output_path": "/tmp",
 812                 "priority": 1,
 813                 "runtime_constraints": {},
 814                 "state": "Locked"
 815         }`, nil, 0, func() {
 816                 time.Sleep(time.Second)
 817         })
 818
 819         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 820         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 821
 822         // We didn't actually start a container, so crunchstat didn't
 823         // find accounting files and therefore didn't log any stats.
 824         // It should have logged a "can't find accounting files"
 825         // message after one poll interval, though, so we can confirm
 826         // it's alive:
 827         c.Assert(s.api.Logs["crunchstat"], NotNil)
 828         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 829
 830         // The "files never appeared" log assures us that we called
 831         // (*crunchstat.Reporter)Stop(), and that we set it up with
 832         // the correct container ID "abcde":
 833         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 834 }
 835
 836 func (s *TestSuite) TestNodeInfoLog(c *C) {
 837         os.Setenv("SLURMD_NODENAME", "compute2")
 838         s.fullRunHelper(c, `{
 839                 "command": ["sleep", "1"],
 840                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 841                 "cwd": ".",
 842                 "environment": {},
 843                 "mounts": {"/tmp": {"kind": "tmp"} },
 844                 "output_path": "/tmp",
 845                 "priority": 1,
 846                 "runtime_constraints": {},
 847                 "state": "Locked"
 848         }`, nil, 0,
 849                 func() {
 850                         time.Sleep(time.Second)
 851                 })
 852
 853         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 854         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 855
 856         c.Assert(s.api.Logs["node"], NotNil)
 857         json := s.api.Logs["node"].String()
 858         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 859         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 860         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 861
 862         c.Assert(s.api.Logs["node-info"], NotNil)
 863         json = s.api.Logs["node-info"].String()
 864         c.Check(json, Matches, `(?ms).*Host Information.*`)
 865         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 866         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 867         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 868         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 869 }
 870
 871 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 872         s.fullRunHelper(c, `{
 873                 "command": ["sleep", "1"],
 874                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 875                 "cwd": ".",
 876                 "environment": {},
 877                 "mounts": {"/tmp": {"kind": "tmp"} },
 878                 "output_path": "/tmp",
 879                 "priority": 1,
 880                 "runtime_constraints": {},
 881                 "state": "Locked"
 882         }`, nil, 0,
 883                 func() {
 884                 })
 885
 886         c.Assert(s.api.Logs["crunch-run"], NotNil)
 887         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 888         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`)
 889         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' using stub runtime.*`)
 890 }
 891
 892 func (s *TestSuite) TestContainerRecordLog(c *C) {
 893         s.fullRunHelper(c, `{
 894                 "command": ["sleep", "1"],
 895                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 896                 "cwd": ".",
 897                 "environment": {},
 898                 "mounts": {"/tmp": {"kind": "tmp"} },
 899                 "output_path": "/tmp",
 900                 "priority": 1,
 901                 "runtime_constraints": {},
 902                 "state": "Locked"
 903         }`, nil, 0,
 904                 func() {
 905                         time.Sleep(time.Second)
 906                 })
 907
 908         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 909         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 910
 911         c.Assert(s.api.Logs["container"], NotNil)
 912         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 913 }
 914
 915 func (s *TestSuite) TestFullRunStderr(c *C) {
 916         s.fullRunHelper(c, `{
 917     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 918     "container_image": "`+arvadostest.DockerImage112PDH+`",
 919     "cwd": ".",
 920     "environment": {},
 921     "mounts": {"/tmp": {"kind": "tmp"} },
 922     "output_path": "/tmp",
 923     "priority": 1,
 924     "runtime_constraints": {},
 925     "state": "Locked"
 926 }`, nil, 1, func() {
 927                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 928                 fmt.Fprintln(s.executor.created.Stderr, "world")
 929         })
 930
 931         final := s.api.CalledWith("container.state", "Complete")
 932         c.Assert(final, NotNil)
 933         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 934         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 935
 936         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 937         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 938 }
 939
 940 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 941         s.fullRunHelper(c, `{
 942     "command": ["pwd"],
 943     "container_image": "`+arvadostest.DockerImage112PDH+`",
 944     "cwd": ".",
 945     "environment": {},
 946     "mounts": {"/tmp": {"kind": "tmp"} },
 947     "output_path": "/tmp",
 948     "priority": 1,
 949     "runtime_constraints": {},
 950     "state": "Locked"
 951 }`, nil, 0, func() {
 952                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 953         })
 954
 955         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 956         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 957         c.Log(s.api.Logs["stdout"])
 958         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 959 }
 960
 961 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 962         s.fullRunHelper(c, `{
 963     "command": ["pwd"],
 964     "container_image": "`+arvadostest.DockerImage112PDH+`",
 965     "cwd": "/bin",
 966     "environment": {},
 967     "mounts": {"/tmp": {"kind": "tmp"} },
 968     "output_path": "/tmp",
 969     "priority": 1,
 970     "runtime_constraints": {},
 971     "state": "Locked"
 972 }`, nil, 0, func() {
 973                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 974         })
 975
 976         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 977         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 978         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 979 }
 980
 981 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
 982         s.fullRunHelper(c, `{
 983     "command": ["pwd"],
 984     "container_image": "`+arvadostest.DockerImage112PDH+`",
 985     "cwd": "/bin",
 986     "environment": {},
 987     "mounts": {"/tmp": {"kind": "tmp"} },
 988     "output_path": "/tmp",
 989     "priority": 1,
 990     "runtime_constraints": {},
 991     "state": "Locked",
 992     "output_storage_classes": ["foo", "bar"]
 993 }`, nil, 0, func() {
 994                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 995         })
 996
 997         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 998         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 999         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1000         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1001         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1002 }
1003
1004 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
1005         s.fullRunHelper(c, `{
1006     "command": ["pwd"],
1007     "container_image": "`+arvadostest.DockerImage112PDH+`",
1008     "cwd": "/bin",
1009     "environment": {},
1010     "mounts": {"/tmp": {"kind": "tmp"} },
1011     "output_path": "/tmp",
1012     "priority": 1,
1013     "runtime_constraints": {"cuda": {"device_count": 2}},
1014     "state": "Locked",
1015     "output_storage_classes": ["foo", "bar"]
1016 }`, nil, 0, func() {
1017                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1018         })
1019         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1020 }
1021
1022 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1023         s.fullRunHelper(c, `{
1024     "command": ["pwd"],
1025     "container_image": "`+arvadostest.DockerImage112PDH+`",
1026     "cwd": "/bin",
1027     "environment": {},
1028     "mounts": {"/tmp": {"kind": "tmp"} },
1029     "output_path": "/tmp",
1030     "priority": 1,
1031     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1032     "state": "Locked",
1033     "output_storage_classes": ["foo", "bar"]
1034 }`, nil, 0, func() {
1035                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1036         })
1037         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1038 }
1039
1040 func (s *TestSuite) TestStopOnSignal(c *C) {
1041         s.executor.runFunc = func() {
1042                 s.executor.created.Stdout.Write([]byte("foo\n"))
1043                 s.runner.SigChan <- syscall.SIGINT
1044         }
1045         s.testStopContainer(c)
1046 }
1047
1048 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1049         s.executor.runFunc = func() {
1050                 s.executor.created.Stdout.Write([]byte("foo\n"))
1051                 s.runner.ArvMountExit <- nil
1052                 close(s.runner.ArvMountExit)
1053         }
1054         s.runner.ArvMountExit = make(chan error)
1055         s.testStopContainer(c)
1056 }
1057
1058 func (s *TestSuite) testStopContainer(c *C) {
1059         record := `{
1060     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1061     "container_image": "` + arvadostest.DockerImage112PDH + `",
1062     "cwd": ".",
1063     "environment": {},
1064     "mounts": {"/tmp": {"kind": "tmp"} },
1065     "output_path": "/tmp",
1066     "priority": 1,
1067     "runtime_constraints": {},
1068     "state": "Locked"
1069 }`
1070
1071         err := json.Unmarshal([]byte(record), &s.api.Container)
1072         c.Assert(err, IsNil)
1073
1074         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1075         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1076                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1077         }
1078
1079         done := make(chan error)
1080         go func() {
1081                 done <- s.runner.Run()
1082         }()
1083         select {
1084         case <-time.After(20 * time.Second):
1085                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1086                 c.Fatal("timed out")
1087         case err = <-done:
1088                 c.Check(err, IsNil)
1089         }
1090         for k, v := range s.api.Logs {
1091                 c.Log(k)
1092                 c.Log(v.String(), "\n")
1093         }
1094
1095         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1096         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1097         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1098 }
1099
1100 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1101         s.fullRunHelper(c, `{
1102     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1103     "container_image": "`+arvadostest.DockerImage112PDH+`",
1104     "cwd": "/bin",
1105     "environment": {"FROBIZ": "bilbo"},
1106     "mounts": {"/tmp": {"kind": "tmp"} },
1107     "output_path": "/tmp",
1108     "priority": 1,
1109     "runtime_constraints": {},
1110     "state": "Locked"
1111 }`, nil, 0, func() {
1112                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1113         })
1114
1115         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1116         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1117         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1118 }
1119
1120 type ArvMountCmdLine struct {
1121         Cmd   []string
1122         token string
1123 }
1124
1125 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1126         am.Cmd = c
1127         am.token = token
1128         return nil, nil
1129 }
1130
1131 func stubCert(temp string) string {
1132         path := temp + "/ca-certificates.crt"
1133         crt, _ := os.Create(path)
1134         crt.Close()
1135         arvadosclient.CertFiles = []string{path}
1136         return path
1137 }
1138
1139 func (s *TestSuite) TestSetupMounts(c *C) {
1140         cr := s.runner
1141         am := &ArvMountCmdLine{}
1142         cr.RunArvMount = am.ArvMountTest
1143         cr.ContainerArvClient = &ArvTestClient{}
1144         cr.ContainerKeepClient = &KeepTestClient{}
1145         cr.Container.OutputStorageClasses = []string{"default"}
1146
1147         realTemp := c.MkDir()
1148         certTemp := c.MkDir()
1149         stubCertPath := stubCert(certTemp)
1150         cr.parentTemp = realTemp
1151
1152         i := 0
1153         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1154                 i++
1155                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1156                 err := os.Mkdir(d, os.ModePerm)
1157                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1158                         // Test case must have pre-populated the tempdir
1159                         err = nil
1160                 }
1161                 return d, err
1162         }
1163
1164         checkEmpty := func() {
1165                 // Should be deleted.
1166                 _, err := os.Stat(realTemp)
1167                 c.Assert(os.IsNotExist(err), Equals, true)
1168
1169                 // Now recreate it for the next test.
1170                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1171         }
1172
1173         {
1174                 i = 0
1175                 cr.ArvMountPoint = ""
1176                 cr.Container.Mounts = make(map[string]arvados.Mount)
1177                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1178                 cr.Container.OutputPath = "/tmp"
1179                 cr.statInterval = 5 * time.Second
1180                 bindmounts, err := cr.SetupMounts()
1181                 c.Check(err, IsNil)
1182                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1183                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1184                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1185                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1186                 os.RemoveAll(cr.ArvMountPoint)
1187                 cr.CleanupDirs()
1188                 checkEmpty()
1189         }
1190
1191         {
1192                 i = 0
1193                 cr.ArvMountPoint = ""
1194                 cr.Container.Mounts = make(map[string]arvados.Mount)
1195                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1196                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1197                 cr.Container.OutputPath = "/out"
1198                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1199
1200                 bindmounts, err := cr.SetupMounts()
1201                 c.Check(err, IsNil)
1202                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1203                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1204                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1205                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1206                 os.RemoveAll(cr.ArvMountPoint)
1207                 cr.CleanupDirs()
1208                 checkEmpty()
1209         }
1210
1211         {
1212                 i = 0
1213                 cr.ArvMountPoint = ""
1214                 cr.Container.Mounts = make(map[string]arvados.Mount)
1215                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1216                 cr.Container.OutputPath = "/tmp"
1217                 cr.Container.RuntimeConstraints.API = true
1218                 cr.Container.OutputStorageClasses = []string{"default"}
1219
1220                 bindmounts, err := cr.SetupMounts()
1221                 c.Check(err, IsNil)
1222                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1223                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1224                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1225                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1226                 os.RemoveAll(cr.ArvMountPoint)
1227                 cr.CleanupDirs()
1228                 checkEmpty()
1229
1230                 cr.Container.RuntimeConstraints.API = false
1231         }
1232
1233         {
1234                 i = 0
1235                 cr.ArvMountPoint = ""
1236                 cr.Container.Mounts = map[string]arvados.Mount{
1237                         "/keeptmp": {Kind: "collection", Writable: true},
1238                 }
1239                 cr.Container.OutputPath = "/keeptmp"
1240
1241                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1242
1243                 bindmounts, err := cr.SetupMounts()
1244                 c.Check(err, IsNil)
1245                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1246                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1247                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1248                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1249                 os.RemoveAll(cr.ArvMountPoint)
1250                 cr.CleanupDirs()
1251                 checkEmpty()
1252         }
1253
1254         {
1255                 i = 0
1256                 cr.ArvMountPoint = ""
1257                 cr.Container.Mounts = map[string]arvados.Mount{
1258                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1259                         "/keepout": {Kind: "collection", Writable: true},
1260                 }
1261                 cr.Container.OutputPath = "/keepout"
1262
1263                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1264                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1265
1266                 bindmounts, err := cr.SetupMounts()
1267                 c.Check(err, IsNil)
1268                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1269                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1270                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1271                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1272                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1273                         "/keepout": {realTemp + "/keep1/tmp0", false},
1274                 })
1275                 os.RemoveAll(cr.ArvMountPoint)
1276                 cr.CleanupDirs()
1277                 checkEmpty()
1278         }
1279
1280         {
1281                 i = 0
1282                 cr.ArvMountPoint = ""
1283                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1284                 cr.Container.Mounts = map[string]arvados.Mount{
1285                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1286                         "/keepout": {Kind: "collection", Writable: true},
1287                 }
1288                 cr.Container.OutputPath = "/keepout"
1289
1290                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1291                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1292
1293                 bindmounts, err := cr.SetupMounts()
1294                 c.Check(err, IsNil)
1295                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1296                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1297                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1298                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1299                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1300                         "/keepout": {realTemp + "/keep1/tmp0", false},
1301                 })
1302                 os.RemoveAll(cr.ArvMountPoint)
1303                 cr.CleanupDirs()
1304                 checkEmpty()
1305         }
1306
1307         for _, test := range []struct {
1308                 in  interface{}
1309                 out string
1310         }{
1311                 {in: "foo", out: `"foo"`},
1312                 {in: nil, out: `null`},
1313                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1314         } {
1315                 i = 0
1316                 cr.ArvMountPoint = ""
1317                 cr.Container.Mounts = map[string]arvados.Mount{
1318                         "/mnt/test.json": {Kind: "json", Content: test.in},
1319                 }
1320                 bindmounts, err := cr.SetupMounts()
1321                 c.Check(err, IsNil)
1322                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1323                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1324                 })
1325                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1326                 c.Check(err, IsNil)
1327                 c.Check(content, DeepEquals, []byte(test.out))
1328                 os.RemoveAll(cr.ArvMountPoint)
1329                 cr.CleanupDirs()
1330                 checkEmpty()
1331         }
1332
1333         for _, test := range []struct {
1334                 in  interface{}
1335                 out string
1336         }{
1337                 {in: "foo", out: `foo`},
1338                 {in: nil, out: "error"},
1339                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1340         } {
1341                 i = 0
1342                 cr.ArvMountPoint = ""
1343                 cr.Container.Mounts = map[string]arvados.Mount{
1344                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1345                 }
1346                 bindmounts, err := cr.SetupMounts()
1347                 if test.out == "error" {
1348                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1349                 } else {
1350                         c.Check(err, IsNil)
1351                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1352                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1353                         })
1354                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1355                         c.Check(err, IsNil)
1356                         c.Check(content, DeepEquals, []byte(test.out))
1357                 }
1358                 os.RemoveAll(cr.ArvMountPoint)
1359                 cr.CleanupDirs()
1360                 checkEmpty()
1361         }
1362
1363         // Read-only mount points are allowed underneath output_dir mount point
1364         {
1365                 i = 0
1366                 cr.ArvMountPoint = ""
1367                 cr.Container.Mounts = make(map[string]arvados.Mount)
1368                 cr.Container.Mounts = map[string]arvados.Mount{
1369                         "/tmp":     {Kind: "tmp"},
1370                         "/tmp/foo": {Kind: "collection"},
1371                 }
1372                 cr.Container.OutputPath = "/tmp"
1373
1374                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1375
1376                 bindmounts, err := cr.SetupMounts()
1377                 c.Check(err, IsNil)
1378                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1379                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1380                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1381                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1382                         "/tmp":     {realTemp + "/tmp2", false},
1383                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1384                 })
1385                 os.RemoveAll(cr.ArvMountPoint)
1386                 cr.CleanupDirs()
1387                 checkEmpty()
1388         }
1389
1390         // Writable mount points copied to output_dir mount point
1391         {
1392                 i = 0
1393                 cr.ArvMountPoint = ""
1394                 cr.Container.Mounts = make(map[string]arvados.Mount)
1395                 cr.Container.Mounts = map[string]arvados.Mount{
1396                         "/tmp": {Kind: "tmp"},
1397                         "/tmp/foo": {Kind: "collection",
1398                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1399                                 Writable:         true},
1400                         "/tmp/bar": {Kind: "collection",
1401                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1402                                 Path:             "baz",
1403                                 Writable:         true},
1404                 }
1405                 cr.Container.OutputPath = "/tmp"
1406
1407                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1408                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1409
1410                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1411                 rf.Write([]byte("bar"))
1412                 rf.Close()
1413
1414                 _, err := cr.SetupMounts()
1415                 c.Check(err, IsNil)
1416                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1417                 c.Check(err, IsNil)
1418                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1419                 c.Check(err, IsNil)
1420                 os.RemoveAll(cr.ArvMountPoint)
1421                 cr.CleanupDirs()
1422                 checkEmpty()
1423         }
1424
1425         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1426         {
1427                 i = 0
1428                 cr.ArvMountPoint = ""
1429                 cr.Container.Mounts = make(map[string]arvados.Mount)
1430                 cr.Container.Mounts = map[string]arvados.Mount{
1431                         "/tmp":     {Kind: "tmp"},
1432                         "/tmp/foo": {Kind: "tmp"},
1433                 }
1434                 cr.Container.OutputPath = "/tmp"
1435
1436                 _, err := cr.SetupMounts()
1437                 c.Check(err, NotNil)
1438                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1439                 os.RemoveAll(cr.ArvMountPoint)
1440                 cr.CleanupDirs()
1441                 checkEmpty()
1442         }
1443
1444         // Only mount point of kind 'collection' is allowed for stdin
1445         {
1446                 i = 0
1447                 cr.ArvMountPoint = ""
1448                 cr.Container.Mounts = make(map[string]arvados.Mount)
1449                 cr.Container.Mounts = map[string]arvados.Mount{
1450                         "stdin": {Kind: "tmp"},
1451                 }
1452
1453                 _, err := cr.SetupMounts()
1454                 c.Check(err, NotNil)
1455                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1456                 os.RemoveAll(cr.ArvMountPoint)
1457                 cr.CleanupDirs()
1458                 checkEmpty()
1459         }
1460
1461         // git_tree mounts
1462         {
1463                 i = 0
1464                 cr.ArvMountPoint = ""
1465                 (*GitMountSuite)(nil).useTestGitServer(c)
1466                 cr.token = arvadostest.ActiveToken
1467                 cr.Container.Mounts = make(map[string]arvados.Mount)
1468                 cr.Container.Mounts = map[string]arvados.Mount{
1469                         "/tip": {
1470                                 Kind:   "git_tree",
1471                                 UUID:   arvadostest.Repository2UUID,
1472                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1473                                 Path:   "/",
1474                         },
1475                         "/non-tip": {
1476                                 Kind:   "git_tree",
1477                                 UUID:   arvadostest.Repository2UUID,
1478                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1479                                 Path:   "/",
1480                         },
1481                 }
1482                 cr.Container.OutputPath = "/tmp"
1483
1484                 bindmounts, err := cr.SetupMounts()
1485                 c.Check(err, IsNil)
1486
1487                 for path, mount := range bindmounts {
1488                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1489                 }
1490
1491                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1492                 c.Check(err, IsNil)
1493                 c.Check(string(data), Equals, "\000\001\002\003")
1494                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1495                 c.Check(err, FitsTypeOf, &os.PathError{})
1496                 c.Check(os.IsNotExist(err), Equals, true)
1497
1498                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1499                 c.Check(err, IsNil)
1500                 c.Check(string(data), Equals, "\000\001\002\003")
1501                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1502                 c.Check(err, IsNil)
1503                 c.Check(string(data), Equals, "testfile\n")
1504
1505                 cr.CleanupDirs()
1506                 checkEmpty()
1507         }
1508 }
1509
1510 func (s *TestSuite) TestStdout(c *C) {
1511         helperRecord := `{
1512                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1513                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1514                 "cwd": "/bin",
1515                 "environment": {"FROBIZ": "bilbo"},
1516                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1517                 "output_path": "/tmp",
1518                 "priority": 1,
1519                 "runtime_constraints": {},
1520                 "state": "Locked"
1521         }`
1522
1523         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1524                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1525         })
1526
1527         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1528         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1529         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1530 }
1531
1532 // Used by the TestStdoutWithWrongPath*()
1533 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1534         err := json.Unmarshal([]byte(record), &s.api.Container)
1535         c.Assert(err, IsNil)
1536         s.executor.runFunc = fn
1537         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1538         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1539                 return s.api, &KeepTestClient{}, nil, nil
1540         }
1541         return s.api, s.runner, s.runner.Run()
1542 }
1543
1544 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1545         _, _, err := s.stdoutErrorRunHelper(c, `{
1546     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1547     "output_path": "/tmp",
1548     "state": "Locked"
1549 }`, func() {})
1550         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1551 }
1552
1553 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1554         _, _, err := s.stdoutErrorRunHelper(c, `{
1555     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1556     "output_path": "/tmp",
1557     "state": "Locked"
1558 }`, func() {})
1559         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1560 }
1561
1562 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1563         _, _, err := s.stdoutErrorRunHelper(c, `{
1564     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1565     "output_path": "/tmp",
1566     "state": "Locked"
1567 }`, func() {})
1568         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1569 }
1570
1571 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1572         s.fullRunHelper(c, `{
1573     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1574     "container_image": "`+arvadostest.DockerImage112PDH+`",
1575     "cwd": "/bin",
1576     "environment": {},
1577     "mounts": {"/tmp": {"kind": "tmp"} },
1578     "output_path": "/tmp",
1579     "priority": 1,
1580     "runtime_constraints": {"API": true},
1581     "state": "Locked"
1582 }`, nil, 0, func() {
1583                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1584                 s.executor.exit <- 3
1585         })
1586         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1587         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1588         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1589 }
1590
1591 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1592         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1593         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1594         s.fullRunHelper(c, `{
1595     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1596     "container_image": "`+arvadostest.DockerImage112PDH+`",
1597     "cwd": "/bin",
1598     "environment": {},
1599     "mounts": {"/tmp": {"kind": "tmp"} },
1600     "output_path": "/tmp",
1601     "priority": 1,
1602     "runtime_constraints": {"API": true},
1603     "state": "Locked"
1604 }`, nil, 0, func() {
1605                 s.api.Container.Output = arvadostest.DockerImage112PDH
1606         })
1607
1608         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1609         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1610         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1611 }
1612
1613 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1614         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1615                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1616                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1617                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1618         }
1619         s.executor.runFunc = func() {
1620                 time.Sleep(time.Second)
1621                 s.executor.exit <- 137
1622         }
1623         record := `{
1624     "command": ["sleep", "1"],
1625     "container_image": "` + arvadostest.DockerImage112PDH + `",
1626     "cwd": "/bin",
1627     "environment": {},
1628     "mounts": {"/tmp": {"kind": "tmp"} },
1629     "output_path": "/tmp",
1630     "priority": 1,
1631     "runtime_constraints": {"API": true},
1632     "state": "Locked"
1633 }`
1634         err := json.Unmarshal([]byte(record), &s.api.Container)
1635         c.Assert(err, IsNil)
1636         err = s.runner.Run()
1637         c.Assert(err, IsNil)
1638         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1639         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1640         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1641         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1642         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1643 }
1644
1645 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1646         helperRecord := `{
1647                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1648                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1649                 "cwd": "/bin",
1650                 "environment": {"FROBIZ": "bilbo"},
1651                 "mounts": {
1652         "/tmp": {"kind": "tmp"},
1653         "/tmp/foo": {"kind": "collection",
1654                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1655                      "exclude_from_output": true
1656         },
1657         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1658     },
1659                 "output_path": "/tmp",
1660                 "priority": 1,
1661                 "runtime_constraints": {},
1662                 "state": "Locked"
1663         }`
1664
1665         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1666
1667         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1668                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1669         })
1670
1671         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1672         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1673         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1674 }
1675
1676 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1677         helperRecord := `{
1678                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1679                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1680                 "cwd": "/bin",
1681                 "environment": {"FROBIZ": "bilbo"},
1682                 "mounts": {
1683         "/tmp": {"kind": "tmp"},
1684         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1685         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1686         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1687         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1688         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1689     },
1690                 "output_path": "/tmp",
1691                 "priority": 1,
1692                 "runtime_constraints": {},
1693                 "state": "Locked"
1694         }`
1695
1696         extraMounts := []string{
1697                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1698                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1699                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1700         }
1701
1702         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1703                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1704         })
1705
1706         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1707                 "/tmp":                   {realtemp + "/tmp1", false},
1708                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1709                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1710                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1711                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1712         })
1713
1714         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1715         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1716         for _, v := range api.Content {
1717                 if v["collection"] != nil {
1718                         c.Check(v["ensure_unique_name"], Equals, true)
1719                         collection := v["collection"].(arvadosclient.Dict)
1720                         if strings.Index(collection["name"].(string), "output") == 0 {
1721                                 manifest := collection["manifest_text"].(string)
1722
1723                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1724 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1725 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1726 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1727 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1728 `)
1729                         }
1730                 }
1731         }
1732 }
1733
1734 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1735         helperRecord := `{
1736                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1737                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1738                 "cwd": "/bin",
1739                 "environment": {"FROBIZ": "bilbo"},
1740                 "mounts": {
1741         "/tmp": {"kind": "tmp"},
1742         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1743         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1744     },
1745                 "output_path": "/tmp",
1746                 "priority": 1,
1747                 "runtime_constraints": {},
1748                 "state": "Locked"
1749         }`
1750
1751         extraMounts := []string{
1752                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1753         }
1754
1755         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1756                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1757         })
1758
1759         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1760         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1761         for _, v := range s.api.Content {
1762                 if v["collection"] != nil {
1763                         collection := v["collection"].(arvadosclient.Dict)
1764                         if strings.Index(collection["name"].(string), "output") == 0 {
1765                                 manifest := collection["manifest_text"].(string)
1766
1767                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1768 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1769 `)
1770                         }
1771                 }
1772         }
1773 }
1774
1775 func (s *TestSuite) TestOutputError(c *C) {
1776         helperRecord := `{
1777                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1778                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1779                 "cwd": "/bin",
1780                 "environment": {"FROBIZ": "bilbo"},
1781                 "mounts": {
1782                         "/tmp": {"kind": "tmp"}
1783                 },
1784                 "output_path": "/tmp",
1785                 "priority": 1,
1786                 "runtime_constraints": {},
1787                 "state": "Locked"
1788         }`
1789         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1790                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1791         })
1792
1793         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1794 }
1795
1796 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1797         helperRecord := `{
1798                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1799                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1800                 "cwd": "/bin",
1801                 "environment": {"FROBIZ": "bilbo"},
1802                 "mounts": {
1803         "/tmp": {"kind": "tmp"},
1804         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1805         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1806     },
1807                 "output_path": "/tmp",
1808                 "priority": 1,
1809                 "runtime_constraints": {},
1810                 "state": "Locked"
1811         }`
1812
1813         extraMounts := []string{
1814                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1815         }
1816
1817         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1818                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1819         })
1820
1821         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1822         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1823         for _, v := range api.Content {
1824                 if v["collection"] != nil {
1825                         collection := v["collection"].(arvadosclient.Dict)
1826                         if strings.Index(collection["name"].(string), "output") == 0 {
1827                                 manifest := collection["manifest_text"].(string)
1828                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1829 `)
1830                         }
1831                 }
1832         }
1833 }
1834
1835 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1836         helperRecord := `{
1837                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1838                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1839                 "cwd": "/bin",
1840                 "environment": {"FROBIZ": "bilbo"},
1841                 "mounts": {
1842         "/tmp": {"kind": "tmp"},
1843         "stdin": {"kind": "json", "content": "foo"},
1844         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1845     },
1846                 "output_path": "/tmp",
1847                 "priority": 1,
1848                 "runtime_constraints": {},
1849                 "state": "Locked"
1850         }`
1851
1852         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1853                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1854         })
1855
1856         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1857         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1858         for _, v := range api.Content {
1859                 if v["collection"] != nil {
1860                         collection := v["collection"].(arvadosclient.Dict)
1861                         if strings.Index(collection["name"].(string), "output") == 0 {
1862                                 manifest := collection["manifest_text"].(string)
1863                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1864 `)
1865                         }
1866                 }
1867         }
1868 }
1869
1870 func (s *TestSuite) TestStderrMount(c *C) {
1871         api, cr, _ := s.fullRunHelper(c, `{
1872     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1873     "container_image": "`+arvadostest.DockerImage112PDH+`",
1874     "cwd": ".",
1875     "environment": {},
1876     "mounts": {"/tmp": {"kind": "tmp"},
1877                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1878                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1879     "output_path": "/tmp",
1880     "priority": 1,
1881     "runtime_constraints": {},
1882     "state": "Locked"
1883 }`, nil, 1, func() {
1884                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1885                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1886         })
1887
1888         final := api.CalledWith("container.state", "Complete")
1889         c.Assert(final, NotNil)
1890         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1891         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1892
1893         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1894 }
1895
1896 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1897         s.api.callraw = true
1898         err := s.runner.fetchContainerRecord()
1899         c.Assert(err, IsNil)
1900         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1901         c.Logf("%#v", s.runner.Container)
1902         c.Check(err, IsNil)
1903         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1904 }
1905
1906 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1907         nextState := ""
1908         for _, setup := range []func(){
1909                 func() {
1910                         c.Log("// waitErr = ocl runtime error")
1911                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1912                         nextState = "Cancelled"
1913                 },
1914                 func() {
1915                         c.Log("// loadErr = cannot connect")
1916                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1917                         *brokenNodeHook = c.MkDir() + "/broken-node-hook"
1918                         err := ioutil.WriteFile(*brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1919                         c.Assert(err, IsNil)
1920                         nextState = "Queued"
1921                 },
1922         } {
1923                 s.SetUpTest(c)
1924                 setup()
1925                 s.fullRunHelper(c, `{
1926     "command": ["echo", "hello world"],
1927     "container_image": "`+arvadostest.DockerImage112PDH+`",
1928     "cwd": ".",
1929     "environment": {},
1930     "mounts": {"/tmp": {"kind": "tmp"} },
1931     "output_path": "/tmp",
1932     "priority": 1,
1933     "runtime_constraints": {},
1934     "state": "Locked"
1935 }`, nil, 0, func() {})
1936                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1937                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1938                 if *brokenNodeHook != "" {
1939                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1940                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1941                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1942                 } else {
1943                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1944                 }
1945         }
1946 }
1947
1948 func (s *TestSuite) TestBadCommand(c *C) {
1949         for _, startError := range []string{
1950                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1951                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1952                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1953         } {
1954                 s.SetUpTest(c)
1955                 s.executor.startErr = errors.New(startError)
1956                 s.fullRunHelper(c, `{
1957     "command": ["echo", "hello world"],
1958     "container_image": "`+arvadostest.DockerImage112PDH+`",
1959     "cwd": ".",
1960     "environment": {},
1961     "mounts": {"/tmp": {"kind": "tmp"} },
1962     "output_path": "/tmp",
1963     "priority": 1,
1964     "runtime_constraints": {},
1965     "state": "Locked"
1966 }`, nil, 0, func() {})
1967                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1968                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1969         }
1970 }
1971
1972 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1973         helperRecord := `{
1974                 "command": ["true"],
1975                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1976                 "cwd": "/bin",
1977                 "mounts": {
1978                     "/tmp": {"kind": "tmp"},
1979                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1980                 },
1981                 "secret_mounts": {
1982                 },
1983                 "output_path": "/tmp",
1984                 "priority": 1,
1985                 "runtime_constraints": {},
1986                 "state": "Locked"
1987         }`
1988
1989         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1990                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1991                 c.Check(err, IsNil)
1992                 c.Check(string(content), Equals, "mypassword")
1993         })
1994
1995         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1996         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1997         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
1998         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
1999
2000         // under secret mounts, not captured in output
2001         helperRecord = `{
2002                 "command": ["true"],
2003                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2004                 "cwd": "/bin",
2005                 "mounts": {
2006                     "/tmp": {"kind": "tmp"}
2007                 },
2008                 "secret_mounts": {
2009                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2010                 },
2011                 "output_path": "/tmp",
2012                 "priority": 1,
2013                 "runtime_constraints": {},
2014                 "state": "Locked"
2015         }`
2016
2017         s.SetUpTest(c)
2018         s.fullRunHelper(c, helperRecord, nil, 0, func() {
2019                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2020                 c.Check(err, IsNil)
2021                 c.Check(string(content), Equals, "mypassword")
2022         })
2023
2024         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2025         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2026         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2027         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2028
2029         // under secret mounts, output dir is a collection, not captured in output
2030         helperRecord = `{
2031                 "command": ["true"],
2032                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2033                 "cwd": "/bin",
2034                 "mounts": {
2035                     "/tmp": {"kind": "collection", "writable": true}
2036                 },
2037                 "secret_mounts": {
2038                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2039                 },
2040                 "output_path": "/tmp",
2041                 "priority": 1,
2042                 "runtime_constraints": {},
2043                 "state": "Locked"
2044         }`
2045
2046         s.SetUpTest(c)
2047         _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, 0, func() {
2048                 // secret.conf should be provisioned as a separate
2049                 // bind mount, i.e., it should not appear in the
2050                 // (fake) fuse filesystem as viewed from the host.
2051                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2052                 if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) {
2053                         c.Logf("secret.conf: content %q, err %#v", content, err)
2054                 }
2055                 err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700)
2056                 c.Check(err, IsNil)
2057         })
2058
2059         content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text")
2060         c.Check(err, IsNil)
2061         c.Check(string(content), Equals, "mypassword")
2062         c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true})
2063         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2064         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2065         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil)
2066 }
2067
2068 type FakeProcess struct {
2069         cmdLine []string
2070 }
2071
2072 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2073         return fp.cmdLine, nil
2074 }