lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/lib/cmd"
  26         "git.arvados.org/arvados.git/sdk/go/arvados"
  27         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  28         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  29         "git.arvados.org/arvados.git/sdk/go/manifest"
  30         "golang.org/x/net/context"
  31
  32         . "gopkg.in/check.v1"
  33 )
  34
  35 // Gocheck boilerplate
  36 func TestCrunchExec(t *testing.T) {
  37         TestingT(t)
  38 }
  39
  40 var _ = Suite(&TestSuite{})
  41
  42 type TestSuite struct {
  43         client                   *arvados.Client
  44         api                      *ArvTestClient
  45         runner                   *ContainerRunner
  46         executor                 *stubExecutor
  47         keepmount                string
  48         keepmountTmp             []string
  49         testDispatcherKeepClient KeepTestClient
  50         testContainerKeepClient  KeepTestClient
  51 }
  52
  53 func (s *TestSuite) SetUpTest(c *C) {
  54         s.client = arvados.NewClientFromEnv()
  55         s.executor = &stubExecutor{}
  56         var err error
  57         s.api = &ArvTestClient{}
  58         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  59         c.Assert(err, IsNil)
  60         s.runner.executor = s.executor
  61         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  62                 return s.api, &s.testContainerKeepClient, s.client, nil
  63         }
  64         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  65                 s.runner.ArvMountPoint = s.keepmount
  66                 for i, opt := range cmd {
  67                         if opt == "--mount-tmp" {
  68                                 err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700)
  69                                 if err != nil {
  70                                         return nil, err
  71                                 }
  72                                 s.keepmountTmp = append(s.keepmountTmp, cmd[i+1])
  73                         }
  74                 }
  75                 return nil, nil
  76         }
  77         s.keepmount = c.MkDir()
  78         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  79         s.keepmountTmp = nil
  80         c.Assert(err, IsNil)
  81         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  82         c.Assert(err, IsNil)
  83         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  84         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  85         c.Assert(err, IsNil)
  86         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  87         c.Assert(err, IsNil)
  88         s.runner.ArvMountPoint = s.keepmount
  89 }
  90
  91 type ArvTestClient struct {
  92         Total   int64
  93         Calls   int
  94         Content []arvadosclient.Dict
  95         arvados.Container
  96         secretMounts []byte
  97         Logs         map[string]*bytes.Buffer
  98         sync.Mutex
  99         WasSetRunning bool
 100         callraw       bool
 101 }
 102
 103 type KeepTestClient struct {
 104         Called         bool
 105         Content        []byte
 106         StorageClasses []string
 107 }
 108
 109 type stubExecutor struct {
 110         imageLoaded bool
 111         loaded      string
 112         loadErr     error
 113         exitCode    int
 114         createErr   error
 115         created     containerSpec
 116         startErr    error
 117         waitSleep   time.Duration
 118         waitErr     error
 119         stopErr     error
 120         stopped     bool
 121         closed      bool
 122         runFunc     func()
 123         exit        chan int
 124 }
 125
 126 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 127         containerClient *arvados.Client) error {
 128         e.loaded = tarball
 129         return e.loadErr
 130 }
 131 func (e *stubExecutor) Runtime() string                 { return "stub" }
 132 func (e *stubExecutor) Version() string                 { return "stub " + cmd.Version.String() }
 133 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 134 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 135 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 136 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 137 func (e *stubExecutor) Close()                          { e.closed = true }
 138 func (e *stubExecutor) Wait(context.Context) (int, error) {
 139         return <-e.exit, e.waitErr
 140 }
 141
 142 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 143
 144 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 145 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 146 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 147
 148 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 149 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 150
 151 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 152 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 153 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 154 `
 155
 156 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 157
 158 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 159 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 160
 161 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 162 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 163
 164 func (client *ArvTestClient) Create(resourceType string,
 165         parameters arvadosclient.Dict,
 166         output interface{}) error {
 167
 168         client.Mutex.Lock()
 169         defer client.Mutex.Unlock()
 170
 171         client.Calls++
 172         client.Content = append(client.Content, parameters)
 173
 174         if resourceType == "logs" {
 175                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 176                 if client.Logs == nil {
 177                         client.Logs = make(map[string]*bytes.Buffer)
 178                 }
 179                 if client.Logs[et] == nil {
 180                         client.Logs[et] = &bytes.Buffer{}
 181                 }
 182                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 183         }
 184
 185         if resourceType == "collections" && output != nil {
 186                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 187                 outmap := output.(*arvados.Collection)
 188                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 189                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 190         }
 191
 192         return nil
 193 }
 194
 195 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 196         switch {
 197         case method == "GET" && resourceType == "containers" && action == "auth":
 198                 return json.Unmarshal([]byte(`{
 199                         "kind": "arvados#api_client_authorization",
 200                         "uuid": "`+fakeAuthUUID+`",
 201                         "api_token": "`+fakeAuthToken+`"
 202                         }`), output)
 203         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 204                 if client.secretMounts != nil {
 205                         return json.Unmarshal(client.secretMounts, output)
 206                 }
 207                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 208         default:
 209                 return fmt.Errorf("Not found")
 210         }
 211 }
 212
 213 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 214         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 215         var j []byte
 216         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 217                 j = []byte(`{
 218                         "kind": "arvados#nodeList",
 219                         "items": [{
 220                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 221                                 "hostname": "compute2",
 222                                 "properties": {"total_cpu_cores": 16}
 223                         }]}`)
 224         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 225                 if uuid == "" {
 226                         j, err = json.Marshal(map[string]interface{}{
 227                                 "items": []interface{}{client.Container},
 228                                 "kind":  "arvados#nodeList",
 229                         })
 230                 } else {
 231                         j, err = json.Marshal(client.Container)
 232                 }
 233         } else {
 234                 j = []byte(`{
 235                         "command": ["sleep", "1"],
 236                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 237                         "cwd": ".",
 238                         "environment": {},
 239                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 240                         "output_path": "/tmp",
 241                         "priority": 1,
 242                         "runtime_constraints": {}
 243                 }`)
 244         }
 245         return ioutil.NopCloser(bytes.NewReader(j)), err
 246 }
 247
 248 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 249         if resourceType == "collections" {
 250                 if uuid == hwPDH {
 251                         output.(*arvados.Collection).ManifestText = hwManifest
 252                 } else if uuid == otherPDH {
 253                         output.(*arvados.Collection).ManifestText = otherManifest
 254                 } else if uuid == normalizedWithSubdirsPDH {
 255                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 256                 } else if uuid == denormalizedWithSubdirsPDH {
 257                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 258                 }
 259         }
 260         if resourceType == "containers" {
 261                 (*output.(*arvados.Container)) = client.Container
 262         }
 263         return nil
 264 }
 265
 266 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 267         client.Mutex.Lock()
 268         defer client.Mutex.Unlock()
 269         client.Calls++
 270         client.Content = append(client.Content, parameters)
 271         if resourceType == "containers" {
 272                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 273                         client.WasSetRunning = true
 274                 }
 275         } else if resourceType == "collections" {
 276                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 277                 output.(*arvados.Collection).UUID = uuid
 278                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 279         }
 280         return nil
 281 }
 282
 283 var discoveryMap = map[string]interface{}{
 284         "defaultTrashLifetime":               float64(1209600),
 285         "crunchLimitLogBytesPerJob":          float64(67108864),
 286         "crunchLogThrottleBytes":             float64(65536),
 287         "crunchLogThrottlePeriod":            float64(60),
 288         "crunchLogThrottleLines":             float64(1024),
 289         "crunchLogPartialLineThrottlePeriod": float64(5),
 290         "crunchLogBytesPerEvent":             float64(4096),
 291         "crunchLogSecondsBetweenEvents":      float64(1),
 292 }
 293
 294 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 295         return discoveryMap[key], nil
 296 }
 297
 298 // CalledWith returns the parameters from the first API call whose
 299 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 300 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 301 // no call matches, it returns nil.
 302 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 303 call:
 304         for _, content := range client.Content {
 305                 var v interface{} = content
 306                 for _, k := range strings.Split(jpath, ".") {
 307                         if dict, ok := v.(arvadosclient.Dict); !ok {
 308                                 continue call
 309                         } else {
 310                                 v = dict[k]
 311                         }
 312                 }
 313                 if v == expect {
 314                         return content
 315                 }
 316         }
 317         return nil
 318 }
 319
 320 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 321         return locator, nil
 322 }
 323
 324 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 325         client.Content = opts.Data
 326         return arvados.BlockWriteResponse{
 327                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 328         }, nil
 329 }
 330
 331 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 332         return 0, errors.New("not implemented")
 333 }
 334
 335 func (client *KeepTestClient) ClearBlockCache() {
 336 }
 337
 338 func (client *KeepTestClient) Close() {
 339         client.Content = nil
 340 }
 341
 342 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 343         client.StorageClasses = sc
 344 }
 345
 346 type FileWrapper struct {
 347         io.ReadCloser
 348         len int64
 349 }
 350
 351 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 352         return nil, errors.New("not implemented")
 353 }
 354
 355 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 356         return 0, errors.New("not implemented")
 357 }
 358
 359 func (fw FileWrapper) Size() int64 {
 360         return fw.len
 361 }
 362
 363 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 364         return nil, errors.New("not implemented")
 365 }
 366
 367 func (fw FileWrapper) Truncate(int64) error {
 368         return errors.New("not implemented")
 369 }
 370
 371 func (fw FileWrapper) Write([]byte) (int, error) {
 372         return 0, errors.New("not implemented")
 373 }
 374
 375 func (fw FileWrapper) Sync() error {
 376         return errors.New("not implemented")
 377 }
 378
 379 func (fw FileWrapper) Snapshot() (*arvados.Subtree, error) {
 380         return nil, errors.New("not implemented")
 381 }
 382
 383 func (fw FileWrapper) Splice(*arvados.Subtree) error {
 384         return errors.New("not implemented")
 385 }
 386
 387 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 388         if filename == hwImageID+".tar" {
 389                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 390                 client.Called = true
 391                 return FileWrapper{rdr, 1321984}, nil
 392         } else if filename == "/file1_in_main.txt" {
 393                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 394                 client.Called = true
 395                 return FileWrapper{rdr, 3}, nil
 396         }
 397         return nil, nil
 398 }
 399
 400 func (s *TestSuite) TestLoadImage(c *C) {
 401         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 402         s.runner.Container.Mounts = map[string]arvados.Mount{
 403                 "/out": {Kind: "tmp", Writable: true},
 404         }
 405         s.runner.Container.OutputPath = "/out"
 406
 407         _, err := s.runner.SetupMounts()
 408         c.Assert(err, IsNil)
 409
 410         imageID, err := s.runner.LoadImage()
 411         c.Check(err, IsNil)
 412         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 413         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 414
 415         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 416         s.executor.imageLoaded = false
 417         s.executor.loaded = ""
 418         s.executor.loadErr = errors.New("bork")
 419         imageID, err = s.runner.LoadImage()
 420         c.Check(err, ErrorMatches, ".*bork")
 421         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 422
 423         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 424         s.executor.imageLoaded = false
 425         s.executor.loaded = ""
 426         s.executor.loadErr = nil
 427         imageID, err = s.runner.LoadImage()
 428         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 429         c.Check(s.executor.loaded, Equals, "")
 430 }
 431
 432 type ArvErrorTestClient struct{}
 433
 434 func (ArvErrorTestClient) Create(resourceType string,
 435         parameters arvadosclient.Dict,
 436         output interface{}) error {
 437         return nil
 438 }
 439
 440 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 441         if method == "GET" && resourceType == "containers" && action == "auth" {
 442                 return nil
 443         }
 444         return errors.New("ArvError")
 445 }
 446
 447 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 448         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 449         return nil, errors.New("ArvError")
 450 }
 451
 452 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 453         return errors.New("ArvError")
 454 }
 455
 456 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 457         return nil
 458 }
 459
 460 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 461         return discoveryMap[key], nil
 462 }
 463
 464 type KeepErrorTestClient struct {
 465         KeepTestClient
 466 }
 467
 468 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 469         return nil, errors.New("KeepError")
 470 }
 471
 472 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 473         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 474 }
 475
 476 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 477         return "", errors.New("KeepError")
 478 }
 479
 480 type KeepReadErrorTestClient struct {
 481         KeepTestClient
 482 }
 483
 484 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 485         return 0, errors.New("KeepError")
 486 }
 487
 488 type ErrorReader struct {
 489         FileWrapper
 490 }
 491
 492 func (ErrorReader) Read(p []byte) (n int, err error) {
 493         return 0, errors.New("ErrorReader")
 494 }
 495
 496 func (ErrorReader) Seek(int64, int) (int64, error) {
 497         return 0, errors.New("ErrorReader")
 498 }
 499
 500 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 501         return ErrorReader{}, nil
 502 }
 503
 504 type ClosableBuffer struct {
 505         bytes.Buffer
 506 }
 507
 508 func (*ClosableBuffer) Close() error {
 509         return nil
 510 }
 511
 512 type TestLogs struct {
 513         Stdout ClosableBuffer
 514         Stderr ClosableBuffer
 515 }
 516
 517 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 518         if logstr == "stdout" {
 519                 return &tl.Stdout, nil
 520         }
 521         if logstr == "stderr" {
 522                 return &tl.Stderr, nil
 523         }
 524         return nil, errors.New("???")
 525 }
 526
 527 func dockerLog(fd byte, msg string) []byte {
 528         by := []byte(msg)
 529         header := make([]byte, 8+len(by))
 530         header[0] = fd
 531         header[7] = byte(len(by))
 532         copy(header[8:], by)
 533         return header
 534 }
 535
 536 func (s *TestSuite) TestRunContainer(c *C) {
 537         s.executor.runFunc = func() {
 538                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 539                 s.executor.exit <- 0
 540         }
 541
 542         var logs TestLogs
 543         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 544         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 545         s.runner.Container.Command = []string{"./hw"}
 546         s.runner.Container.OutputStorageClasses = []string{"default"}
 547
 548         imageID, err := s.runner.LoadImage()
 549         c.Assert(err, IsNil)
 550
 551         err = s.runner.CreateContainer(imageID, nil)
 552         c.Assert(err, IsNil)
 553
 554         err = s.runner.StartContainer()
 555         c.Assert(err, IsNil)
 556
 557         err = s.runner.WaitFinish()
 558         c.Assert(err, IsNil)
 559
 560         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 561         c.Check(logs.Stderr.String(), Equals, "")
 562 }
 563
 564 func (s *TestSuite) TestCommitLogs(c *C) {
 565         api := &ArvTestClient{}
 566         kc := &KeepTestClient{}
 567         defer kc.Close()
 568         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 569         c.Assert(err, IsNil)
 570         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 571
 572         cr.CrunchLog.Print("Hello world!")
 573         cr.CrunchLog.Print("Goodbye")
 574         cr.finalState = "Complete"
 575
 576         err = cr.CommitLogs()
 577         c.Check(err, IsNil)
 578
 579         c.Check(api.Calls, Equals, 2)
 580         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 581         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 582         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 583         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 584 }
 585
 586 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 587         api := &ArvTestClient{}
 588         kc := &KeepTestClient{}
 589         defer kc.Close()
 590         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 591         c.Assert(err, IsNil)
 592
 593         err = cr.UpdateContainerRunning()
 594         c.Check(err, IsNil)
 595
 596         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 597 }
 598
 599 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 600         api := &ArvTestClient{}
 601         kc := &KeepTestClient{}
 602         defer kc.Close()
 603         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 604         c.Assert(err, IsNil)
 605
 606         cr.LogsPDH = new(string)
 607         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 608
 609         cr.ExitCode = new(int)
 610         *cr.ExitCode = 42
 611         cr.finalState = "Complete"
 612
 613         err = cr.UpdateContainerFinal()
 614         c.Check(err, IsNil)
 615
 616         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 617         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 618         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 619 }
 620
 621 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 622         api := &ArvTestClient{}
 623         kc := &KeepTestClient{}
 624         defer kc.Close()
 625         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 626         c.Assert(err, IsNil)
 627         cr.cCancelled = true
 628         cr.finalState = "Cancelled"
 629
 630         err = cr.UpdateContainerFinal()
 631         c.Check(err, IsNil)
 632
 633         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 634         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 635         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 636 }
 637
 638 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 639 // dress rehearsal of the Run() function, starting from a JSON container record.
 640 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 641         err := json.Unmarshal([]byte(record), &s.api.Container)
 642         c.Assert(err, IsNil)
 643         initialState := s.api.Container.State
 644
 645         var sm struct {
 646                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 647         }
 648         err = json.Unmarshal([]byte(record), &sm)
 649         c.Check(err, IsNil)
 650         secretMounts, err := json.Marshal(sm)
 651         c.Assert(err, IsNil)
 652         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 653
 654         s.executor.runFunc = func() {
 655                 fn()
 656                 s.executor.exit <- exitCode
 657         }
 658
 659         s.runner.statInterval = 100 * time.Millisecond
 660         s.runner.containerWatchdogInterval = time.Second
 661
 662         realTemp := c.MkDir()
 663         tempcount := 0
 664         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 665                 tempcount++
 666                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 667                 err := os.Mkdir(d, os.ModePerm)
 668                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 669                         // Test case must have pre-populated the tempdir
 670                         err = nil
 671                 }
 672                 return d, err
 673         }
 674         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 675                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 676         }
 677
 678         if extraMounts != nil && len(extraMounts) > 0 {
 679                 err := s.runner.SetupArvMountPoint("keep")
 680                 c.Check(err, IsNil)
 681
 682                 for _, m := range extraMounts {
 683                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 684                 }
 685         }
 686
 687         err = s.runner.Run()
 688         if s.api.CalledWith("container.state", "Complete") != nil {
 689                 c.Check(err, IsNil)
 690         }
 691         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 692                 c.Check(s.api.WasSetRunning, Equals, true)
 693                 var lastupdate arvadosclient.Dict
 694                 for _, content := range s.api.Content {
 695                         if content["container"] != nil {
 696                                 lastupdate = content["container"].(arvadosclient.Dict)
 697                         }
 698                 }
 699                 if lastupdate["log"] == nil {
 700                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 701                 }
 702         }
 703
 704         if err != nil {
 705                 for k, v := range s.api.Logs {
 706                         c.Log(k)
 707                         c.Log(v.String())
 708                 }
 709         }
 710
 711         return s.api, s.runner, realTemp
 712 }
 713
 714 func (s *TestSuite) TestFullRunHello(c *C) {
 715         s.runner.enableMemoryLimit = true
 716         s.runner.networkMode = "default"
 717         s.fullRunHelper(c, `{
 718     "command": ["echo", "hello world"],
 719     "container_image": "`+arvadostest.DockerImage112PDH+`",
 720     "cwd": ".",
 721     "environment": {"foo":"bar","baz":"waz"},
 722     "mounts": {"/tmp": {"kind": "tmp"} },
 723     "output_path": "/tmp",
 724     "priority": 1,
 725     "runtime_constraints": {"vcpus":1,"ram":1000000},
 726     "state": "Locked",
 727     "output_storage_classes": ["default"]
 728 }`, nil, 0, func() {
 729                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 730                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 731                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 732                 c.Check(s.executor.created.VCPUs, Equals, 1)
 733                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 734                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 735                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 736                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
 737                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 738         })
 739
 740         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 741         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 742         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 743         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 744         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 745 }
 746
 747 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 748         var ran bool
 749         s.fullRunHelper(c, `{
 750     "command": ["sleep", "3"],
 751     "container_image": "`+arvadostest.DockerImage112PDH+`",
 752     "cwd": ".",
 753     "environment": {},
 754     "mounts": {"/tmp": {"kind": "tmp"} },
 755     "output_path": "/tmp",
 756     "priority": 1,
 757     "runtime_constraints": {},
 758     "scheduling_parameters":{"max_run_time": 1},
 759     "state": "Running"
 760 }`, nil, 2, func() {
 761                 ran = true
 762         })
 763         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 764         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 765         c.Check(ran, Equals, false)
 766 }
 767
 768 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 769         s.fullRunHelper(c, `{
 770     "command": ["sleep", "3"],
 771     "container_image": "`+arvadostest.DockerImage112PDH+`",
 772     "cwd": ".",
 773     "environment": {},
 774     "mounts": {"/tmp": {"kind": "tmp"} },
 775     "output_path": "/tmp",
 776     "priority": 1,
 777     "runtime_constraints": {},
 778     "scheduling_parameters":{"max_run_time": 1},
 779     "state": "Locked"
 780 }`, nil, 0, func() {
 781                 time.Sleep(3 * time.Second)
 782         })
 783
 784         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 785         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 786 }
 787
 788 func (s *TestSuite) TestContainerWaitFails(c *C) {
 789         s.fullRunHelper(c, `{
 790     "command": ["sleep", "3"],
 791     "container_image": "`+arvadostest.DockerImage112PDH+`",
 792     "cwd": ".",
 793     "mounts": {"/tmp": {"kind": "tmp"} },
 794     "output_path": "/tmp",
 795     "priority": 1,
 796     "state": "Locked"
 797 }`, nil, 0, func() {
 798                 s.executor.waitErr = errors.New("Container is not running")
 799         })
 800
 801         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 802         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 803 }
 804
 805 func (s *TestSuite) TestCrunchstat(c *C) {
 806         s.fullRunHelper(c, `{
 807                 "command": ["sleep", "1"],
 808                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 809                 "cwd": ".",
 810                 "environment": {},
 811                 "mounts": {"/tmp": {"kind": "tmp"} },
 812                 "output_path": "/tmp",
 813                 "priority": 1,
 814                 "runtime_constraints": {},
 815                 "state": "Locked"
 816         }`, nil, 0, func() {
 817                 time.Sleep(time.Second)
 818         })
 819
 820         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 821         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 822
 823         // We didn't actually start a container, so crunchstat didn't
 824         // find accounting files and therefore didn't log any stats.
 825         // It should have logged a "can't find accounting files"
 826         // message after one poll interval, though, so we can confirm
 827         // it's alive:
 828         c.Assert(s.api.Logs["crunchstat"], NotNil)
 829         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 830
 831         // The "files never appeared" log assures us that we called
 832         // (*crunchstat.Reporter)Stop(), and that we set it up with
 833         // the correct container ID "abcde":
 834         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 835 }
 836
 837 func (s *TestSuite) TestNodeInfoLog(c *C) {
 838         os.Setenv("SLURMD_NODENAME", "compute2")
 839         s.fullRunHelper(c, `{
 840                 "command": ["sleep", "1"],
 841                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 842                 "cwd": ".",
 843                 "environment": {},
 844                 "mounts": {"/tmp": {"kind": "tmp"} },
 845                 "output_path": "/tmp",
 846                 "priority": 1,
 847                 "runtime_constraints": {},
 848                 "state": "Locked"
 849         }`, nil, 0,
 850                 func() {
 851                         time.Sleep(time.Second)
 852                 })
 853
 854         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 855         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 856
 857         c.Assert(s.api.Logs["node"], NotNil)
 858         json := s.api.Logs["node"].String()
 859         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 860         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 861         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 862
 863         c.Assert(s.api.Logs["node-info"], NotNil)
 864         json = s.api.Logs["node-info"].String()
 865         c.Check(json, Matches, `(?ms).*Host Information.*`)
 866         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 867         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 868         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 869         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 870 }
 871
 872 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 873         s.fullRunHelper(c, `{
 874                 "command": ["sleep", "1"],
 875                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 876                 "cwd": ".",
 877                 "environment": {},
 878                 "mounts": {"/tmp": {"kind": "tmp"} },
 879                 "output_path": "/tmp",
 880                 "priority": 1,
 881                 "runtime_constraints": {},
 882                 "state": "Locked"
 883         }`, nil, 0,
 884                 func() {
 885                 })
 886
 887         c.Assert(s.api.Logs["crunch-run"], NotNil)
 888         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 889         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`)
 890         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container: zzzzz-zzzzz-zzzzzzzzzzzzzzz.*`)
 891         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Using container runtime: stub.*`)
 892 }
 893
 894 func (s *TestSuite) TestContainerRecordLog(c *C) {
 895         s.fullRunHelper(c, `{
 896                 "command": ["sleep", "1"],
 897                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 898                 "cwd": ".",
 899                 "environment": {},
 900                 "mounts": {"/tmp": {"kind": "tmp"} },
 901                 "output_path": "/tmp",
 902                 "priority": 1,
 903                 "runtime_constraints": {},
 904                 "state": "Locked"
 905         }`, nil, 0,
 906                 func() {
 907                         time.Sleep(time.Second)
 908                 })
 909
 910         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 911         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 912
 913         c.Assert(s.api.Logs["container"], NotNil)
 914         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 915 }
 916
 917 func (s *TestSuite) TestFullRunStderr(c *C) {
 918         s.fullRunHelper(c, `{
 919     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 920     "container_image": "`+arvadostest.DockerImage112PDH+`",
 921     "cwd": ".",
 922     "environment": {},
 923     "mounts": {"/tmp": {"kind": "tmp"} },
 924     "output_path": "/tmp",
 925     "priority": 1,
 926     "runtime_constraints": {},
 927     "state": "Locked"
 928 }`, nil, 1, func() {
 929                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 930                 fmt.Fprintln(s.executor.created.Stderr, "world")
 931         })
 932
 933         final := s.api.CalledWith("container.state", "Complete")
 934         c.Assert(final, NotNil)
 935         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 936         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 937
 938         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 939         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 940 }
 941
 942 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 943         s.fullRunHelper(c, `{
 944     "command": ["pwd"],
 945     "container_image": "`+arvadostest.DockerImage112PDH+`",
 946     "cwd": ".",
 947     "environment": {},
 948     "mounts": {"/tmp": {"kind": "tmp"} },
 949     "output_path": "/tmp",
 950     "priority": 1,
 951     "runtime_constraints": {},
 952     "state": "Locked"
 953 }`, nil, 0, func() {
 954                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 955         })
 956
 957         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 958         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 959         c.Log(s.api.Logs["stdout"])
 960         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 961 }
 962
 963 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 964         s.fullRunHelper(c, `{
 965     "command": ["pwd"],
 966     "container_image": "`+arvadostest.DockerImage112PDH+`",
 967     "cwd": "/bin",
 968     "environment": {},
 969     "mounts": {"/tmp": {"kind": "tmp"} },
 970     "output_path": "/tmp",
 971     "priority": 1,
 972     "runtime_constraints": {},
 973     "state": "Locked"
 974 }`, nil, 0, func() {
 975                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 976         })
 977
 978         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 979         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 980         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 981 }
 982
 983 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
 984         s.fullRunHelper(c, `{
 985     "command": ["pwd"],
 986     "container_image": "`+arvadostest.DockerImage112PDH+`",
 987     "cwd": "/bin",
 988     "environment": {},
 989     "mounts": {"/tmp": {"kind": "tmp"} },
 990     "output_path": "/tmp",
 991     "priority": 1,
 992     "runtime_constraints": {},
 993     "state": "Locked",
 994     "output_storage_classes": ["foo", "bar"]
 995 }`, nil, 0, func() {
 996                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 997         })
 998
 999         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1000         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1001         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1002         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1003         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1004 }
1005
1006 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
1007         s.fullRunHelper(c, `{
1008     "command": ["pwd"],
1009     "container_image": "`+arvadostest.DockerImage112PDH+`",
1010     "cwd": "/bin",
1011     "environment": {},
1012     "mounts": {"/tmp": {"kind": "tmp"} },
1013     "output_path": "/tmp",
1014     "priority": 1,
1015     "runtime_constraints": {"cuda": {"device_count": 2}},
1016     "state": "Locked",
1017     "output_storage_classes": ["foo", "bar"]
1018 }`, nil, 0, func() {
1019                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1020         })
1021         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1022 }
1023
1024 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1025         s.fullRunHelper(c, `{
1026     "command": ["pwd"],
1027     "container_image": "`+arvadostest.DockerImage112PDH+`",
1028     "cwd": "/bin",
1029     "environment": {},
1030     "mounts": {"/tmp": {"kind": "tmp"} },
1031     "output_path": "/tmp",
1032     "priority": 1,
1033     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1034     "state": "Locked",
1035     "output_storage_classes": ["foo", "bar"]
1036 }`, nil, 0, func() {
1037                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1038         })
1039         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1040 }
1041
1042 func (s *TestSuite) TestStopOnSignal(c *C) {
1043         s.executor.runFunc = func() {
1044                 s.executor.created.Stdout.Write([]byte("foo\n"))
1045                 s.runner.SigChan <- syscall.SIGINT
1046         }
1047         s.testStopContainer(c)
1048 }
1049
1050 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1051         s.executor.runFunc = func() {
1052                 s.executor.created.Stdout.Write([]byte("foo\n"))
1053                 s.runner.ArvMountExit <- nil
1054                 close(s.runner.ArvMountExit)
1055         }
1056         s.runner.ArvMountExit = make(chan error)
1057         s.testStopContainer(c)
1058 }
1059
1060 func (s *TestSuite) testStopContainer(c *C) {
1061         record := `{
1062     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1063     "container_image": "` + arvadostest.DockerImage112PDH + `",
1064     "cwd": ".",
1065     "environment": {},
1066     "mounts": {"/tmp": {"kind": "tmp"} },
1067     "output_path": "/tmp",
1068     "priority": 1,
1069     "runtime_constraints": {},
1070     "state": "Locked"
1071 }`
1072
1073         err := json.Unmarshal([]byte(record), &s.api.Container)
1074         c.Assert(err, IsNil)
1075
1076         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1077         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1078                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1079         }
1080
1081         done := make(chan error)
1082         go func() {
1083                 done <- s.runner.Run()
1084         }()
1085         select {
1086         case <-time.After(20 * time.Second):
1087                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1088                 c.Fatal("timed out")
1089         case err = <-done:
1090                 c.Check(err, IsNil)
1091         }
1092         for k, v := range s.api.Logs {
1093                 c.Log(k)
1094                 c.Log(v.String(), "\n")
1095         }
1096
1097         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1098         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1099         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1100 }
1101
1102 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1103         s.fullRunHelper(c, `{
1104     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1105     "container_image": "`+arvadostest.DockerImage112PDH+`",
1106     "cwd": "/bin",
1107     "environment": {"FROBIZ": "bilbo"},
1108     "mounts": {"/tmp": {"kind": "tmp"} },
1109     "output_path": "/tmp",
1110     "priority": 1,
1111     "runtime_constraints": {},
1112     "state": "Locked"
1113 }`, nil, 0, func() {
1114                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1115         })
1116
1117         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1118         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1119         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1120 }
1121
1122 type ArvMountCmdLine struct {
1123         Cmd   []string
1124         token string
1125 }
1126
1127 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1128         am.Cmd = c
1129         am.token = token
1130         return nil, nil
1131 }
1132
1133 func stubCert(temp string) string {
1134         path := temp + "/ca-certificates.crt"
1135         crt, _ := os.Create(path)
1136         crt.Close()
1137         arvadosclient.CertFiles = []string{path}
1138         return path
1139 }
1140
1141 func (s *TestSuite) TestSetupMounts(c *C) {
1142         cr := s.runner
1143         am := &ArvMountCmdLine{}
1144         cr.RunArvMount = am.ArvMountTest
1145         cr.ContainerArvClient = &ArvTestClient{}
1146         cr.ContainerKeepClient = &KeepTestClient{}
1147         cr.Container.OutputStorageClasses = []string{"default"}
1148
1149         realTemp := c.MkDir()
1150         certTemp := c.MkDir()
1151         stubCertPath := stubCert(certTemp)
1152         cr.parentTemp = realTemp
1153
1154         i := 0
1155         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1156                 i++
1157                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1158                 err := os.Mkdir(d, os.ModePerm)
1159                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1160                         // Test case must have pre-populated the tempdir
1161                         err = nil
1162                 }
1163                 return d, err
1164         }
1165
1166         checkEmpty := func() {
1167                 // Should be deleted.
1168                 _, err := os.Stat(realTemp)
1169                 c.Assert(os.IsNotExist(err), Equals, true)
1170
1171                 // Now recreate it for the next test.
1172                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1173         }
1174
1175         {
1176                 i = 0
1177                 cr.ArvMountPoint = ""
1178                 cr.Container.Mounts = make(map[string]arvados.Mount)
1179                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1180                 cr.Container.OutputPath = "/tmp"
1181                 cr.statInterval = 5 * time.Second
1182                 bindmounts, err := cr.SetupMounts()
1183                 c.Check(err, IsNil)
1184                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1185                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1186                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1187                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1188                 os.RemoveAll(cr.ArvMountPoint)
1189                 cr.CleanupDirs()
1190                 checkEmpty()
1191         }
1192
1193         {
1194                 i = 0
1195                 cr.ArvMountPoint = ""
1196                 cr.Container.Mounts = make(map[string]arvados.Mount)
1197                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1198                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1199                 cr.Container.OutputPath = "/out"
1200                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1201
1202                 bindmounts, err := cr.SetupMounts()
1203                 c.Check(err, IsNil)
1204                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1205                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1206                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1207                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1208                 os.RemoveAll(cr.ArvMountPoint)
1209                 cr.CleanupDirs()
1210                 checkEmpty()
1211         }
1212
1213         {
1214                 i = 0
1215                 cr.ArvMountPoint = ""
1216                 cr.Container.Mounts = make(map[string]arvados.Mount)
1217                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1218                 cr.Container.OutputPath = "/tmp"
1219                 cr.Container.RuntimeConstraints.API = true
1220                 cr.Container.OutputStorageClasses = []string{"default"}
1221
1222                 bindmounts, err := cr.SetupMounts()
1223                 c.Check(err, IsNil)
1224                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1225                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1226                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1227                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1228                 os.RemoveAll(cr.ArvMountPoint)
1229                 cr.CleanupDirs()
1230                 checkEmpty()
1231
1232                 cr.Container.RuntimeConstraints.API = false
1233         }
1234
1235         {
1236                 i = 0
1237                 cr.ArvMountPoint = ""
1238                 cr.Container.Mounts = map[string]arvados.Mount{
1239                         "/keeptmp": {Kind: "collection", Writable: true},
1240                 }
1241                 cr.Container.OutputPath = "/keeptmp"
1242
1243                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1244
1245                 bindmounts, err := cr.SetupMounts()
1246                 c.Check(err, IsNil)
1247                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1248                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1249                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1250                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1251                 os.RemoveAll(cr.ArvMountPoint)
1252                 cr.CleanupDirs()
1253                 checkEmpty()
1254         }
1255
1256         {
1257                 i = 0
1258                 cr.ArvMountPoint = ""
1259                 cr.Container.Mounts = map[string]arvados.Mount{
1260                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1261                         "/keepout": {Kind: "collection", Writable: true},
1262                 }
1263                 cr.Container.OutputPath = "/keepout"
1264
1265                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1266                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1267
1268                 bindmounts, err := cr.SetupMounts()
1269                 c.Check(err, IsNil)
1270                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1271                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1272                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1273                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1274                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1275                         "/keepout": {realTemp + "/keep1/tmp0", false},
1276                 })
1277                 os.RemoveAll(cr.ArvMountPoint)
1278                 cr.CleanupDirs()
1279                 checkEmpty()
1280         }
1281
1282         {
1283                 i = 0
1284                 cr.ArvMountPoint = ""
1285                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1286                 cr.Container.Mounts = map[string]arvados.Mount{
1287                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1288                         "/keepout": {Kind: "collection", Writable: true},
1289                 }
1290                 cr.Container.OutputPath = "/keepout"
1291
1292                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1293                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1294
1295                 bindmounts, err := cr.SetupMounts()
1296                 c.Check(err, IsNil)
1297                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1298                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1299                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1300                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1301                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1302                         "/keepout": {realTemp + "/keep1/tmp0", false},
1303                 })
1304                 os.RemoveAll(cr.ArvMountPoint)
1305                 cr.CleanupDirs()
1306                 checkEmpty()
1307         }
1308
1309         for _, test := range []struct {
1310                 in  interface{}
1311                 out string
1312         }{
1313                 {in: "foo", out: `"foo"`},
1314                 {in: nil, out: `null`},
1315                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1316         } {
1317                 i = 0
1318                 cr.ArvMountPoint = ""
1319                 cr.Container.Mounts = map[string]arvados.Mount{
1320                         "/mnt/test.json": {Kind: "json", Content: test.in},
1321                 }
1322                 bindmounts, err := cr.SetupMounts()
1323                 c.Check(err, IsNil)
1324                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1325                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1326                 })
1327                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1328                 c.Check(err, IsNil)
1329                 c.Check(content, DeepEquals, []byte(test.out))
1330                 os.RemoveAll(cr.ArvMountPoint)
1331                 cr.CleanupDirs()
1332                 checkEmpty()
1333         }
1334
1335         for _, test := range []struct {
1336                 in  interface{}
1337                 out string
1338         }{
1339                 {in: "foo", out: `foo`},
1340                 {in: nil, out: "error"},
1341                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1342         } {
1343                 i = 0
1344                 cr.ArvMountPoint = ""
1345                 cr.Container.Mounts = map[string]arvados.Mount{
1346                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1347                 }
1348                 bindmounts, err := cr.SetupMounts()
1349                 if test.out == "error" {
1350                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1351                 } else {
1352                         c.Check(err, IsNil)
1353                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1354                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1355                         })
1356                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1357                         c.Check(err, IsNil)
1358                         c.Check(content, DeepEquals, []byte(test.out))
1359                 }
1360                 os.RemoveAll(cr.ArvMountPoint)
1361                 cr.CleanupDirs()
1362                 checkEmpty()
1363         }
1364
1365         // Read-only mount points are allowed underneath output_dir mount point
1366         {
1367                 i = 0
1368                 cr.ArvMountPoint = ""
1369                 cr.Container.Mounts = make(map[string]arvados.Mount)
1370                 cr.Container.Mounts = map[string]arvados.Mount{
1371                         "/tmp":     {Kind: "tmp"},
1372                         "/tmp/foo": {Kind: "collection"},
1373                 }
1374                 cr.Container.OutputPath = "/tmp"
1375
1376                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1377
1378                 bindmounts, err := cr.SetupMounts()
1379                 c.Check(err, IsNil)
1380                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1381                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1382                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1383                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1384                         "/tmp":     {realTemp + "/tmp2", false},
1385                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1386                 })
1387                 os.RemoveAll(cr.ArvMountPoint)
1388                 cr.CleanupDirs()
1389                 checkEmpty()
1390         }
1391
1392         // Writable mount points copied to output_dir mount point
1393         {
1394                 i = 0
1395                 cr.ArvMountPoint = ""
1396                 cr.Container.Mounts = make(map[string]arvados.Mount)
1397                 cr.Container.Mounts = map[string]arvados.Mount{
1398                         "/tmp": {Kind: "tmp"},
1399                         "/tmp/foo": {Kind: "collection",
1400                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1401                                 Writable:         true},
1402                         "/tmp/bar": {Kind: "collection",
1403                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1404                                 Path:             "baz",
1405                                 Writable:         true},
1406                 }
1407                 cr.Container.OutputPath = "/tmp"
1408
1409                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1410                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1411
1412                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1413                 rf.Write([]byte("bar"))
1414                 rf.Close()
1415
1416                 _, err := cr.SetupMounts()
1417                 c.Check(err, IsNil)
1418                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1419                 c.Check(err, IsNil)
1420                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1421                 c.Check(err, IsNil)
1422                 os.RemoveAll(cr.ArvMountPoint)
1423                 cr.CleanupDirs()
1424                 checkEmpty()
1425         }
1426
1427         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1428         {
1429                 i = 0
1430                 cr.ArvMountPoint = ""
1431                 cr.Container.Mounts = make(map[string]arvados.Mount)
1432                 cr.Container.Mounts = map[string]arvados.Mount{
1433                         "/tmp":     {Kind: "tmp"},
1434                         "/tmp/foo": {Kind: "tmp"},
1435                 }
1436                 cr.Container.OutputPath = "/tmp"
1437
1438                 _, err := cr.SetupMounts()
1439                 c.Check(err, NotNil)
1440                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1441                 os.RemoveAll(cr.ArvMountPoint)
1442                 cr.CleanupDirs()
1443                 checkEmpty()
1444         }
1445
1446         // Only mount point of kind 'collection' is allowed for stdin
1447         {
1448                 i = 0
1449                 cr.ArvMountPoint = ""
1450                 cr.Container.Mounts = make(map[string]arvados.Mount)
1451                 cr.Container.Mounts = map[string]arvados.Mount{
1452                         "stdin": {Kind: "tmp"},
1453                 }
1454
1455                 _, err := cr.SetupMounts()
1456                 c.Check(err, NotNil)
1457                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1458                 os.RemoveAll(cr.ArvMountPoint)
1459                 cr.CleanupDirs()
1460                 checkEmpty()
1461         }
1462
1463         // git_tree mounts
1464         {
1465                 i = 0
1466                 cr.ArvMountPoint = ""
1467                 (*GitMountSuite)(nil).useTestGitServer(c)
1468                 cr.token = arvadostest.ActiveToken
1469                 cr.Container.Mounts = make(map[string]arvados.Mount)
1470                 cr.Container.Mounts = map[string]arvados.Mount{
1471                         "/tip": {
1472                                 Kind:   "git_tree",
1473                                 UUID:   arvadostest.Repository2UUID,
1474                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1475                                 Path:   "/",
1476                         },
1477                         "/non-tip": {
1478                                 Kind:   "git_tree",
1479                                 UUID:   arvadostest.Repository2UUID,
1480                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1481                                 Path:   "/",
1482                         },
1483                 }
1484                 cr.Container.OutputPath = "/tmp"
1485
1486                 bindmounts, err := cr.SetupMounts()
1487                 c.Check(err, IsNil)
1488
1489                 for path, mount := range bindmounts {
1490                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1491                 }
1492
1493                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1494                 c.Check(err, IsNil)
1495                 c.Check(string(data), Equals, "\000\001\002\003")
1496                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1497                 c.Check(err, FitsTypeOf, &os.PathError{})
1498                 c.Check(os.IsNotExist(err), Equals, true)
1499
1500                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1501                 c.Check(err, IsNil)
1502                 c.Check(string(data), Equals, "\000\001\002\003")
1503                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1504                 c.Check(err, IsNil)
1505                 c.Check(string(data), Equals, "testfile\n")
1506
1507                 cr.CleanupDirs()
1508                 checkEmpty()
1509         }
1510 }
1511
1512 func (s *TestSuite) TestStdout(c *C) {
1513         helperRecord := `{
1514                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1515                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1516                 "cwd": "/bin",
1517                 "environment": {"FROBIZ": "bilbo"},
1518                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1519                 "output_path": "/tmp",
1520                 "priority": 1,
1521                 "runtime_constraints": {},
1522                 "state": "Locked"
1523         }`
1524
1525         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1526                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1527         })
1528
1529         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1530         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1531         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1532 }
1533
1534 // Used by the TestStdoutWithWrongPath*()
1535 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1536         err := json.Unmarshal([]byte(record), &s.api.Container)
1537         c.Assert(err, IsNil)
1538         s.executor.runFunc = fn
1539         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1540         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1541                 return s.api, &KeepTestClient{}, nil, nil
1542         }
1543         return s.api, s.runner, s.runner.Run()
1544 }
1545
1546 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1547         _, _, err := s.stdoutErrorRunHelper(c, `{
1548     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1549     "output_path": "/tmp",
1550     "state": "Locked"
1551 }`, func() {})
1552         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1553 }
1554
1555 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1556         _, _, err := s.stdoutErrorRunHelper(c, `{
1557     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1558     "output_path": "/tmp",
1559     "state": "Locked"
1560 }`, func() {})
1561         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1562 }
1563
1564 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1565         _, _, err := s.stdoutErrorRunHelper(c, `{
1566     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1567     "output_path": "/tmp",
1568     "state": "Locked"
1569 }`, func() {})
1570         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1571 }
1572
1573 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1574         s.fullRunHelper(c, `{
1575     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1576     "container_image": "`+arvadostest.DockerImage112PDH+`",
1577     "cwd": "/bin",
1578     "environment": {},
1579     "mounts": {"/tmp": {"kind": "tmp"} },
1580     "output_path": "/tmp",
1581     "priority": 1,
1582     "runtime_constraints": {"API": true},
1583     "state": "Locked"
1584 }`, nil, 0, func() {
1585                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1586                 s.executor.exit <- 3
1587         })
1588         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1589         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1590         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1591 }
1592
1593 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1594         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1595         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1596         s.fullRunHelper(c, `{
1597     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1598     "container_image": "`+arvadostest.DockerImage112PDH+`",
1599     "cwd": "/bin",
1600     "environment": {},
1601     "mounts": {"/tmp": {"kind": "tmp"} },
1602     "output_path": "/tmp",
1603     "priority": 1,
1604     "runtime_constraints": {"API": true},
1605     "state": "Locked"
1606 }`, nil, 0, func() {
1607                 s.api.Container.Output = arvadostest.DockerImage112PDH
1608         })
1609
1610         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1611         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1612         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1613 }
1614
1615 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1616         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1617                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1618                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1619                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1620         }
1621         s.executor.runFunc = func() {
1622                 time.Sleep(time.Second)
1623                 s.executor.exit <- 137
1624         }
1625         record := `{
1626     "command": ["sleep", "1"],
1627     "container_image": "` + arvadostest.DockerImage112PDH + `",
1628     "cwd": "/bin",
1629     "environment": {},
1630     "mounts": {"/tmp": {"kind": "tmp"} },
1631     "output_path": "/tmp",
1632     "priority": 1,
1633     "runtime_constraints": {"API": true},
1634     "state": "Locked"
1635 }`
1636         err := json.Unmarshal([]byte(record), &s.api.Container)
1637         c.Assert(err, IsNil)
1638         err = s.runner.Run()
1639         c.Assert(err, IsNil)
1640         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1641         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1642         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1643         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1644         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1645 }
1646
1647 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1648         helperRecord := `{
1649                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1650                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1651                 "cwd": "/bin",
1652                 "environment": {"FROBIZ": "bilbo"},
1653                 "mounts": {
1654         "/tmp": {"kind": "tmp"},
1655         "/tmp/foo": {"kind": "collection",
1656                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1657                      "exclude_from_output": true
1658         },
1659         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1660     },
1661                 "output_path": "/tmp",
1662                 "priority": 1,
1663                 "runtime_constraints": {},
1664                 "state": "Locked"
1665         }`
1666
1667         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1668
1669         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1670                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1671         })
1672
1673         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1674         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1675         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1676 }
1677
1678 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1679         helperRecord := `{
1680                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1681                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1682                 "cwd": "/bin",
1683                 "environment": {"FROBIZ": "bilbo"},
1684                 "mounts": {
1685         "/tmp": {"kind": "tmp"},
1686         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1687         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1688         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1689         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1690         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1691     },
1692                 "output_path": "/tmp",
1693                 "priority": 1,
1694                 "runtime_constraints": {},
1695                 "state": "Locked"
1696         }`
1697
1698         extraMounts := []string{
1699                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1700                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1701                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1702         }
1703
1704         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1705                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1706         })
1707
1708         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1709                 "/tmp":                   {realtemp + "/tmp1", false},
1710                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1711                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1712                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1713                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1714         })
1715
1716         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1717         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1718         for _, v := range api.Content {
1719                 if v["collection"] != nil {
1720                         c.Check(v["ensure_unique_name"], Equals, true)
1721                         collection := v["collection"].(arvadosclient.Dict)
1722                         if strings.Index(collection["name"].(string), "output") == 0 {
1723                                 manifest := collection["manifest_text"].(string)
1724
1725                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1726 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1727 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1728 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1729 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1730 `)
1731                         }
1732                 }
1733         }
1734 }
1735
1736 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1737         helperRecord := `{
1738                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1739                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1740                 "cwd": "/bin",
1741                 "environment": {"FROBIZ": "bilbo"},
1742                 "mounts": {
1743         "/tmp": {"kind": "tmp"},
1744         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1745         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1746     },
1747                 "output_path": "/tmp",
1748                 "priority": 1,
1749                 "runtime_constraints": {},
1750                 "state": "Locked"
1751         }`
1752
1753         extraMounts := []string{
1754                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1755         }
1756
1757         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1758                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1759         })
1760
1761         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1762         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1763         for _, v := range s.api.Content {
1764                 if v["collection"] != nil {
1765                         collection := v["collection"].(arvadosclient.Dict)
1766                         if strings.Index(collection["name"].(string), "output") == 0 {
1767                                 manifest := collection["manifest_text"].(string)
1768
1769                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1770 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1771 `)
1772                         }
1773                 }
1774         }
1775 }
1776
1777 func (s *TestSuite) TestOutputError(c *C) {
1778         helperRecord := `{
1779                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1780                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1781                 "cwd": "/bin",
1782                 "environment": {"FROBIZ": "bilbo"},
1783                 "mounts": {
1784                         "/tmp": {"kind": "tmp"}
1785                 },
1786                 "output_path": "/tmp",
1787                 "priority": 1,
1788                 "runtime_constraints": {},
1789                 "state": "Locked"
1790         }`
1791         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1792                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1793         })
1794
1795         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1796 }
1797
1798 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1799         helperRecord := `{
1800                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1801                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1802                 "cwd": "/bin",
1803                 "environment": {"FROBIZ": "bilbo"},
1804                 "mounts": {
1805         "/tmp": {"kind": "tmp"},
1806         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1807         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1808     },
1809                 "output_path": "/tmp",
1810                 "priority": 1,
1811                 "runtime_constraints": {},
1812                 "state": "Locked"
1813         }`
1814
1815         extraMounts := []string{
1816                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1817         }
1818
1819         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1820                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1821         })
1822
1823         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1824         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1825         for _, v := range api.Content {
1826                 if v["collection"] != nil {
1827                         collection := v["collection"].(arvadosclient.Dict)
1828                         if strings.Index(collection["name"].(string), "output") == 0 {
1829                                 manifest := collection["manifest_text"].(string)
1830                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1831 `)
1832                         }
1833                 }
1834         }
1835 }
1836
1837 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1838         helperRecord := `{
1839                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1840                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1841                 "cwd": "/bin",
1842                 "environment": {"FROBIZ": "bilbo"},
1843                 "mounts": {
1844         "/tmp": {"kind": "tmp"},
1845         "stdin": {"kind": "json", "content": "foo"},
1846         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1847     },
1848                 "output_path": "/tmp",
1849                 "priority": 1,
1850                 "runtime_constraints": {},
1851                 "state": "Locked"
1852         }`
1853
1854         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1855                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1856         })
1857
1858         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1859         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1860         for _, v := range api.Content {
1861                 if v["collection"] != nil {
1862                         collection := v["collection"].(arvadosclient.Dict)
1863                         if strings.Index(collection["name"].(string), "output") == 0 {
1864                                 manifest := collection["manifest_text"].(string)
1865                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1866 `)
1867                         }
1868                 }
1869         }
1870 }
1871
1872 func (s *TestSuite) TestStderrMount(c *C) {
1873         api, cr, _ := s.fullRunHelper(c, `{
1874     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1875     "container_image": "`+arvadostest.DockerImage112PDH+`",
1876     "cwd": ".",
1877     "environment": {},
1878     "mounts": {"/tmp": {"kind": "tmp"},
1879                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1880                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1881     "output_path": "/tmp",
1882     "priority": 1,
1883     "runtime_constraints": {},
1884     "state": "Locked"
1885 }`, nil, 1, func() {
1886                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1887                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1888         })
1889
1890         final := api.CalledWith("container.state", "Complete")
1891         c.Assert(final, NotNil)
1892         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1893         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1894
1895         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1896 }
1897
1898 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1899         s.api.callraw = true
1900         err := s.runner.fetchContainerRecord()
1901         c.Assert(err, IsNil)
1902         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1903         c.Logf("%#v", s.runner.Container)
1904         c.Check(err, IsNil)
1905         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1906 }
1907
1908 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1909         nextState := ""
1910         for _, setup := range []func(){
1911                 func() {
1912                         c.Log("// waitErr = ocl runtime error")
1913                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1914                         nextState = "Cancelled"
1915                 },
1916                 func() {
1917                         c.Log("// loadErr = cannot connect")
1918                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1919                         s.runner.brokenNodeHook = c.MkDir() + "/broken-node-hook"
1920                         err := ioutil.WriteFile(s.runner.brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1921                         c.Assert(err, IsNil)
1922                         nextState = "Queued"
1923                 },
1924         } {
1925                 s.SetUpTest(c)
1926                 setup()
1927                 s.fullRunHelper(c, `{
1928     "command": ["echo", "hello world"],
1929     "container_image": "`+arvadostest.DockerImage112PDH+`",
1930     "cwd": ".",
1931     "environment": {},
1932     "mounts": {"/tmp": {"kind": "tmp"} },
1933     "output_path": "/tmp",
1934     "priority": 1,
1935     "runtime_constraints": {},
1936     "state": "Locked"
1937 }`, nil, 0, func() {})
1938                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1939                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1940                 if s.runner.brokenNodeHook != "" {
1941                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1942                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1943                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1944                 } else {
1945                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1946                 }
1947         }
1948 }
1949
1950 func (s *TestSuite) TestBadCommand(c *C) {
1951         for _, startError := range []string{
1952                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1953                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1954                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1955         } {
1956                 s.SetUpTest(c)
1957                 s.executor.startErr = errors.New(startError)
1958                 s.fullRunHelper(c, `{
1959     "command": ["echo", "hello world"],
1960     "container_image": "`+arvadostest.DockerImage112PDH+`",
1961     "cwd": ".",
1962     "environment": {},
1963     "mounts": {"/tmp": {"kind": "tmp"} },
1964     "output_path": "/tmp",
1965     "priority": 1,
1966     "runtime_constraints": {},
1967     "state": "Locked"
1968 }`, nil, 0, func() {})
1969                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1970                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1971         }
1972 }
1973
1974 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1975         helperRecord := `{
1976                 "command": ["true"],
1977                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1978                 "cwd": "/bin",
1979                 "mounts": {
1980                     "/tmp": {"kind": "tmp"},
1981                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1982                 },
1983                 "secret_mounts": {
1984                 },
1985                 "output_path": "/tmp",
1986                 "priority": 1,
1987                 "runtime_constraints": {},
1988                 "state": "Locked"
1989         }`
1990
1991         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1992                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1993                 c.Check(err, IsNil)
1994                 c.Check(string(content), Equals, "mypassword")
1995         })
1996
1997         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1998         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1999         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
2000         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
2001
2002         // under secret mounts, not captured in output
2003         helperRecord = `{
2004                 "command": ["true"],
2005                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2006                 "cwd": "/bin",
2007                 "mounts": {
2008                     "/tmp": {"kind": "tmp"}
2009                 },
2010                 "secret_mounts": {
2011                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2012                 },
2013                 "output_path": "/tmp",
2014                 "priority": 1,
2015                 "runtime_constraints": {},
2016                 "state": "Locked"
2017         }`
2018
2019         s.SetUpTest(c)
2020         s.fullRunHelper(c, helperRecord, nil, 0, func() {
2021                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2022                 c.Check(err, IsNil)
2023                 c.Check(string(content), Equals, "mypassword")
2024         })
2025
2026         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2027         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2028         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2029         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2030
2031         // under secret mounts, output dir is a collection, not captured in output
2032         helperRecord = `{
2033                 "command": ["true"],
2034                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2035                 "cwd": "/bin",
2036                 "mounts": {
2037                     "/tmp": {"kind": "collection", "writable": true}
2038                 },
2039                 "secret_mounts": {
2040                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2041                 },
2042                 "output_path": "/tmp",
2043                 "priority": 1,
2044                 "runtime_constraints": {},
2045                 "state": "Locked"
2046         }`
2047
2048         s.SetUpTest(c)
2049         _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, 0, func() {
2050                 // secret.conf should be provisioned as a separate
2051                 // bind mount, i.e., it should not appear in the
2052                 // (fake) fuse filesystem as viewed from the host.
2053                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2054                 if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) {
2055                         c.Logf("secret.conf: content %q, err %#v", content, err)
2056                 }
2057                 err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700)
2058                 c.Check(err, IsNil)
2059         })
2060
2061         content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text")
2062         c.Check(err, IsNil)
2063         c.Check(string(content), Equals, "mypassword")
2064         c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true})
2065         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2066         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2067         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil)
2068 }
2069
2070 type FakeProcess struct {
2071         cmdLine []string
2072 }
2073
2074 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2075         return fp.cmdLine, nil
2076 }