lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/manifest"
  29         "golang.org/x/net/context"
  30
  31         . "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func TestCrunchExec(t *testing.T) {
  36         TestingT(t)
  37 }
  38
  39 var _ = Suite(&TestSuite{})
  40
  41 type TestSuite struct {
  42         client                   *arvados.Client
  43         api                      *ArvTestClient
  44         runner                   *ContainerRunner
  45         executor                 *stubExecutor
  46         keepmount                string
  47         testDispatcherKeepClient KeepTestClient
  48         testContainerKeepClient  KeepTestClient
  49 }
  50
  51 func (s *TestSuite) SetUpTest(c *C) {
  52         *brokenNodeHook = ""
  53         s.client = arvados.NewClientFromEnv()
  54         s.executor = &stubExecutor{}
  55         var err error
  56         s.api = &ArvTestClient{}
  57         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  58         c.Assert(err, IsNil)
  59         s.runner.executor = s.executor
  60         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  61                 return s.api, &s.testContainerKeepClient, s.client, nil
  62         }
  63         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  64                 s.runner.ArvMountPoint = s.keepmount
  65                 return nil, nil
  66         }
  67         s.keepmount = c.MkDir()
  68         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  69         c.Assert(err, IsNil)
  70         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  71         c.Assert(err, IsNil)
  72         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  73         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  74         c.Assert(err, IsNil)
  75         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  76         c.Assert(err, IsNil)
  77         s.runner.ArvMountPoint = s.keepmount
  78 }
  79
  80 type ArvTestClient struct {
  81         Total   int64
  82         Calls   int
  83         Content []arvadosclient.Dict
  84         arvados.Container
  85         secretMounts []byte
  86         Logs         map[string]*bytes.Buffer
  87         sync.Mutex
  88         WasSetRunning bool
  89         callraw       bool
  90 }
  91
  92 type KeepTestClient struct {
  93         Called         bool
  94         Content        []byte
  95         StorageClasses []string
  96 }
  97
  98 type stubExecutor struct {
  99         imageLoaded bool
 100         loaded      string
 101         loadErr     error
 102         exitCode    int
 103         createErr   error
 104         created     containerSpec
 105         startErr    error
 106         waitSleep   time.Duration
 107         waitErr     error
 108         stopErr     error
 109         stopped     bool
 110         closed      bool
 111         runFunc     func()
 112         exit        chan int
 113 }
 114
 115 func (e *stubExecutor) ImageLoaded(imageID string) bool { return e.imageLoaded }
 116 func (e *stubExecutor) LoadImage(filename string) error { e.loaded = filename; return e.loadErr }
 117 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 118 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 119 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 120 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 121 func (e *stubExecutor) Close()                          { e.closed = true }
 122 func (e *stubExecutor) Wait(context.Context) (int, error) {
 123         defer e.created.Stdout.Close()
 124         defer e.created.Stderr.Close()
 125         return <-e.exit, e.waitErr
 126 }
 127
 128 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 129
 130 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 131 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 132 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 133
 134 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 135 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 136
 137 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 138 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 139 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 140 `
 141
 142 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 143
 144 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 145 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 146
 147 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 148 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 149
 150 func (client *ArvTestClient) Create(resourceType string,
 151         parameters arvadosclient.Dict,
 152         output interface{}) error {
 153
 154         client.Mutex.Lock()
 155         defer client.Mutex.Unlock()
 156
 157         client.Calls++
 158         client.Content = append(client.Content, parameters)
 159
 160         if resourceType == "logs" {
 161                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 162                 if client.Logs == nil {
 163                         client.Logs = make(map[string]*bytes.Buffer)
 164                 }
 165                 if client.Logs[et] == nil {
 166                         client.Logs[et] = &bytes.Buffer{}
 167                 }
 168                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 169         }
 170
 171         if resourceType == "collections" && output != nil {
 172                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 173                 outmap := output.(*arvados.Collection)
 174                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 175                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 176         }
 177
 178         return nil
 179 }
 180
 181 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 182         switch {
 183         case method == "GET" && resourceType == "containers" && action == "auth":
 184                 return json.Unmarshal([]byte(`{
 185                         "kind": "arvados#api_client_authorization",
 186                         "uuid": "`+fakeAuthUUID+`",
 187                         "api_token": "`+fakeAuthToken+`"
 188                         }`), output)
 189         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 190                 if client.secretMounts != nil {
 191                         return json.Unmarshal(client.secretMounts, output)
 192                 }
 193                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 194         default:
 195                 return fmt.Errorf("Not found")
 196         }
 197 }
 198
 199 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 200         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 201         var j []byte
 202         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 203                 j = []byte(`{
 204                         "kind": "arvados#nodeList",
 205                         "items": [{
 206                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 207                                 "hostname": "compute2",
 208                                 "properties": {"total_cpu_cores": 16}
 209                         }]}`)
 210         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 211                 if uuid == "" {
 212                         j, err = json.Marshal(map[string]interface{}{
 213                                 "items": []interface{}{client.Container},
 214                                 "kind":  "arvados#nodeList",
 215                         })
 216                 } else {
 217                         j, err = json.Marshal(client.Container)
 218                 }
 219         } else {
 220                 j = []byte(`{
 221                         "command": ["sleep", "1"],
 222                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 223                         "cwd": ".",
 224                         "environment": {},
 225                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 226                         "output_path": "/tmp",
 227                         "priority": 1,
 228                         "runtime_constraints": {}
 229                 }`)
 230         }
 231         return ioutil.NopCloser(bytes.NewReader(j)), err
 232 }
 233
 234 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 235         if resourceType == "collections" {
 236                 if uuid == hwPDH {
 237                         output.(*arvados.Collection).ManifestText = hwManifest
 238                 } else if uuid == otherPDH {
 239                         output.(*arvados.Collection).ManifestText = otherManifest
 240                 } else if uuid == normalizedWithSubdirsPDH {
 241                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 242                 } else if uuid == denormalizedWithSubdirsPDH {
 243                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 244                 }
 245         }
 246         if resourceType == "containers" {
 247                 (*output.(*arvados.Container)) = client.Container
 248         }
 249         return nil
 250 }
 251
 252 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 253         client.Mutex.Lock()
 254         defer client.Mutex.Unlock()
 255         client.Calls++
 256         client.Content = append(client.Content, parameters)
 257         if resourceType == "containers" {
 258                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 259                         client.WasSetRunning = true
 260                 }
 261         } else if resourceType == "collections" {
 262                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 263                 output.(*arvados.Collection).UUID = uuid
 264                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 265         }
 266         return nil
 267 }
 268
 269 var discoveryMap = map[string]interface{}{
 270         "defaultTrashLifetime":               float64(1209600),
 271         "crunchLimitLogBytesPerJob":          float64(67108864),
 272         "crunchLogThrottleBytes":             float64(65536),
 273         "crunchLogThrottlePeriod":            float64(60),
 274         "crunchLogThrottleLines":             float64(1024),
 275         "crunchLogPartialLineThrottlePeriod": float64(5),
 276         "crunchLogBytesPerEvent":             float64(4096),
 277         "crunchLogSecondsBetweenEvents":      float64(1),
 278 }
 279
 280 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 281         return discoveryMap[key], nil
 282 }
 283
 284 // CalledWith returns the parameters from the first API call whose
 285 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 286 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 287 // no call matches, it returns nil.
 288 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 289 call:
 290         for _, content := range client.Content {
 291                 var v interface{} = content
 292                 for _, k := range strings.Split(jpath, ".") {
 293                         if dict, ok := v.(arvadosclient.Dict); !ok {
 294                                 continue call
 295                         } else {
 296                                 v = dict[k]
 297                         }
 298                 }
 299                 if v == expect {
 300                         return content
 301                 }
 302         }
 303         return nil
 304 }
 305
 306 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 307         return locator, nil
 308 }
 309
 310 func (client *KeepTestClient) PutB(buf []byte) (string, int, error) {
 311         client.Content = buf
 312         return fmt.Sprintf("%x+%d", md5.Sum(buf), len(buf)), len(buf), nil
 313 }
 314
 315 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 316         return 0, errors.New("not implemented")
 317 }
 318
 319 func (client *KeepTestClient) ClearBlockCache() {
 320 }
 321
 322 func (client *KeepTestClient) Close() {
 323         client.Content = nil
 324 }
 325
 326 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 327         client.StorageClasses = sc
 328 }
 329
 330 type FileWrapper struct {
 331         io.ReadCloser
 332         len int64
 333 }
 334
 335 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 336         return nil, errors.New("not implemented")
 337 }
 338
 339 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 340         return 0, errors.New("not implemented")
 341 }
 342
 343 func (fw FileWrapper) Size() int64 {
 344         return fw.len
 345 }
 346
 347 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 348         return nil, errors.New("not implemented")
 349 }
 350
 351 func (fw FileWrapper) Truncate(int64) error {
 352         return errors.New("not implemented")
 353 }
 354
 355 func (fw FileWrapper) Write([]byte) (int, error) {
 356         return 0, errors.New("not implemented")
 357 }
 358
 359 func (fw FileWrapper) Sync() error {
 360         return errors.New("not implemented")
 361 }
 362
 363 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 364         if filename == hwImageID+".tar" {
 365                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 366                 client.Called = true
 367                 return FileWrapper{rdr, 1321984}, nil
 368         } else if filename == "/file1_in_main.txt" {
 369                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 370                 client.Called = true
 371                 return FileWrapper{rdr, 3}, nil
 372         }
 373         return nil, nil
 374 }
 375
 376 func (s *TestSuite) TestLoadImage(c *C) {
 377         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 378         s.runner.Container.Mounts = map[string]arvados.Mount{
 379                 "/out": {Kind: "tmp", Writable: true},
 380         }
 381         s.runner.Container.OutputPath = "/out"
 382
 383         _, err := s.runner.SetupMounts()
 384         c.Assert(err, IsNil)
 385
 386         imageID, err := s.runner.LoadImage()
 387         c.Check(err, IsNil)
 388         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 389         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 390
 391         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 392         s.executor.imageLoaded = false
 393         s.executor.loaded = ""
 394         s.executor.loadErr = errors.New("bork")
 395         imageID, err = s.runner.LoadImage()
 396         c.Check(err, ErrorMatches, ".*bork")
 397         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 398
 399         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 400         s.executor.imageLoaded = false
 401         s.executor.loaded = ""
 402         s.executor.loadErr = nil
 403         imageID, err = s.runner.LoadImage()
 404         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 405         c.Check(s.executor.loaded, Equals, "")
 406
 407         // if executor reports image is already loaded, LoadImage should not be called
 408         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 409         s.executor.imageLoaded = true
 410         s.executor.loaded = ""
 411         s.executor.loadErr = nil
 412         imageID, err = s.runner.LoadImage()
 413         c.Check(err, IsNil)
 414         c.Check(s.executor.loaded, Equals, "")
 415         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 416 }
 417
 418 type ArvErrorTestClient struct{}
 419
 420 func (ArvErrorTestClient) Create(resourceType string,
 421         parameters arvadosclient.Dict,
 422         output interface{}) error {
 423         return nil
 424 }
 425
 426 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 427         if method == "GET" && resourceType == "containers" && action == "auth" {
 428                 return nil
 429         }
 430         return errors.New("ArvError")
 431 }
 432
 433 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 434         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 435         return nil, errors.New("ArvError")
 436 }
 437
 438 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 439         return errors.New("ArvError")
 440 }
 441
 442 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 443         return nil
 444 }
 445
 446 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 447         return discoveryMap[key], nil
 448 }
 449
 450 type KeepErrorTestClient struct {
 451         KeepTestClient
 452 }
 453
 454 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 455         return nil, errors.New("KeepError")
 456 }
 457
 458 func (*KeepErrorTestClient) PutB(buf []byte) (string, int, error) {
 459         return "", 0, errors.New("KeepError")
 460 }
 461
 462 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 463         return "", errors.New("KeepError")
 464 }
 465
 466 type KeepReadErrorTestClient struct {
 467         KeepTestClient
 468 }
 469
 470 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 471         return 0, errors.New("KeepError")
 472 }
 473
 474 type ErrorReader struct {
 475         FileWrapper
 476 }
 477
 478 func (ErrorReader) Read(p []byte) (n int, err error) {
 479         return 0, errors.New("ErrorReader")
 480 }
 481
 482 func (ErrorReader) Seek(int64, int) (int64, error) {
 483         return 0, errors.New("ErrorReader")
 484 }
 485
 486 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 487         return ErrorReader{}, nil
 488 }
 489
 490 type ClosableBuffer struct {
 491         bytes.Buffer
 492 }
 493
 494 func (*ClosableBuffer) Close() error {
 495         return nil
 496 }
 497
 498 type TestLogs struct {
 499         Stdout ClosableBuffer
 500         Stderr ClosableBuffer
 501 }
 502
 503 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 504         if logstr == "stdout" {
 505                 return &tl.Stdout, nil
 506         }
 507         if logstr == "stderr" {
 508                 return &tl.Stderr, nil
 509         }
 510         return nil, errors.New("???")
 511 }
 512
 513 func dockerLog(fd byte, msg string) []byte {
 514         by := []byte(msg)
 515         header := make([]byte, 8+len(by))
 516         header[0] = fd
 517         header[7] = byte(len(by))
 518         copy(header[8:], by)
 519         return header
 520 }
 521
 522 func (s *TestSuite) TestRunContainer(c *C) {
 523         s.executor.runFunc = func() {
 524                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 525                 s.executor.created.Stdout.Close()
 526                 s.executor.created.Stderr.Close()
 527                 s.executor.exit <- 0
 528         }
 529
 530         var logs TestLogs
 531         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 532         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 533         s.runner.Container.Command = []string{"./hw"}
 534         s.runner.Container.OutputStorageClasses = []string{"default"}
 535
 536         imageID, err := s.runner.LoadImage()
 537         c.Assert(err, IsNil)
 538
 539         err = s.runner.CreateContainer(imageID, nil)
 540         c.Assert(err, IsNil)
 541
 542         err = s.runner.StartContainer()
 543         c.Assert(err, IsNil)
 544
 545         err = s.runner.WaitFinish()
 546         c.Assert(err, IsNil)
 547
 548         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 549         c.Check(logs.Stderr.String(), Equals, "")
 550         c.Check(s.testDispatcherKeepClient, Equals, []string{"default"})
 551         c.Check(s.testContainerKeepClient, Equals, []string{"default"})
 552 }
 553
 554 func (s *TestSuite) TestCommitLogs(c *C) {
 555         api := &ArvTestClient{}
 556         kc := &KeepTestClient{}
 557         defer kc.Close()
 558         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 559         c.Assert(err, IsNil)
 560         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 561
 562         cr.CrunchLog.Print("Hello world!")
 563         cr.CrunchLog.Print("Goodbye")
 564         cr.finalState = "Complete"
 565
 566         err = cr.CommitLogs()
 567         c.Check(err, IsNil)
 568
 569         c.Check(api.Calls, Equals, 2)
 570         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 571         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 572         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 573         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 574 }
 575
 576 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 577         api := &ArvTestClient{}
 578         kc := &KeepTestClient{}
 579         defer kc.Close()
 580         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 581         c.Assert(err, IsNil)
 582
 583         err = cr.UpdateContainerRunning()
 584         c.Check(err, IsNil)
 585
 586         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 587 }
 588
 589 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 590         api := &ArvTestClient{}
 591         kc := &KeepTestClient{}
 592         defer kc.Close()
 593         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 594         c.Assert(err, IsNil)
 595
 596         cr.LogsPDH = new(string)
 597         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 598
 599         cr.ExitCode = new(int)
 600         *cr.ExitCode = 42
 601         cr.finalState = "Complete"
 602
 603         err = cr.UpdateContainerFinal()
 604         c.Check(err, IsNil)
 605
 606         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 607         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 608         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 609 }
 610
 611 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 612         api := &ArvTestClient{}
 613         kc := &KeepTestClient{}
 614         defer kc.Close()
 615         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 616         c.Assert(err, IsNil)
 617         cr.cCancelled = true
 618         cr.finalState = "Cancelled"
 619
 620         err = cr.UpdateContainerFinal()
 621         c.Check(err, IsNil)
 622
 623         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 624         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 625         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 626 }
 627
 628 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 629 // dress rehearsal of the Run() function, starting from a JSON container record.
 630 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 631         err := json.Unmarshal([]byte(record), &s.api.Container)
 632         c.Assert(err, IsNil)
 633         initialState := s.api.Container.State
 634
 635         var sm struct {
 636                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 637         }
 638         err = json.Unmarshal([]byte(record), &sm)
 639         c.Check(err, IsNil)
 640         secretMounts, err := json.Marshal(sm)
 641         c.Assert(err, IsNil)
 642         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 643
 644         s.executor.runFunc = func() {
 645                 fn()
 646                 s.executor.exit <- exitCode
 647         }
 648
 649         s.runner.statInterval = 100 * time.Millisecond
 650         s.runner.containerWatchdogInterval = time.Second
 651         am := &ArvMountCmdLine{}
 652         s.runner.RunArvMount = am.ArvMountTest
 653
 654         realTemp := c.MkDir()
 655         tempcount := 0
 656         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 657                 tempcount++
 658                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 659                 err := os.Mkdir(d, os.ModePerm)
 660                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 661                         // Test case must have pre-populated the tempdir
 662                         err = nil
 663                 }
 664                 return d, err
 665         }
 666         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 667                 return &ArvTestClient{secretMounts: secretMounts}, &KeepTestClient{}, nil, nil
 668         }
 669
 670         if extraMounts != nil && len(extraMounts) > 0 {
 671                 err := s.runner.SetupArvMountPoint("keep")
 672                 c.Check(err, IsNil)
 673
 674                 for _, m := range extraMounts {
 675                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 676                 }
 677         }
 678
 679         err = s.runner.Run()
 680         if s.api.CalledWith("container.state", "Complete") != nil {
 681                 c.Check(err, IsNil)
 682         }
 683         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 684                 c.Check(s.api.WasSetRunning, Equals, true)
 685                 var lastupdate arvadosclient.Dict
 686                 for _, content := range s.api.Content {
 687                         if content["container"] != nil {
 688                                 lastupdate = content["container"].(arvadosclient.Dict)
 689                         }
 690                 }
 691                 if lastupdate["log"] == nil {
 692                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 693                 }
 694         }
 695
 696         if err != nil {
 697                 for k, v := range s.api.Logs {
 698                         c.Log(k)
 699                         c.Log(v.String())
 700                 }
 701         }
 702
 703         return s.api, s.runner, realTemp
 704 }
 705
 706 func (s *TestSuite) TestFullRunHello(c *C) {
 707         s.runner.enableMemoryLimit = true
 708         s.runner.networkMode = "default"
 709         s.fullRunHelper(c, `{
 710     "command": ["echo", "hello world"],
 711     "container_image": "`+arvadostest.DockerImage112PDH+`",
 712     "cwd": ".",
 713     "environment": {"foo":"bar","baz":"waz"},
 714     "mounts": {"/tmp": {"kind": "tmp"} },
 715     "output_path": "/tmp",
 716     "priority": 1,
 717     "runtime_constraints": {"vcpus":1,"ram":1000000},
 718     "state": "Locked"
 719 }`, nil, 0, func() {
 720                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 721                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 722                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 723                 c.Check(s.executor.created.VCPUs, Equals, 1)
 724                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 725                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 726                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 727                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 728         })
 729
 730         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 731         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 732         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 733
 734 }
 735
 736 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 737         var ran bool
 738         s.fullRunHelper(c, `{
 739     "command": ["sleep", "3"],
 740     "container_image": "`+arvadostest.DockerImage112PDH+`",
 741     "cwd": ".",
 742     "environment": {},
 743     "mounts": {"/tmp": {"kind": "tmp"} },
 744     "output_path": "/tmp",
 745     "priority": 1,
 746     "runtime_constraints": {},
 747     "scheduling_parameters":{"max_run_time": 1},
 748     "state": "Running"
 749 }`, nil, 2, func() {
 750                 ran = true
 751         })
 752         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 753         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 754         c.Check(ran, Equals, false)
 755 }
 756
 757 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 758         s.fullRunHelper(c, `{
 759     "command": ["sleep", "3"],
 760     "container_image": "`+arvadostest.DockerImage112PDH+`",
 761     "cwd": ".",
 762     "environment": {},
 763     "mounts": {"/tmp": {"kind": "tmp"} },
 764     "output_path": "/tmp",
 765     "priority": 1,
 766     "runtime_constraints": {},
 767     "scheduling_parameters":{"max_run_time": 1},
 768     "state": "Locked"
 769 }`, nil, 0, func() {
 770                 time.Sleep(3 * time.Second)
 771         })
 772
 773         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 774         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 775 }
 776
 777 func (s *TestSuite) TestContainerWaitFails(c *C) {
 778         s.fullRunHelper(c, `{
 779     "command": ["sleep", "3"],
 780     "container_image": "`+arvadostest.DockerImage112PDH+`",
 781     "cwd": ".",
 782     "mounts": {"/tmp": {"kind": "tmp"} },
 783     "output_path": "/tmp",
 784     "priority": 1,
 785     "state": "Locked"
 786 }`, nil, 0, func() {
 787                 s.executor.waitErr = errors.New("Container is not running")
 788         })
 789
 790         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 791         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 792 }
 793
 794 func (s *TestSuite) TestCrunchstat(c *C) {
 795         s.fullRunHelper(c, `{
 796                 "command": ["sleep", "1"],
 797                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 798                 "cwd": ".",
 799                 "environment": {},
 800                 "mounts": {"/tmp": {"kind": "tmp"} },
 801                 "output_path": "/tmp",
 802                 "priority": 1,
 803                 "runtime_constraints": {},
 804                 "state": "Locked"
 805         }`, nil, 0, func() {
 806                 time.Sleep(time.Second)
 807         })
 808
 809         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 810         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 811
 812         // We didn't actually start a container, so crunchstat didn't
 813         // find accounting files and therefore didn't log any stats.
 814         // It should have logged a "can't find accounting files"
 815         // message after one poll interval, though, so we can confirm
 816         // it's alive:
 817         c.Assert(s.api.Logs["crunchstat"], NotNil)
 818         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 819
 820         // The "files never appeared" log assures us that we called
 821         // (*crunchstat.Reporter)Stop(), and that we set it up with
 822         // the correct container ID "abcde":
 823         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 824 }
 825
 826 func (s *TestSuite) TestNodeInfoLog(c *C) {
 827         os.Setenv("SLURMD_NODENAME", "compute2")
 828         s.fullRunHelper(c, `{
 829                 "command": ["sleep", "1"],
 830                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 831                 "cwd": ".",
 832                 "environment": {},
 833                 "mounts": {"/tmp": {"kind": "tmp"} },
 834                 "output_path": "/tmp",
 835                 "priority": 1,
 836                 "runtime_constraints": {},
 837                 "state": "Locked"
 838         }`, nil, 0,
 839                 func() {
 840                         time.Sleep(time.Second)
 841                 })
 842
 843         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 844         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 845
 846         c.Assert(s.api.Logs["node"], NotNil)
 847         json := s.api.Logs["node"].String()
 848         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 849         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 850         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 851
 852         c.Assert(s.api.Logs["node-info"], NotNil)
 853         json = s.api.Logs["node-info"].String()
 854         c.Check(json, Matches, `(?ms).*Host Information.*`)
 855         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 856         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 857         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 858         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 859 }
 860
 861 func (s *TestSuite) TestContainerRecordLog(c *C) {
 862         s.fullRunHelper(c, `{
 863                 "command": ["sleep", "1"],
 864                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 865                 "cwd": ".",
 866                 "environment": {},
 867                 "mounts": {"/tmp": {"kind": "tmp"} },
 868                 "output_path": "/tmp",
 869                 "priority": 1,
 870                 "runtime_constraints": {},
 871                 "state": "Locked"
 872         }`, nil, 0,
 873                 func() {
 874                         time.Sleep(time.Second)
 875                 })
 876
 877         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 878         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 879
 880         c.Assert(s.api.Logs["container"], NotNil)
 881         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 882 }
 883
 884 func (s *TestSuite) TestFullRunStderr(c *C) {
 885         s.fullRunHelper(c, `{
 886     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 887     "container_image": "`+arvadostest.DockerImage112PDH+`",
 888     "cwd": ".",
 889     "environment": {},
 890     "mounts": {"/tmp": {"kind": "tmp"} },
 891     "output_path": "/tmp",
 892     "priority": 1,
 893     "runtime_constraints": {},
 894     "state": "Locked"
 895 }`, nil, 1, func() {
 896                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 897                 fmt.Fprintln(s.executor.created.Stderr, "world")
 898         })
 899
 900         final := s.api.CalledWith("container.state", "Complete")
 901         c.Assert(final, NotNil)
 902         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 903         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 904
 905         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 906         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 907 }
 908
 909 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 910         s.fullRunHelper(c, `{
 911     "command": ["pwd"],
 912     "container_image": "`+arvadostest.DockerImage112PDH+`",
 913     "cwd": ".",
 914     "environment": {},
 915     "mounts": {"/tmp": {"kind": "tmp"} },
 916     "output_path": "/tmp",
 917     "priority": 1,
 918     "runtime_constraints": {},
 919     "state": "Locked"
 920 }`, nil, 0, func() {
 921                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 922         })
 923
 924         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 925         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 926         c.Log(s.api.Logs["stdout"])
 927         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 928 }
 929
 930 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 931         s.fullRunHelper(c, `{
 932     "command": ["pwd"],
 933     "container_image": "`+arvadostest.DockerImage112PDH+`",
 934     "cwd": "/bin",
 935     "environment": {},
 936     "mounts": {"/tmp": {"kind": "tmp"} },
 937     "output_path": "/tmp",
 938     "priority": 1,
 939     "runtime_constraints": {},
 940     "state": "Locked"
 941 }`, nil, 0, func() {
 942                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 943         })
 944
 945         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 946         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 947         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 948 }
 949
 950 func (s *TestSuite) TestStopOnSignal(c *C) {
 951         s.executor.runFunc = func() {
 952                 s.executor.created.Stdout.Write([]byte("foo\n"))
 953                 s.runner.SigChan <- syscall.SIGINT
 954         }
 955         s.testStopContainer(c)
 956 }
 957
 958 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
 959         s.executor.runFunc = func() {
 960                 s.executor.created.Stdout.Write([]byte("foo\n"))
 961                 s.runner.ArvMountExit <- nil
 962                 close(s.runner.ArvMountExit)
 963         }
 964         s.runner.ArvMountExit = make(chan error)
 965         s.testStopContainer(c)
 966 }
 967
 968 func (s *TestSuite) testStopContainer(c *C) {
 969         record := `{
 970     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
 971     "container_image": "` + arvadostest.DockerImage112PDH + `",
 972     "cwd": ".",
 973     "environment": {},
 974     "mounts": {"/tmp": {"kind": "tmp"} },
 975     "output_path": "/tmp",
 976     "priority": 1,
 977     "runtime_constraints": {},
 978     "state": "Locked"
 979 }`
 980
 981         err := json.Unmarshal([]byte(record), &s.api.Container)
 982         c.Assert(err, IsNil)
 983
 984         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
 985         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 986                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
 987         }
 988
 989         done := make(chan error)
 990         go func() {
 991                 done <- s.runner.Run()
 992         }()
 993         select {
 994         case <-time.After(20 * time.Second):
 995                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
 996                 c.Fatal("timed out")
 997         case err = <-done:
 998                 c.Check(err, IsNil)
 999         }
1000         for k, v := range s.api.Logs {
1001                 c.Log(k)
1002                 c.Log(v.String(), "\n")
1003         }
1004
1005         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1006         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1007         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1008 }
1009
1010 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1011         s.fullRunHelper(c, `{
1012     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1013     "container_image": "`+arvadostest.DockerImage112PDH+`",
1014     "cwd": "/bin",
1015     "environment": {"FROBIZ": "bilbo"},
1016     "mounts": {"/tmp": {"kind": "tmp"} },
1017     "output_path": "/tmp",
1018     "priority": 1,
1019     "runtime_constraints": {},
1020     "state": "Locked"
1021 }`, nil, 0, func() {
1022                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1023         })
1024
1025         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1026         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1027         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1028 }
1029
1030 type ArvMountCmdLine struct {
1031         Cmd   []string
1032         token string
1033 }
1034
1035 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1036         am.Cmd = c
1037         am.token = token
1038         return nil, nil
1039 }
1040
1041 func stubCert(temp string) string {
1042         path := temp + "/ca-certificates.crt"
1043         crt, _ := os.Create(path)
1044         crt.Close()
1045         arvadosclient.CertFiles = []string{path}
1046         return path
1047 }
1048
1049 func (s *TestSuite) TestSetupMounts(c *C) {
1050         cr := s.runner
1051         am := &ArvMountCmdLine{}
1052         cr.RunArvMount = am.ArvMountTest
1053         cr.ContainerArvClient = &ArvTestClient{}
1054         cr.ContainerKeepClient = &KeepTestClient{}
1055         cr.Container.OutputStorageClasses = []string{"default"}
1056
1057         realTemp := c.MkDir()
1058         certTemp := c.MkDir()
1059         stubCertPath := stubCert(certTemp)
1060         cr.parentTemp = realTemp
1061
1062         i := 0
1063         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1064                 i++
1065                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1066                 err := os.Mkdir(d, os.ModePerm)
1067                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1068                         // Test case must have pre-populated the tempdir
1069                         err = nil
1070                 }
1071                 return d, err
1072         }
1073
1074         checkEmpty := func() {
1075                 // Should be deleted.
1076                 _, err := os.Stat(realTemp)
1077                 c.Assert(os.IsNotExist(err), Equals, true)
1078
1079                 // Now recreate it for the next test.
1080                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1081         }
1082
1083         {
1084                 i = 0
1085                 cr.ArvMountPoint = ""
1086                 cr.Container.Mounts = make(map[string]arvados.Mount)
1087                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1088                 cr.Container.OutputPath = "/tmp"
1089                 cr.statInterval = 5 * time.Second
1090                 bindmounts, err := cr.SetupMounts()
1091                 c.Check(err, IsNil)
1092                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1093                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1094                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1095                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1096                 os.RemoveAll(cr.ArvMountPoint)
1097                 cr.CleanupDirs()
1098                 checkEmpty()
1099         }
1100
1101         {
1102                 i = 0
1103                 cr.ArvMountPoint = ""
1104                 cr.Container.Mounts = make(map[string]arvados.Mount)
1105                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1106                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1107                 cr.Container.OutputPath = "/out"
1108                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1109
1110                 bindmounts, err := cr.SetupMounts()
1111                 c.Check(err, IsNil)
1112                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1113                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1114                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1115                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1116                 os.RemoveAll(cr.ArvMountPoint)
1117                 cr.CleanupDirs()
1118                 checkEmpty()
1119         }
1120
1121         {
1122                 i = 0
1123                 cr.ArvMountPoint = ""
1124                 cr.Container.Mounts = make(map[string]arvados.Mount)
1125                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1126                 cr.Container.OutputPath = "/tmp"
1127                 cr.Container.RuntimeConstraints.API = true
1128                 cr.Container.OutputStorageClasses = []string{"default"}
1129
1130                 bindmounts, err := cr.SetupMounts()
1131                 c.Check(err, IsNil)
1132                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1133                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1134                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1135                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1136                 os.RemoveAll(cr.ArvMountPoint)
1137                 cr.CleanupDirs()
1138                 checkEmpty()
1139
1140                 cr.Container.RuntimeConstraints.API = false
1141         }
1142
1143         {
1144                 i = 0
1145                 cr.ArvMountPoint = ""
1146                 cr.Container.Mounts = map[string]arvados.Mount{
1147                         "/keeptmp": {Kind: "collection", Writable: true},
1148                 }
1149                 cr.Container.OutputPath = "/keeptmp"
1150
1151                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1152
1153                 bindmounts, err := cr.SetupMounts()
1154                 c.Check(err, IsNil)
1155                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1156                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1157                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1158                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1159                 os.RemoveAll(cr.ArvMountPoint)
1160                 cr.CleanupDirs()
1161                 checkEmpty()
1162         }
1163
1164         {
1165                 i = 0
1166                 cr.ArvMountPoint = ""
1167                 cr.Container.Mounts = map[string]arvados.Mount{
1168                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1169                         "/keepout": {Kind: "collection", Writable: true},
1170                 }
1171                 cr.Container.OutputPath = "/keepout"
1172
1173                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1174                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1175
1176                 bindmounts, err := cr.SetupMounts()
1177                 c.Check(err, IsNil)
1178                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1179                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1180                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1181                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1182                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1183                         "/keepout": {realTemp + "/keep1/tmp0", false},
1184                 })
1185                 os.RemoveAll(cr.ArvMountPoint)
1186                 cr.CleanupDirs()
1187                 checkEmpty()
1188         }
1189
1190         {
1191                 i = 0
1192                 cr.ArvMountPoint = ""
1193                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1194                 cr.Container.Mounts = map[string]arvados.Mount{
1195                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1196                         "/keepout": {Kind: "collection", Writable: true},
1197                 }
1198                 cr.Container.OutputPath = "/keepout"
1199
1200                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1201                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1202
1203                 bindmounts, err := cr.SetupMounts()
1204                 c.Check(err, IsNil)
1205                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1206                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1207                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1208                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1209                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1210                         "/keepout": {realTemp + "/keep1/tmp0", false},
1211                 })
1212                 os.RemoveAll(cr.ArvMountPoint)
1213                 cr.CleanupDirs()
1214                 checkEmpty()
1215         }
1216
1217         for _, test := range []struct {
1218                 in  interface{}
1219                 out string
1220         }{
1221                 {in: "foo", out: `"foo"`},
1222                 {in: nil, out: `null`},
1223                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1224         } {
1225                 i = 0
1226                 cr.ArvMountPoint = ""
1227                 cr.Container.Mounts = map[string]arvados.Mount{
1228                         "/mnt/test.json": {Kind: "json", Content: test.in},
1229                 }
1230                 bindmounts, err := cr.SetupMounts()
1231                 c.Check(err, IsNil)
1232                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1233                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1234                 })
1235                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1236                 c.Check(err, IsNil)
1237                 c.Check(content, DeepEquals, []byte(test.out))
1238                 os.RemoveAll(cr.ArvMountPoint)
1239                 cr.CleanupDirs()
1240                 checkEmpty()
1241         }
1242
1243         for _, test := range []struct {
1244                 in  interface{}
1245                 out string
1246         }{
1247                 {in: "foo", out: `foo`},
1248                 {in: nil, out: "error"},
1249                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1250         } {
1251                 i = 0
1252                 cr.ArvMountPoint = ""
1253                 cr.Container.Mounts = map[string]arvados.Mount{
1254                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1255                 }
1256                 bindmounts, err := cr.SetupMounts()
1257                 if test.out == "error" {
1258                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1259                 } else {
1260                         c.Check(err, IsNil)
1261                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1262                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1263                         })
1264                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1265                         c.Check(err, IsNil)
1266                         c.Check(content, DeepEquals, []byte(test.out))
1267                 }
1268                 os.RemoveAll(cr.ArvMountPoint)
1269                 cr.CleanupDirs()
1270                 checkEmpty()
1271         }
1272
1273         // Read-only mount points are allowed underneath output_dir mount point
1274         {
1275                 i = 0
1276                 cr.ArvMountPoint = ""
1277                 cr.Container.Mounts = make(map[string]arvados.Mount)
1278                 cr.Container.Mounts = map[string]arvados.Mount{
1279                         "/tmp":     {Kind: "tmp"},
1280                         "/tmp/foo": {Kind: "collection"},
1281                 }
1282                 cr.Container.OutputPath = "/tmp"
1283
1284                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1285
1286                 bindmounts, err := cr.SetupMounts()
1287                 c.Check(err, IsNil)
1288                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1289                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1290                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1291                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1292                         "/tmp":     {realTemp + "/tmp2", false},
1293                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1294                 })
1295                 os.RemoveAll(cr.ArvMountPoint)
1296                 cr.CleanupDirs()
1297                 checkEmpty()
1298         }
1299
1300         // Writable mount points copied to output_dir mount point
1301         {
1302                 i = 0
1303                 cr.ArvMountPoint = ""
1304                 cr.Container.Mounts = make(map[string]arvados.Mount)
1305                 cr.Container.Mounts = map[string]arvados.Mount{
1306                         "/tmp": {Kind: "tmp"},
1307                         "/tmp/foo": {Kind: "collection",
1308                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1309                                 Writable:         true},
1310                         "/tmp/bar": {Kind: "collection",
1311                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1312                                 Path:             "baz",
1313                                 Writable:         true},
1314                 }
1315                 cr.Container.OutputPath = "/tmp"
1316
1317                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1318                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1319
1320                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1321                 rf.Write([]byte("bar"))
1322                 rf.Close()
1323
1324                 _, err := cr.SetupMounts()
1325                 c.Check(err, IsNil)
1326                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1327                 c.Check(err, IsNil)
1328                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1329                 c.Check(err, IsNil)
1330                 os.RemoveAll(cr.ArvMountPoint)
1331                 cr.CleanupDirs()
1332                 checkEmpty()
1333         }
1334
1335         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1336         {
1337                 i = 0
1338                 cr.ArvMountPoint = ""
1339                 cr.Container.Mounts = make(map[string]arvados.Mount)
1340                 cr.Container.Mounts = map[string]arvados.Mount{
1341                         "/tmp":     {Kind: "tmp"},
1342                         "/tmp/foo": {Kind: "tmp"},
1343                 }
1344                 cr.Container.OutputPath = "/tmp"
1345
1346                 _, err := cr.SetupMounts()
1347                 c.Check(err, NotNil)
1348                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1349                 os.RemoveAll(cr.ArvMountPoint)
1350                 cr.CleanupDirs()
1351                 checkEmpty()
1352         }
1353
1354         // Only mount point of kind 'collection' is allowed for stdin
1355         {
1356                 i = 0
1357                 cr.ArvMountPoint = ""
1358                 cr.Container.Mounts = make(map[string]arvados.Mount)
1359                 cr.Container.Mounts = map[string]arvados.Mount{
1360                         "stdin": {Kind: "tmp"},
1361                 }
1362
1363                 _, err := cr.SetupMounts()
1364                 c.Check(err, NotNil)
1365                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1366                 os.RemoveAll(cr.ArvMountPoint)
1367                 cr.CleanupDirs()
1368                 checkEmpty()
1369         }
1370
1371         // git_tree mounts
1372         {
1373                 i = 0
1374                 cr.ArvMountPoint = ""
1375                 (*GitMountSuite)(nil).useTestGitServer(c)
1376                 cr.token = arvadostest.ActiveToken
1377                 cr.Container.Mounts = make(map[string]arvados.Mount)
1378                 cr.Container.Mounts = map[string]arvados.Mount{
1379                         "/tip": {
1380                                 Kind:   "git_tree",
1381                                 UUID:   arvadostest.Repository2UUID,
1382                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1383                                 Path:   "/",
1384                         },
1385                         "/non-tip": {
1386                                 Kind:   "git_tree",
1387                                 UUID:   arvadostest.Repository2UUID,
1388                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1389                                 Path:   "/",
1390                         },
1391                 }
1392                 cr.Container.OutputPath = "/tmp"
1393
1394                 bindmounts, err := cr.SetupMounts()
1395                 c.Check(err, IsNil)
1396
1397                 for path, mount := range bindmounts {
1398                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1399                 }
1400
1401                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1402                 c.Check(err, IsNil)
1403                 c.Check(string(data), Equals, "\000\001\002\003")
1404                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1405                 c.Check(err, FitsTypeOf, &os.PathError{})
1406                 c.Check(os.IsNotExist(err), Equals, true)
1407
1408                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1409                 c.Check(err, IsNil)
1410                 c.Check(string(data), Equals, "\000\001\002\003")
1411                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1412                 c.Check(err, IsNil)
1413                 c.Check(string(data), Equals, "testfile\n")
1414
1415                 cr.CleanupDirs()
1416                 checkEmpty()
1417         }
1418 }
1419
1420 func (s *TestSuite) TestStdout(c *C) {
1421         helperRecord := `{
1422                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1423                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1424                 "cwd": "/bin",
1425                 "environment": {"FROBIZ": "bilbo"},
1426                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1427                 "output_path": "/tmp",
1428                 "priority": 1,
1429                 "runtime_constraints": {},
1430                 "state": "Locked"
1431         }`
1432
1433         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1434                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1435         })
1436
1437         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1438         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1439         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1440 }
1441
1442 // Used by the TestStdoutWithWrongPath*()
1443 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1444         err := json.Unmarshal([]byte(record), &s.api.Container)
1445         c.Assert(err, IsNil)
1446         s.executor.runFunc = fn
1447         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1448         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1449                 return s.api, &KeepTestClient{}, nil, nil
1450         }
1451         return s.api, s.runner, s.runner.Run()
1452 }
1453
1454 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1455         _, _, err := s.stdoutErrorRunHelper(c, `{
1456     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1457     "output_path": "/tmp",
1458     "state": "Locked"
1459 }`, func() {})
1460         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1461 }
1462
1463 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1464         _, _, err := s.stdoutErrorRunHelper(c, `{
1465     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1466     "output_path": "/tmp",
1467     "state": "Locked"
1468 }`, func() {})
1469         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1470 }
1471
1472 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1473         _, _, err := s.stdoutErrorRunHelper(c, `{
1474     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1475     "output_path": "/tmp",
1476     "state": "Locked"
1477 }`, func() {})
1478         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1479 }
1480
1481 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1482         s.fullRunHelper(c, `{
1483     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1484     "container_image": "`+arvadostest.DockerImage112PDH+`",
1485     "cwd": "/bin",
1486     "environment": {},
1487     "mounts": {"/tmp": {"kind": "tmp"} },
1488     "output_path": "/tmp",
1489     "priority": 1,
1490     "runtime_constraints": {"API": true},
1491     "state": "Locked"
1492 }`, nil, 0, func() {
1493                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1494                 s.executor.exit <- 3
1495         })
1496         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1497         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1498 }
1499
1500 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1501         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1502         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1503         s.fullRunHelper(c, `{
1504     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1505     "container_image": "`+arvadostest.DockerImage112PDH+`",
1506     "cwd": "/bin",
1507     "environment": {},
1508     "mounts": {"/tmp": {"kind": "tmp"} },
1509     "output_path": "/tmp",
1510     "priority": 1,
1511     "runtime_constraints": {"API": true},
1512     "state": "Locked"
1513 }`, nil, 0, func() {
1514                 s.api.Container.Output = arvadostest.DockerImage112PDH
1515         })
1516
1517         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1518         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1519         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1520 }
1521
1522 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1523         helperRecord := `{
1524                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1525                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1526                 "cwd": "/bin",
1527                 "environment": {"FROBIZ": "bilbo"},
1528                 "mounts": {
1529         "/tmp": {"kind": "tmp"},
1530         "/tmp/foo": {"kind": "collection",
1531                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1532                      "exclude_from_output": true
1533         },
1534         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1535     },
1536                 "output_path": "/tmp",
1537                 "priority": 1,
1538                 "runtime_constraints": {},
1539                 "state": "Locked"
1540         }`
1541
1542         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1543
1544         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1545                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1546         })
1547
1548         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1549         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1550         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1551 }
1552
1553 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1554         helperRecord := `{
1555                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1556                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1557                 "cwd": "/bin",
1558                 "environment": {"FROBIZ": "bilbo"},
1559                 "mounts": {
1560         "/tmp": {"kind": "tmp"},
1561         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1562         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1563         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1564         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1565         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1566     },
1567                 "output_path": "/tmp",
1568                 "priority": 1,
1569                 "runtime_constraints": {},
1570                 "state": "Locked"
1571         }`
1572
1573         extraMounts := []string{
1574                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1575                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1576                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1577         }
1578
1579         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1580                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1581         })
1582
1583         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1584                 "/tmp":                   {realtemp + "/tmp1", false},
1585                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1586                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1587                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1588                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1589         })
1590
1591         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1592         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1593         for _, v := range api.Content {
1594                 if v["collection"] != nil {
1595                         c.Check(v["ensure_unique_name"], Equals, true)
1596                         collection := v["collection"].(arvadosclient.Dict)
1597                         if strings.Index(collection["name"].(string), "output") == 0 {
1598                                 manifest := collection["manifest_text"].(string)
1599
1600                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1601 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1602 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1603 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1604 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1605 `)
1606                         }
1607                 }
1608         }
1609 }
1610
1611 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1612         helperRecord := `{
1613                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1614                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1615                 "cwd": "/bin",
1616                 "environment": {"FROBIZ": "bilbo"},
1617                 "mounts": {
1618         "/tmp": {"kind": "tmp"},
1619         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1620         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1621     },
1622                 "output_path": "/tmp",
1623                 "priority": 1,
1624                 "runtime_constraints": {},
1625                 "state": "Locked"
1626         }`
1627
1628         extraMounts := []string{
1629                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1630         }
1631
1632         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1633                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1634         })
1635
1636         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1637         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1638         for _, v := range s.api.Content {
1639                 if v["collection"] != nil {
1640                         collection := v["collection"].(arvadosclient.Dict)
1641                         if strings.Index(collection["name"].(string), "output") == 0 {
1642                                 manifest := collection["manifest_text"].(string)
1643
1644                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1645 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1646 `)
1647                         }
1648                 }
1649         }
1650 }
1651
1652 func (s *TestSuite) TestOutputError(c *C) {
1653         helperRecord := `{
1654                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1655                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1656                 "cwd": "/bin",
1657                 "environment": {"FROBIZ": "bilbo"},
1658                 "mounts": {
1659                         "/tmp": {"kind": "tmp"}
1660                 },
1661                 "output_path": "/tmp",
1662                 "priority": 1,
1663                 "runtime_constraints": {},
1664                 "state": "Locked"
1665         }`
1666         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1667                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1668         })
1669
1670         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1671 }
1672
1673 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1674         helperRecord := `{
1675                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1676                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1677                 "cwd": "/bin",
1678                 "environment": {"FROBIZ": "bilbo"},
1679                 "mounts": {
1680         "/tmp": {"kind": "tmp"},
1681         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1682         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1683     },
1684                 "output_path": "/tmp",
1685                 "priority": 1,
1686                 "runtime_constraints": {},
1687                 "state": "Locked"
1688         }`
1689
1690         extraMounts := []string{
1691                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1692         }
1693
1694         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1695                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1696         })
1697
1698         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1699         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1700         for _, v := range api.Content {
1701                 if v["collection"] != nil {
1702                         collection := v["collection"].(arvadosclient.Dict)
1703                         if strings.Index(collection["name"].(string), "output") == 0 {
1704                                 manifest := collection["manifest_text"].(string)
1705                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1706 `)
1707                         }
1708                 }
1709         }
1710 }
1711
1712 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1713         helperRecord := `{
1714                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1715                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1716                 "cwd": "/bin",
1717                 "environment": {"FROBIZ": "bilbo"},
1718                 "mounts": {
1719         "/tmp": {"kind": "tmp"},
1720         "stdin": {"kind": "json", "content": "foo"},
1721         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1722     },
1723                 "output_path": "/tmp",
1724                 "priority": 1,
1725                 "runtime_constraints": {},
1726                 "state": "Locked"
1727         }`
1728
1729         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1730                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1731         })
1732
1733         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1734         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1735         for _, v := range api.Content {
1736                 if v["collection"] != nil {
1737                         collection := v["collection"].(arvadosclient.Dict)
1738                         if strings.Index(collection["name"].(string), "output") == 0 {
1739                                 manifest := collection["manifest_text"].(string)
1740                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1741 `)
1742                         }
1743                 }
1744         }
1745 }
1746
1747 func (s *TestSuite) TestStderrMount(c *C) {
1748         api, cr, _ := s.fullRunHelper(c, `{
1749     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1750     "container_image": "`+arvadostest.DockerImage112PDH+`",
1751     "cwd": ".",
1752     "environment": {},
1753     "mounts": {"/tmp": {"kind": "tmp"},
1754                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1755                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1756     "output_path": "/tmp",
1757     "priority": 1,
1758     "runtime_constraints": {},
1759     "state": "Locked"
1760 }`, nil, 1, func() {
1761                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1762                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1763         })
1764
1765         final := api.CalledWith("container.state", "Complete")
1766         c.Assert(final, NotNil)
1767         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1768         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1769
1770         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1771 }
1772
1773 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1774         s.api.callraw = true
1775         err := s.runner.fetchContainerRecord()
1776         c.Assert(err, IsNil)
1777         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1778         c.Logf("%#v", s.runner.Container)
1779         c.Check(err, IsNil)
1780         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1781 }
1782
1783 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1784         nextState := ""
1785         for _, setup := range []func(){
1786                 func() {
1787                         c.Log("// waitErr = ocl runtime error")
1788                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1789                         nextState = "Cancelled"
1790                 },
1791                 func() {
1792                         c.Log("// loadErr = cannot connect")
1793                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1794                         *brokenNodeHook = c.MkDir() + "/broken-node-hook"
1795                         err := ioutil.WriteFile(*brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1796                         c.Assert(err, IsNil)
1797                         nextState = "Queued"
1798                 },
1799         } {
1800                 s.SetUpTest(c)
1801                 setup()
1802                 s.fullRunHelper(c, `{
1803     "command": ["echo", "hello world"],
1804     "container_image": "`+arvadostest.DockerImage112PDH+`",
1805     "cwd": ".",
1806     "environment": {},
1807     "mounts": {"/tmp": {"kind": "tmp"} },
1808     "output_path": "/tmp",
1809     "priority": 1,
1810     "runtime_constraints": {},
1811     "state": "Locked"
1812 }`, nil, 0, func() {})
1813                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1814                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1815                 if *brokenNodeHook != "" {
1816                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1817                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1818                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1819                 } else {
1820                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1821                 }
1822         }
1823 }
1824
1825 func (s *TestSuite) TestBadCommand(c *C) {
1826         for _, startError := range []string{
1827                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1828                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1829                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1830         } {
1831                 s.SetUpTest(c)
1832                 s.executor.startErr = errors.New(startError)
1833                 s.fullRunHelper(c, `{
1834     "command": ["echo", "hello world"],
1835     "container_image": "`+arvadostest.DockerImage112PDH+`",
1836     "cwd": ".",
1837     "environment": {},
1838     "mounts": {"/tmp": {"kind": "tmp"} },
1839     "output_path": "/tmp",
1840     "priority": 1,
1841     "runtime_constraints": {},
1842     "state": "Locked"
1843 }`, nil, 0, func() {})
1844                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1845                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1846         }
1847 }
1848
1849 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1850         helperRecord := `{
1851                 "command": ["true"],
1852                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1853                 "cwd": "/bin",
1854                 "mounts": {
1855                     "/tmp": {"kind": "tmp"},
1856                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1857                 },
1858                 "secret_mounts": {
1859                 },
1860                 "output_path": "/tmp",
1861                 "priority": 1,
1862                 "runtime_constraints": {},
1863                 "state": "Locked"
1864         }`
1865
1866         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1867                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1868                 c.Check(err, IsNil)
1869                 c.Check(string(content), Equals, "mypassword")
1870         })
1871
1872         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1873         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1874         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
1875         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
1876
1877         // under secret mounts, not captured in output
1878         helperRecord = `{
1879                 "command": ["true"],
1880                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1881                 "cwd": "/bin",
1882                 "mounts": {
1883                     "/tmp": {"kind": "tmp"}
1884                 },
1885                 "secret_mounts": {
1886                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1887                 },
1888                 "output_path": "/tmp",
1889                 "priority": 1,
1890                 "runtime_constraints": {},
1891                 "state": "Locked"
1892         }`
1893
1894         s.SetUpTest(c)
1895         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1896                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1897                 c.Check(err, IsNil)
1898                 c.Check(string(content), Equals, "mypassword")
1899         })
1900
1901         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1902         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1903         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
1904         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
1905 }
1906
1907 type FakeProcess struct {
1908         cmdLine []string
1909 }
1910
1911 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
1912         return fp.cmdLine, nil
1913 }