lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/manifest"
  29         "golang.org/x/net/context"
  30
  31         . "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func TestCrunchExec(t *testing.T) {
  36         TestingT(t)
  37 }
  38
  39 var _ = Suite(&TestSuite{})
  40
  41 type TestSuite struct {
  42         client    *arvados.Client
  43         api       *ArvTestClient
  44         runner    *ContainerRunner
  45         executor  *stubExecutor
  46         keepmount string
  47 }
  48
  49 func (s *TestSuite) SetUpTest(c *C) {
  50         *brokenNodeHook = ""
  51         s.client = arvados.NewClientFromEnv()
  52         s.executor = &stubExecutor{}
  53         var err error
  54         s.api = &ArvTestClient{}
  55         s.runner, err = NewContainerRunner(s.client, s.api, &KeepTestClient{}, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  56         c.Assert(err, IsNil)
  57         s.runner.executor = s.executor
  58         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  59                 return s.api, &KeepTestClient{}, s.client, nil
  60         }
  61         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  62                 s.runner.ArvMountPoint = s.keepmount
  63                 return nil, nil
  64         }
  65         s.keepmount = c.MkDir()
  66         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  67         c.Assert(err, IsNil)
  68         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  69         c.Assert(err, IsNil)
  70         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  71         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  72         c.Assert(err, IsNil)
  73         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  74         c.Assert(err, IsNil)
  75         s.runner.ArvMountPoint = s.keepmount
  76 }
  77
  78 type ArvTestClient struct {
  79         Total   int64
  80         Calls   int
  81         Content []arvadosclient.Dict
  82         arvados.Container
  83         secretMounts []byte
  84         Logs         map[string]*bytes.Buffer
  85         sync.Mutex
  86         WasSetRunning bool
  87         callraw       bool
  88 }
  89
  90 type KeepTestClient struct {
  91         Called  bool
  92         Content []byte
  93 }
  94
  95 type stubExecutor struct {
  96         imageLoaded bool
  97         loaded      string
  98         loadErr     error
  99         exitCode    int
 100         createErr   error
 101         created     containerSpec
 102         startErr    error
 103         waitSleep   time.Duration
 104         waitErr     error
 105         stopErr     error
 106         stopped     bool
 107         closed      bool
 108         runFunc     func()
 109         exit        chan int
 110 }
 111
 112 func (e *stubExecutor) ImageLoaded(imageID string) bool { return e.imageLoaded }
 113 func (e *stubExecutor) LoadImage(filename string) error { e.loaded = filename; return e.loadErr }
 114 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 115 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 116 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 117 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 118 func (e *stubExecutor) Close()                          { e.closed = true }
 119 func (e *stubExecutor) Wait(context.Context) (int, error) {
 120         defer e.created.Stdout.Close()
 121         defer e.created.Stderr.Close()
 122         return <-e.exit, e.waitErr
 123 }
 124
 125 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 126
 127 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 128 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 129 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 130
 131 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 132 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 133
 134 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 135 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 136 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 137 `
 138
 139 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 140
 141 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 142 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 143
 144 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 145 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 146
 147 func (client *ArvTestClient) Create(resourceType string,
 148         parameters arvadosclient.Dict,
 149         output interface{}) error {
 150
 151         client.Mutex.Lock()
 152         defer client.Mutex.Unlock()
 153
 154         client.Calls++
 155         client.Content = append(client.Content, parameters)
 156
 157         if resourceType == "logs" {
 158                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 159                 if client.Logs == nil {
 160                         client.Logs = make(map[string]*bytes.Buffer)
 161                 }
 162                 if client.Logs[et] == nil {
 163                         client.Logs[et] = &bytes.Buffer{}
 164                 }
 165                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 166         }
 167
 168         if resourceType == "collections" && output != nil {
 169                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 170                 outmap := output.(*arvados.Collection)
 171                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 172                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 173         }
 174
 175         return nil
 176 }
 177
 178 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 179         switch {
 180         case method == "GET" && resourceType == "containers" && action == "auth":
 181                 return json.Unmarshal([]byte(`{
 182                         "kind": "arvados#api_client_authorization",
 183                         "uuid": "`+fakeAuthUUID+`",
 184                         "api_token": "`+fakeAuthToken+`"
 185                         }`), output)
 186         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 187                 if client.secretMounts != nil {
 188                         return json.Unmarshal(client.secretMounts, output)
 189                 }
 190                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 191         default:
 192                 return fmt.Errorf("Not found")
 193         }
 194 }
 195
 196 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 197         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 198         var j []byte
 199         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 200                 j = []byte(`{
 201                         "kind": "arvados#nodeList",
 202                         "items": [{
 203                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 204                                 "hostname": "compute2",
 205                                 "properties": {"total_cpu_cores": 16}
 206                         }]}`)
 207         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 208                 if uuid == "" {
 209                         j, err = json.Marshal(map[string]interface{}{
 210                                 "items": []interface{}{client.Container},
 211                                 "kind":  "arvados#nodeList",
 212                         })
 213                 } else {
 214                         j, err = json.Marshal(client.Container)
 215                 }
 216         } else {
 217                 j = []byte(`{
 218                         "command": ["sleep", "1"],
 219                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 220                         "cwd": ".",
 221                         "environment": {},
 222                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 223                         "output_path": "/tmp",
 224                         "priority": 1,
 225                         "runtime_constraints": {}
 226                 }`)
 227         }
 228         return ioutil.NopCloser(bytes.NewReader(j)), err
 229 }
 230
 231 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 232         if resourceType == "collections" {
 233                 if uuid == hwPDH {
 234                         output.(*arvados.Collection).ManifestText = hwManifest
 235                 } else if uuid == otherPDH {
 236                         output.(*arvados.Collection).ManifestText = otherManifest
 237                 } else if uuid == normalizedWithSubdirsPDH {
 238                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 239                 } else if uuid == denormalizedWithSubdirsPDH {
 240                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 241                 }
 242         }
 243         if resourceType == "containers" {
 244                 (*output.(*arvados.Container)) = client.Container
 245         }
 246         return nil
 247 }
 248
 249 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 250         client.Mutex.Lock()
 251         defer client.Mutex.Unlock()
 252         client.Calls++
 253         client.Content = append(client.Content, parameters)
 254         if resourceType == "containers" {
 255                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 256                         client.WasSetRunning = true
 257                 }
 258         } else if resourceType == "collections" {
 259                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 260                 output.(*arvados.Collection).UUID = uuid
 261                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 262         }
 263         return nil
 264 }
 265
 266 var discoveryMap = map[string]interface{}{
 267         "defaultTrashLifetime":               float64(1209600),
 268         "crunchLimitLogBytesPerJob":          float64(67108864),
 269         "crunchLogThrottleBytes":             float64(65536),
 270         "crunchLogThrottlePeriod":            float64(60),
 271         "crunchLogThrottleLines":             float64(1024),
 272         "crunchLogPartialLineThrottlePeriod": float64(5),
 273         "crunchLogBytesPerEvent":             float64(4096),
 274         "crunchLogSecondsBetweenEvents":      float64(1),
 275 }
 276
 277 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 278         return discoveryMap[key], nil
 279 }
 280
 281 // CalledWith returns the parameters from the first API call whose
 282 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 283 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 284 // no call matches, it returns nil.
 285 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 286 call:
 287         for _, content := range client.Content {
 288                 var v interface{} = content
 289                 for _, k := range strings.Split(jpath, ".") {
 290                         if dict, ok := v.(arvadosclient.Dict); !ok {
 291                                 continue call
 292                         } else {
 293                                 v = dict[k]
 294                         }
 295                 }
 296                 if v == expect {
 297                         return content
 298                 }
 299         }
 300         return nil
 301 }
 302
 303 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 304         return locator, nil
 305 }
 306
 307 func (client *KeepTestClient) PutB(buf []byte) (string, int, error) {
 308         client.Content = buf
 309         return fmt.Sprintf("%x+%d", md5.Sum(buf), len(buf)), len(buf), nil
 310 }
 311
 312 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 313         return 0, errors.New("not implemented")
 314 }
 315
 316 func (client *KeepTestClient) ClearBlockCache() {
 317 }
 318
 319 func (client *KeepTestClient) Close() {
 320         client.Content = nil
 321 }
 322
 323 type FileWrapper struct {
 324         io.ReadCloser
 325         len int64
 326 }
 327
 328 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 329         return nil, errors.New("not implemented")
 330 }
 331
 332 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 333         return 0, errors.New("not implemented")
 334 }
 335
 336 func (fw FileWrapper) Size() int64 {
 337         return fw.len
 338 }
 339
 340 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 341         return nil, errors.New("not implemented")
 342 }
 343
 344 func (fw FileWrapper) Truncate(int64) error {
 345         return errors.New("not implemented")
 346 }
 347
 348 func (fw FileWrapper) Write([]byte) (int, error) {
 349         return 0, errors.New("not implemented")
 350 }
 351
 352 func (fw FileWrapper) Sync() error {
 353         return errors.New("not implemented")
 354 }
 355
 356 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 357         if filename == hwImageID+".tar" {
 358                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 359                 client.Called = true
 360                 return FileWrapper{rdr, 1321984}, nil
 361         } else if filename == "/file1_in_main.txt" {
 362                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 363                 client.Called = true
 364                 return FileWrapper{rdr, 3}, nil
 365         }
 366         return nil, nil
 367 }
 368
 369 func (s *TestSuite) TestLoadImage(c *C) {
 370         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 371         s.runner.Container.Mounts = map[string]arvados.Mount{
 372                 "/out": {Kind: "tmp", Writable: true},
 373         }
 374         s.runner.Container.OutputPath = "/out"
 375
 376         _, err := s.runner.SetupMounts()
 377         c.Assert(err, IsNil)
 378
 379         imageID, err := s.runner.LoadImage()
 380         c.Check(err, IsNil)
 381         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 382         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 383
 384         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 385         s.executor.imageLoaded = false
 386         s.executor.loaded = ""
 387         s.executor.loadErr = errors.New("bork")
 388         imageID, err = s.runner.LoadImage()
 389         c.Check(err, ErrorMatches, ".*bork")
 390         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 391
 392         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 393         s.executor.imageLoaded = false
 394         s.executor.loaded = ""
 395         s.executor.loadErr = nil
 396         imageID, err = s.runner.LoadImage()
 397         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 398         c.Check(s.executor.loaded, Equals, "")
 399
 400         // if executor reports image is already loaded, LoadImage should not be called
 401         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 402         s.executor.imageLoaded = true
 403         s.executor.loaded = ""
 404         s.executor.loadErr = nil
 405         imageID, err = s.runner.LoadImage()
 406         c.Check(err, IsNil)
 407         c.Check(s.executor.loaded, Equals, "")
 408         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 409 }
 410
 411 type ArvErrorTestClient struct{}
 412
 413 func (ArvErrorTestClient) Create(resourceType string,
 414         parameters arvadosclient.Dict,
 415         output interface{}) error {
 416         return nil
 417 }
 418
 419 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 420         if method == "GET" && resourceType == "containers" && action == "auth" {
 421                 return nil
 422         }
 423         return errors.New("ArvError")
 424 }
 425
 426 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 427         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 428         return nil, errors.New("ArvError")
 429 }
 430
 431 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 432         return errors.New("ArvError")
 433 }
 434
 435 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 436         return nil
 437 }
 438
 439 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 440         return discoveryMap[key], nil
 441 }
 442
 443 type KeepErrorTestClient struct {
 444         KeepTestClient
 445 }
 446
 447 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 448         return nil, errors.New("KeepError")
 449 }
 450
 451 func (*KeepErrorTestClient) PutB(buf []byte) (string, int, error) {
 452         return "", 0, errors.New("KeepError")
 453 }
 454
 455 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 456         return "", errors.New("KeepError")
 457 }
 458
 459 type KeepReadErrorTestClient struct {
 460         KeepTestClient
 461 }
 462
 463 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 464         return 0, errors.New("KeepError")
 465 }
 466
 467 type ErrorReader struct {
 468         FileWrapper
 469 }
 470
 471 func (ErrorReader) Read(p []byte) (n int, err error) {
 472         return 0, errors.New("ErrorReader")
 473 }
 474
 475 func (ErrorReader) Seek(int64, int) (int64, error) {
 476         return 0, errors.New("ErrorReader")
 477 }
 478
 479 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 480         return ErrorReader{}, nil
 481 }
 482
 483 type ClosableBuffer struct {
 484         bytes.Buffer
 485 }
 486
 487 func (*ClosableBuffer) Close() error {
 488         return nil
 489 }
 490
 491 type TestLogs struct {
 492         Stdout ClosableBuffer
 493         Stderr ClosableBuffer
 494 }
 495
 496 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 497         if logstr == "stdout" {
 498                 return &tl.Stdout, nil
 499         }
 500         if logstr == "stderr" {
 501                 return &tl.Stderr, nil
 502         }
 503         return nil, errors.New("???")
 504 }
 505
 506 func dockerLog(fd byte, msg string) []byte {
 507         by := []byte(msg)
 508         header := make([]byte, 8+len(by))
 509         header[0] = fd
 510         header[7] = byte(len(by))
 511         copy(header[8:], by)
 512         return header
 513 }
 514
 515 func (s *TestSuite) TestRunContainer(c *C) {
 516         s.executor.runFunc = func() {
 517                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 518                 s.executor.created.Stdout.Close()
 519                 s.executor.created.Stderr.Close()
 520                 s.executor.exit <- 0
 521         }
 522
 523         var logs TestLogs
 524         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 525         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 526         s.runner.Container.Command = []string{"./hw"}
 527
 528         imageID, err := s.runner.LoadImage()
 529         c.Assert(err, IsNil)
 530
 531         err = s.runner.CreateContainer(imageID, nil)
 532         c.Assert(err, IsNil)
 533
 534         err = s.runner.StartContainer()
 535         c.Assert(err, IsNil)
 536
 537         err = s.runner.WaitFinish()
 538         c.Assert(err, IsNil)
 539
 540         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 541         c.Check(logs.Stderr.String(), Equals, "")
 542 }
 543
 544 func (s *TestSuite) TestCommitLogs(c *C) {
 545         api := &ArvTestClient{}
 546         kc := &KeepTestClient{}
 547         defer kc.Close()
 548         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 549         c.Assert(err, IsNil)
 550         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 551
 552         cr.CrunchLog.Print("Hello world!")
 553         cr.CrunchLog.Print("Goodbye")
 554         cr.finalState = "Complete"
 555
 556         err = cr.CommitLogs()
 557         c.Check(err, IsNil)
 558
 559         c.Check(api.Calls, Equals, 2)
 560         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 561         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 562         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 563         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 564 }
 565
 566 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 567         api := &ArvTestClient{}
 568         kc := &KeepTestClient{}
 569         defer kc.Close()
 570         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 571         c.Assert(err, IsNil)
 572
 573         err = cr.UpdateContainerRunning()
 574         c.Check(err, IsNil)
 575
 576         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 577 }
 578
 579 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 580         api := &ArvTestClient{}
 581         kc := &KeepTestClient{}
 582         defer kc.Close()
 583         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 584         c.Assert(err, IsNil)
 585
 586         cr.LogsPDH = new(string)
 587         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 588
 589         cr.ExitCode = new(int)
 590         *cr.ExitCode = 42
 591         cr.finalState = "Complete"
 592
 593         err = cr.UpdateContainerFinal()
 594         c.Check(err, IsNil)
 595
 596         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 597         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 598         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 599 }
 600
 601 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 602         api := &ArvTestClient{}
 603         kc := &KeepTestClient{}
 604         defer kc.Close()
 605         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 606         c.Assert(err, IsNil)
 607         cr.cCancelled = true
 608         cr.finalState = "Cancelled"
 609
 610         err = cr.UpdateContainerFinal()
 611         c.Check(err, IsNil)
 612
 613         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 614         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 615         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 616 }
 617
 618 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 619 // dress rehearsal of the Run() function, starting from a JSON container record.
 620 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 621         err := json.Unmarshal([]byte(record), &s.api.Container)
 622         c.Assert(err, IsNil)
 623         initialState := s.api.Container.State
 624
 625         var sm struct {
 626                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 627         }
 628         err = json.Unmarshal([]byte(record), &sm)
 629         c.Check(err, IsNil)
 630         secretMounts, err := json.Marshal(sm)
 631         c.Assert(err, IsNil)
 632         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 633
 634         s.executor.runFunc = func() {
 635                 fn()
 636                 s.executor.exit <- exitCode
 637         }
 638
 639         s.runner.statInterval = 100 * time.Millisecond
 640         s.runner.containerWatchdogInterval = time.Second
 641         am := &ArvMountCmdLine{}
 642         s.runner.RunArvMount = am.ArvMountTest
 643
 644         realTemp := c.MkDir()
 645         tempcount := 0
 646         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 647                 tempcount++
 648                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 649                 err := os.Mkdir(d, os.ModePerm)
 650                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 651                         // Test case must have pre-populated the tempdir
 652                         err = nil
 653                 }
 654                 return d, err
 655         }
 656         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 657                 return &ArvTestClient{secretMounts: secretMounts}, &KeepTestClient{}, nil, nil
 658         }
 659
 660         if extraMounts != nil && len(extraMounts) > 0 {
 661                 err := s.runner.SetupArvMountPoint("keep")
 662                 c.Check(err, IsNil)
 663
 664                 for _, m := range extraMounts {
 665                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 666                 }
 667         }
 668
 669         err = s.runner.Run()
 670         if s.api.CalledWith("container.state", "Complete") != nil {
 671                 c.Check(err, IsNil)
 672         }
 673         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 674                 c.Check(s.api.WasSetRunning, Equals, true)
 675                 var lastupdate arvadosclient.Dict
 676                 for _, content := range s.api.Content {
 677                         if content["container"] != nil {
 678                                 lastupdate = content["container"].(arvadosclient.Dict)
 679                         }
 680                 }
 681                 if lastupdate["log"] == nil {
 682                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 683                 }
 684         }
 685
 686         if err != nil {
 687                 for k, v := range s.api.Logs {
 688                         c.Log(k)
 689                         c.Log(v.String())
 690                 }
 691         }
 692
 693         return s.api, s.runner, realTemp
 694 }
 695
 696 func (s *TestSuite) TestFullRunHello(c *C) {
 697         s.runner.enableMemoryLimit = true
 698         s.runner.networkMode = "default"
 699         s.fullRunHelper(c, `{
 700     "command": ["echo", "hello world"],
 701     "container_image": "`+arvadostest.DockerImage112PDH+`",
 702     "cwd": ".",
 703     "environment": {"foo":"bar","baz":"waz"},
 704     "mounts": {"/tmp": {"kind": "tmp"} },
 705     "output_path": "/tmp",
 706     "priority": 1,
 707     "runtime_constraints": {"vcpus":1,"ram":1000000},
 708     "state": "Locked"
 709 }`, nil, 0, func() {
 710                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 711                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 712                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 713                 c.Check(s.executor.created.VCPUs, Equals, 1)
 714                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 715                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 716                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 717                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 718         })
 719
 720         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 721         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 722         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 723
 724 }
 725
 726 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 727         var ran bool
 728         s.fullRunHelper(c, `{
 729     "command": ["sleep", "3"],
 730     "container_image": "`+arvadostest.DockerImage112PDH+`",
 731     "cwd": ".",
 732     "environment": {},
 733     "mounts": {"/tmp": {"kind": "tmp"} },
 734     "output_path": "/tmp",
 735     "priority": 1,
 736     "runtime_constraints": {},
 737     "scheduling_parameters":{"max_run_time": 1},
 738     "state": "Running"
 739 }`, nil, 2, func() {
 740                 ran = true
 741         })
 742         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 743         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 744         c.Check(ran, Equals, false)
 745 }
 746
 747 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 748         s.fullRunHelper(c, `{
 749     "command": ["sleep", "3"],
 750     "container_image": "`+arvadostest.DockerImage112PDH+`",
 751     "cwd": ".",
 752     "environment": {},
 753     "mounts": {"/tmp": {"kind": "tmp"} },
 754     "output_path": "/tmp",
 755     "priority": 1,
 756     "runtime_constraints": {},
 757     "scheduling_parameters":{"max_run_time": 1},
 758     "state": "Locked"
 759 }`, nil, 0, func() {
 760                 time.Sleep(3 * time.Second)
 761         })
 762
 763         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 764         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 765 }
 766
 767 func (s *TestSuite) TestContainerWaitFails(c *C) {
 768         s.fullRunHelper(c, `{
 769     "command": ["sleep", "3"],
 770     "container_image": "`+arvadostest.DockerImage112PDH+`",
 771     "cwd": ".",
 772     "mounts": {"/tmp": {"kind": "tmp"} },
 773     "output_path": "/tmp",
 774     "priority": 1,
 775     "state": "Locked"
 776 }`, nil, 0, func() {
 777                 s.executor.waitErr = errors.New("Container is not running")
 778         })
 779
 780         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 781         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 782 }
 783
 784 func (s *TestSuite) TestCrunchstat(c *C) {
 785         s.fullRunHelper(c, `{
 786                 "command": ["sleep", "1"],
 787                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 788                 "cwd": ".",
 789                 "environment": {},
 790                 "mounts": {"/tmp": {"kind": "tmp"} },
 791                 "output_path": "/tmp",
 792                 "priority": 1,
 793                 "runtime_constraints": {},
 794                 "state": "Locked"
 795         }`, nil, 0, func() {
 796                 time.Sleep(time.Second)
 797         })
 798
 799         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 800         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 801
 802         // We didn't actually start a container, so crunchstat didn't
 803         // find accounting files and therefore didn't log any stats.
 804         // It should have logged a "can't find accounting files"
 805         // message after one poll interval, though, so we can confirm
 806         // it's alive:
 807         c.Assert(s.api.Logs["crunchstat"], NotNil)
 808         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 809
 810         // The "files never appeared" log assures us that we called
 811         // (*crunchstat.Reporter)Stop(), and that we set it up with
 812         // the correct container ID "abcde":
 813         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 814 }
 815
 816 func (s *TestSuite) TestNodeInfoLog(c *C) {
 817         os.Setenv("SLURMD_NODENAME", "compute2")
 818         s.fullRunHelper(c, `{
 819                 "command": ["sleep", "1"],
 820                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 821                 "cwd": ".",
 822                 "environment": {},
 823                 "mounts": {"/tmp": {"kind": "tmp"} },
 824                 "output_path": "/tmp",
 825                 "priority": 1,
 826                 "runtime_constraints": {},
 827                 "state": "Locked"
 828         }`, nil, 0,
 829                 func() {
 830                         time.Sleep(time.Second)
 831                 })
 832
 833         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 834         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 835
 836         c.Assert(s.api.Logs["node"], NotNil)
 837         json := s.api.Logs["node"].String()
 838         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 839         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 840         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 841
 842         c.Assert(s.api.Logs["node-info"], NotNil)
 843         json = s.api.Logs["node-info"].String()
 844         c.Check(json, Matches, `(?ms).*Host Information.*`)
 845         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 846         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 847         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 848         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 849 }
 850
 851 func (s *TestSuite) TestContainerRecordLog(c *C) {
 852         s.fullRunHelper(c, `{
 853                 "command": ["sleep", "1"],
 854                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 855                 "cwd": ".",
 856                 "environment": {},
 857                 "mounts": {"/tmp": {"kind": "tmp"} },
 858                 "output_path": "/tmp",
 859                 "priority": 1,
 860                 "runtime_constraints": {},
 861                 "state": "Locked"
 862         }`, nil, 0,
 863                 func() {
 864                         time.Sleep(time.Second)
 865                 })
 866
 867         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 868         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 869
 870         c.Assert(s.api.Logs["container"], NotNil)
 871         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 872 }
 873
 874 func (s *TestSuite) TestFullRunStderr(c *C) {
 875         s.fullRunHelper(c, `{
 876     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 877     "container_image": "`+arvadostest.DockerImage112PDH+`",
 878     "cwd": ".",
 879     "environment": {},
 880     "mounts": {"/tmp": {"kind": "tmp"} },
 881     "output_path": "/tmp",
 882     "priority": 1,
 883     "runtime_constraints": {},
 884     "state": "Locked"
 885 }`, nil, 1, func() {
 886                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 887                 fmt.Fprintln(s.executor.created.Stderr, "world")
 888         })
 889
 890         final := s.api.CalledWith("container.state", "Complete")
 891         c.Assert(final, NotNil)
 892         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 893         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 894
 895         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 896         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 897 }
 898
 899 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 900         s.fullRunHelper(c, `{
 901     "command": ["pwd"],
 902     "container_image": "`+arvadostest.DockerImage112PDH+`",
 903     "cwd": ".",
 904     "environment": {},
 905     "mounts": {"/tmp": {"kind": "tmp"} },
 906     "output_path": "/tmp",
 907     "priority": 1,
 908     "runtime_constraints": {},
 909     "state": "Locked"
 910 }`, nil, 0, func() {
 911                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 912         })
 913
 914         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 915         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 916         c.Log(s.api.Logs["stdout"])
 917         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 918 }
 919
 920 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 921         s.fullRunHelper(c, `{
 922     "command": ["pwd"],
 923     "container_image": "`+arvadostest.DockerImage112PDH+`",
 924     "cwd": "/bin",
 925     "environment": {},
 926     "mounts": {"/tmp": {"kind": "tmp"} },
 927     "output_path": "/tmp",
 928     "priority": 1,
 929     "runtime_constraints": {},
 930     "state": "Locked"
 931 }`, nil, 0, func() {
 932                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 933         })
 934
 935         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 936         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 937         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 938 }
 939
 940 func (s *TestSuite) TestStopOnSignal(c *C) {
 941         s.executor.runFunc = func() {
 942                 s.executor.created.Stdout.Write([]byte("foo\n"))
 943                 s.runner.SigChan <- syscall.SIGINT
 944         }
 945         s.testStopContainer(c)
 946 }
 947
 948 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
 949         s.executor.runFunc = func() {
 950                 s.executor.created.Stdout.Write([]byte("foo\n"))
 951                 s.runner.ArvMountExit <- nil
 952                 close(s.runner.ArvMountExit)
 953         }
 954         s.runner.ArvMountExit = make(chan error)
 955         s.testStopContainer(c)
 956 }
 957
 958 func (s *TestSuite) testStopContainer(c *C) {
 959         record := `{
 960     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
 961     "container_image": "` + arvadostest.DockerImage112PDH + `",
 962     "cwd": ".",
 963     "environment": {},
 964     "mounts": {"/tmp": {"kind": "tmp"} },
 965     "output_path": "/tmp",
 966     "priority": 1,
 967     "runtime_constraints": {},
 968     "state": "Locked"
 969 }`
 970
 971         err := json.Unmarshal([]byte(record), &s.api.Container)
 972         c.Assert(err, IsNil)
 973
 974         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
 975         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 976                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
 977         }
 978
 979         done := make(chan error)
 980         go func() {
 981                 done <- s.runner.Run()
 982         }()
 983         select {
 984         case <-time.After(20 * time.Second):
 985                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
 986                 c.Fatal("timed out")
 987         case err = <-done:
 988                 c.Check(err, IsNil)
 989         }
 990         for k, v := range s.api.Logs {
 991                 c.Log(k)
 992                 c.Log(v.String(), "\n")
 993         }
 994
 995         c.Check(s.api.CalledWith("container.log", nil), NotNil)
 996         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 997         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
 998 }
 999
1000 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1001         s.fullRunHelper(c, `{
1002     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1003     "container_image": "`+arvadostest.DockerImage112PDH+`",
1004     "cwd": "/bin",
1005     "environment": {"FROBIZ": "bilbo"},
1006     "mounts": {"/tmp": {"kind": "tmp"} },
1007     "output_path": "/tmp",
1008     "priority": 1,
1009     "runtime_constraints": {},
1010     "state": "Locked"
1011 }`, nil, 0, func() {
1012                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1013         })
1014
1015         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1016         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1017         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1018 }
1019
1020 type ArvMountCmdLine struct {
1021         Cmd   []string
1022         token string
1023 }
1024
1025 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1026         am.Cmd = c
1027         am.token = token
1028         return nil, nil
1029 }
1030
1031 func stubCert(temp string) string {
1032         path := temp + "/ca-certificates.crt"
1033         crt, _ := os.Create(path)
1034         crt.Close()
1035         arvadosclient.CertFiles = []string{path}
1036         return path
1037 }
1038
1039 func (s *TestSuite) TestSetupMounts(c *C) {
1040         cr := s.runner
1041         am := &ArvMountCmdLine{}
1042         cr.RunArvMount = am.ArvMountTest
1043         cr.ContainerArvClient = &ArvTestClient{}
1044         cr.ContainerKeepClient = &KeepTestClient{}
1045
1046         realTemp := c.MkDir()
1047         certTemp := c.MkDir()
1048         stubCertPath := stubCert(certTemp)
1049         cr.parentTemp = realTemp
1050
1051         i := 0
1052         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1053                 i++
1054                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1055                 err := os.Mkdir(d, os.ModePerm)
1056                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1057                         // Test case must have pre-populated the tempdir
1058                         err = nil
1059                 }
1060                 return d, err
1061         }
1062
1063         checkEmpty := func() {
1064                 // Should be deleted.
1065                 _, err := os.Stat(realTemp)
1066                 c.Assert(os.IsNotExist(err), Equals, true)
1067
1068                 // Now recreate it for the next test.
1069                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1070         }
1071
1072         {
1073                 i = 0
1074                 cr.ArvMountPoint = ""
1075                 cr.Container.Mounts = make(map[string]arvados.Mount)
1076                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1077                 cr.Container.OutputPath = "/tmp"
1078                 cr.statInterval = 5 * time.Second
1079                 bindmounts, err := cr.SetupMounts()
1080                 c.Check(err, IsNil)
1081                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1082                         "--read-write", "--crunchstat-interval=5",
1083                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1084                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1085                 os.RemoveAll(cr.ArvMountPoint)
1086                 cr.CleanupDirs()
1087                 checkEmpty()
1088         }
1089
1090         {
1091                 i = 0
1092                 cr.ArvMountPoint = ""
1093                 cr.Container.Mounts = make(map[string]arvados.Mount)
1094                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1095                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1096                 cr.Container.OutputPath = "/out"
1097
1098                 bindmounts, err := cr.SetupMounts()
1099                 c.Check(err, IsNil)
1100                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1101                         "--read-write", "--crunchstat-interval=5",
1102                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1103                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1104                 os.RemoveAll(cr.ArvMountPoint)
1105                 cr.CleanupDirs()
1106                 checkEmpty()
1107         }
1108
1109         {
1110                 i = 0
1111                 cr.ArvMountPoint = ""
1112                 cr.Container.Mounts = make(map[string]arvados.Mount)
1113                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1114                 cr.Container.OutputPath = "/tmp"
1115                 cr.Container.RuntimeConstraints.API = true
1116
1117                 bindmounts, err := cr.SetupMounts()
1118                 c.Check(err, IsNil)
1119                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1120                         "--read-write", "--crunchstat-interval=5",
1121                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1122                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1123                 os.RemoveAll(cr.ArvMountPoint)
1124                 cr.CleanupDirs()
1125                 checkEmpty()
1126
1127                 cr.Container.RuntimeConstraints.API = false
1128         }
1129
1130         {
1131                 i = 0
1132                 cr.ArvMountPoint = ""
1133                 cr.Container.Mounts = map[string]arvados.Mount{
1134                         "/keeptmp": {Kind: "collection", Writable: true},
1135                 }
1136                 cr.Container.OutputPath = "/keeptmp"
1137
1138                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1139
1140                 bindmounts, err := cr.SetupMounts()
1141                 c.Check(err, IsNil)
1142                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1143                         "--read-write", "--crunchstat-interval=5",
1144                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1145                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1146                 os.RemoveAll(cr.ArvMountPoint)
1147                 cr.CleanupDirs()
1148                 checkEmpty()
1149         }
1150
1151         {
1152                 i = 0
1153                 cr.ArvMountPoint = ""
1154                 cr.Container.Mounts = map[string]arvados.Mount{
1155                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1156                         "/keepout": {Kind: "collection", Writable: true},
1157                 }
1158                 cr.Container.OutputPath = "/keepout"
1159
1160                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1161                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1162
1163                 bindmounts, err := cr.SetupMounts()
1164                 c.Check(err, IsNil)
1165                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1166                         "--read-write", "--crunchstat-interval=5",
1167                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1168                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1169                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1170                         "/keepout": {realTemp + "/keep1/tmp0", false},
1171                 })
1172                 os.RemoveAll(cr.ArvMountPoint)
1173                 cr.CleanupDirs()
1174                 checkEmpty()
1175         }
1176
1177         {
1178                 i = 0
1179                 cr.ArvMountPoint = ""
1180                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1181                 cr.Container.Mounts = map[string]arvados.Mount{
1182                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1183                         "/keepout": {Kind: "collection", Writable: true},
1184                 }
1185                 cr.Container.OutputPath = "/keepout"
1186
1187                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1188                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1189
1190                 bindmounts, err := cr.SetupMounts()
1191                 c.Check(err, IsNil)
1192                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1193                         "--read-write", "--crunchstat-interval=5",
1194                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1195                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1196                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1197                         "/keepout": {realTemp + "/keep1/tmp0", false},
1198                 })
1199                 os.RemoveAll(cr.ArvMountPoint)
1200                 cr.CleanupDirs()
1201                 checkEmpty()
1202         }
1203
1204         for _, test := range []struct {
1205                 in  interface{}
1206                 out string
1207         }{
1208                 {in: "foo", out: `"foo"`},
1209                 {in: nil, out: `null`},
1210                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1211         } {
1212                 i = 0
1213                 cr.ArvMountPoint = ""
1214                 cr.Container.Mounts = map[string]arvados.Mount{
1215                         "/mnt/test.json": {Kind: "json", Content: test.in},
1216                 }
1217                 bindmounts, err := cr.SetupMounts()
1218                 c.Check(err, IsNil)
1219                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1220                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1221                 })
1222                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1223                 c.Check(err, IsNil)
1224                 c.Check(content, DeepEquals, []byte(test.out))
1225                 os.RemoveAll(cr.ArvMountPoint)
1226                 cr.CleanupDirs()
1227                 checkEmpty()
1228         }
1229
1230         for _, test := range []struct {
1231                 in  interface{}
1232                 out string
1233         }{
1234                 {in: "foo", out: `foo`},
1235                 {in: nil, out: "error"},
1236                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1237         } {
1238                 i = 0
1239                 cr.ArvMountPoint = ""
1240                 cr.Container.Mounts = map[string]arvados.Mount{
1241                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1242                 }
1243                 bindmounts, err := cr.SetupMounts()
1244                 if test.out == "error" {
1245                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1246                 } else {
1247                         c.Check(err, IsNil)
1248                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1249                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1250                         })
1251                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1252                         c.Check(err, IsNil)
1253                         c.Check(content, DeepEquals, []byte(test.out))
1254                 }
1255                 os.RemoveAll(cr.ArvMountPoint)
1256                 cr.CleanupDirs()
1257                 checkEmpty()
1258         }
1259
1260         // Read-only mount points are allowed underneath output_dir mount point
1261         {
1262                 i = 0
1263                 cr.ArvMountPoint = ""
1264                 cr.Container.Mounts = make(map[string]arvados.Mount)
1265                 cr.Container.Mounts = map[string]arvados.Mount{
1266                         "/tmp":     {Kind: "tmp"},
1267                         "/tmp/foo": {Kind: "collection"},
1268                 }
1269                 cr.Container.OutputPath = "/tmp"
1270
1271                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1272
1273                 bindmounts, err := cr.SetupMounts()
1274                 c.Check(err, IsNil)
1275                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1276                         "--read-write", "--crunchstat-interval=5",
1277                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1278                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1279                         "/tmp":     {realTemp + "/tmp2", false},
1280                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1281                 })
1282                 os.RemoveAll(cr.ArvMountPoint)
1283                 cr.CleanupDirs()
1284                 checkEmpty()
1285         }
1286
1287         // Writable mount points copied to output_dir mount point
1288         {
1289                 i = 0
1290                 cr.ArvMountPoint = ""
1291                 cr.Container.Mounts = make(map[string]arvados.Mount)
1292                 cr.Container.Mounts = map[string]arvados.Mount{
1293                         "/tmp": {Kind: "tmp"},
1294                         "/tmp/foo": {Kind: "collection",
1295                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1296                                 Writable:         true},
1297                         "/tmp/bar": {Kind: "collection",
1298                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1299                                 Path:             "baz",
1300                                 Writable:         true},
1301                 }
1302                 cr.Container.OutputPath = "/tmp"
1303
1304                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1305                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1306
1307                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1308                 rf.Write([]byte("bar"))
1309                 rf.Close()
1310
1311                 _, err := cr.SetupMounts()
1312                 c.Check(err, IsNil)
1313                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1314                 c.Check(err, IsNil)
1315                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1316                 c.Check(err, IsNil)
1317                 os.RemoveAll(cr.ArvMountPoint)
1318                 cr.CleanupDirs()
1319                 checkEmpty()
1320         }
1321
1322         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1323         {
1324                 i = 0
1325                 cr.ArvMountPoint = ""
1326                 cr.Container.Mounts = make(map[string]arvados.Mount)
1327                 cr.Container.Mounts = map[string]arvados.Mount{
1328                         "/tmp":     {Kind: "tmp"},
1329                         "/tmp/foo": {Kind: "tmp"},
1330                 }
1331                 cr.Container.OutputPath = "/tmp"
1332
1333                 _, err := cr.SetupMounts()
1334                 c.Check(err, NotNil)
1335                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1336                 os.RemoveAll(cr.ArvMountPoint)
1337                 cr.CleanupDirs()
1338                 checkEmpty()
1339         }
1340
1341         // Only mount point of kind 'collection' is allowed for stdin
1342         {
1343                 i = 0
1344                 cr.ArvMountPoint = ""
1345                 cr.Container.Mounts = make(map[string]arvados.Mount)
1346                 cr.Container.Mounts = map[string]arvados.Mount{
1347                         "stdin": {Kind: "tmp"},
1348                 }
1349
1350                 _, err := cr.SetupMounts()
1351                 c.Check(err, NotNil)
1352                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1353                 os.RemoveAll(cr.ArvMountPoint)
1354                 cr.CleanupDirs()
1355                 checkEmpty()
1356         }
1357
1358         // git_tree mounts
1359         {
1360                 i = 0
1361                 cr.ArvMountPoint = ""
1362                 (*GitMountSuite)(nil).useTestGitServer(c)
1363                 cr.token = arvadostest.ActiveToken
1364                 cr.Container.Mounts = make(map[string]arvados.Mount)
1365                 cr.Container.Mounts = map[string]arvados.Mount{
1366                         "/tip": {
1367                                 Kind:   "git_tree",
1368                                 UUID:   arvadostest.Repository2UUID,
1369                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1370                                 Path:   "/",
1371                         },
1372                         "/non-tip": {
1373                                 Kind:   "git_tree",
1374                                 UUID:   arvadostest.Repository2UUID,
1375                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1376                                 Path:   "/",
1377                         },
1378                 }
1379                 cr.Container.OutputPath = "/tmp"
1380
1381                 bindmounts, err := cr.SetupMounts()
1382                 c.Check(err, IsNil)
1383
1384                 for path, mount := range bindmounts {
1385                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1386                 }
1387
1388                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1389                 c.Check(err, IsNil)
1390                 c.Check(string(data), Equals, "\000\001\002\003")
1391                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1392                 c.Check(err, FitsTypeOf, &os.PathError{})
1393                 c.Check(os.IsNotExist(err), Equals, true)
1394
1395                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1396                 c.Check(err, IsNil)
1397                 c.Check(string(data), Equals, "\000\001\002\003")
1398                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1399                 c.Check(err, IsNil)
1400                 c.Check(string(data), Equals, "testfile\n")
1401
1402                 cr.CleanupDirs()
1403                 checkEmpty()
1404         }
1405 }
1406
1407 func (s *TestSuite) TestStdout(c *C) {
1408         helperRecord := `{
1409                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1410                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1411                 "cwd": "/bin",
1412                 "environment": {"FROBIZ": "bilbo"},
1413                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1414                 "output_path": "/tmp",
1415                 "priority": 1,
1416                 "runtime_constraints": {},
1417                 "state": "Locked"
1418         }`
1419
1420         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1421                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1422         })
1423
1424         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1425         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1426         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1427 }
1428
1429 // Used by the TestStdoutWithWrongPath*()
1430 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1431         err := json.Unmarshal([]byte(record), &s.api.Container)
1432         c.Assert(err, IsNil)
1433         s.executor.runFunc = fn
1434         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1435         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1436                 return s.api, &KeepTestClient{}, nil, nil
1437         }
1438         return s.api, s.runner, s.runner.Run()
1439 }
1440
1441 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1442         _, _, err := s.stdoutErrorRunHelper(c, `{
1443     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1444     "output_path": "/tmp",
1445     "state": "Locked"
1446 }`, func() {})
1447         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1448 }
1449
1450 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1451         _, _, err := s.stdoutErrorRunHelper(c, `{
1452     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1453     "output_path": "/tmp",
1454     "state": "Locked"
1455 }`, func() {})
1456         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1457 }
1458
1459 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1460         _, _, err := s.stdoutErrorRunHelper(c, `{
1461     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1462     "output_path": "/tmp",
1463     "state": "Locked"
1464 }`, func() {})
1465         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1466 }
1467
1468 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1469         s.fullRunHelper(c, `{
1470     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1471     "container_image": "`+arvadostest.DockerImage112PDH+`",
1472     "cwd": "/bin",
1473     "environment": {},
1474     "mounts": {"/tmp": {"kind": "tmp"} },
1475     "output_path": "/tmp",
1476     "priority": 1,
1477     "runtime_constraints": {"API": true},
1478     "state": "Locked"
1479 }`, nil, 0, func() {
1480                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1481                 s.executor.exit <- 3
1482         })
1483         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1484         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1485 }
1486
1487 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1488         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1489         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1490         s.fullRunHelper(c, `{
1491     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1492     "container_image": "`+arvadostest.DockerImage112PDH+`",
1493     "cwd": "/bin",
1494     "environment": {},
1495     "mounts": {"/tmp": {"kind": "tmp"} },
1496     "output_path": "/tmp",
1497     "priority": 1,
1498     "runtime_constraints": {"API": true},
1499     "state": "Locked"
1500 }`, nil, 0, func() {
1501                 s.api.Container.Output = arvadostest.DockerImage112PDH
1502         })
1503
1504         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1505         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1506         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1507 }
1508
1509 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1510         helperRecord := `{
1511                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1512                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1513                 "cwd": "/bin",
1514                 "environment": {"FROBIZ": "bilbo"},
1515                 "mounts": {
1516         "/tmp": {"kind": "tmp"},
1517         "/tmp/foo": {"kind": "collection",
1518                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1519                      "exclude_from_output": true
1520         },
1521         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1522     },
1523                 "output_path": "/tmp",
1524                 "priority": 1,
1525                 "runtime_constraints": {},
1526                 "state": "Locked"
1527         }`
1528
1529         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1530
1531         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1532                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1533         })
1534
1535         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1536         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1537         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1538 }
1539
1540 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1541         helperRecord := `{
1542                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1543                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1544                 "cwd": "/bin",
1545                 "environment": {"FROBIZ": "bilbo"},
1546                 "mounts": {
1547         "/tmp": {"kind": "tmp"},
1548         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1549         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1550         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1551         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1552         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1553     },
1554                 "output_path": "/tmp",
1555                 "priority": 1,
1556                 "runtime_constraints": {},
1557                 "state": "Locked"
1558         }`
1559
1560         extraMounts := []string{
1561                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1562                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1563                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1564         }
1565
1566         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1567                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1568         })
1569
1570         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1571                 "/tmp":                   {realtemp + "/tmp1", false},
1572                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1573                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1574                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1575                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1576         })
1577
1578         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1579         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1580         for _, v := range api.Content {
1581                 if v["collection"] != nil {
1582                         c.Check(v["ensure_unique_name"], Equals, true)
1583                         collection := v["collection"].(arvadosclient.Dict)
1584                         if strings.Index(collection["name"].(string), "output") == 0 {
1585                                 manifest := collection["manifest_text"].(string)
1586
1587                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1588 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1589 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1590 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1591 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1592 `)
1593                         }
1594                 }
1595         }
1596 }
1597
1598 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1599         helperRecord := `{
1600                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1601                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1602                 "cwd": "/bin",
1603                 "environment": {"FROBIZ": "bilbo"},
1604                 "mounts": {
1605         "/tmp": {"kind": "tmp"},
1606         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1607         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1608     },
1609                 "output_path": "/tmp",
1610                 "priority": 1,
1611                 "runtime_constraints": {},
1612                 "state": "Locked"
1613         }`
1614
1615         extraMounts := []string{
1616                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1617         }
1618
1619         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1620                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1621         })
1622
1623         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1624         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1625         for _, v := range s.api.Content {
1626                 if v["collection"] != nil {
1627                         collection := v["collection"].(arvadosclient.Dict)
1628                         if strings.Index(collection["name"].(string), "output") == 0 {
1629                                 manifest := collection["manifest_text"].(string)
1630
1631                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1632 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1633 `)
1634                         }
1635                 }
1636         }
1637 }
1638
1639 func (s *TestSuite) TestOutputError(c *C) {
1640         helperRecord := `{
1641                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1642                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1643                 "cwd": "/bin",
1644                 "environment": {"FROBIZ": "bilbo"},
1645                 "mounts": {
1646                         "/tmp": {"kind": "tmp"}
1647                 },
1648                 "output_path": "/tmp",
1649                 "priority": 1,
1650                 "runtime_constraints": {},
1651                 "state": "Locked"
1652         }`
1653         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1654                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1655         })
1656
1657         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1658 }
1659
1660 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1661         helperRecord := `{
1662                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1663                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1664                 "cwd": "/bin",
1665                 "environment": {"FROBIZ": "bilbo"},
1666                 "mounts": {
1667         "/tmp": {"kind": "tmp"},
1668         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1669         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1670     },
1671                 "output_path": "/tmp",
1672                 "priority": 1,
1673                 "runtime_constraints": {},
1674                 "state": "Locked"
1675         }`
1676
1677         extraMounts := []string{
1678                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1679         }
1680
1681         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1682                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1683         })
1684
1685         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1686         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1687         for _, v := range api.Content {
1688                 if v["collection"] != nil {
1689                         collection := v["collection"].(arvadosclient.Dict)
1690                         if strings.Index(collection["name"].(string), "output") == 0 {
1691                                 manifest := collection["manifest_text"].(string)
1692                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1693 `)
1694                         }
1695                 }
1696         }
1697 }
1698
1699 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1700         helperRecord := `{
1701                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1702                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1703                 "cwd": "/bin",
1704                 "environment": {"FROBIZ": "bilbo"},
1705                 "mounts": {
1706         "/tmp": {"kind": "tmp"},
1707         "stdin": {"kind": "json", "content": "foo"},
1708         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1709     },
1710                 "output_path": "/tmp",
1711                 "priority": 1,
1712                 "runtime_constraints": {},
1713                 "state": "Locked"
1714         }`
1715
1716         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1717                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1718         })
1719
1720         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1721         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1722         for _, v := range api.Content {
1723                 if v["collection"] != nil {
1724                         collection := v["collection"].(arvadosclient.Dict)
1725                         if strings.Index(collection["name"].(string), "output") == 0 {
1726                                 manifest := collection["manifest_text"].(string)
1727                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1728 `)
1729                         }
1730                 }
1731         }
1732 }
1733
1734 func (s *TestSuite) TestStderrMount(c *C) {
1735         api, cr, _ := s.fullRunHelper(c, `{
1736     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1737     "container_image": "`+arvadostest.DockerImage112PDH+`",
1738     "cwd": ".",
1739     "environment": {},
1740     "mounts": {"/tmp": {"kind": "tmp"},
1741                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1742                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1743     "output_path": "/tmp",
1744     "priority": 1,
1745     "runtime_constraints": {},
1746     "state": "Locked"
1747 }`, nil, 1, func() {
1748                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1749                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1750         })
1751
1752         final := api.CalledWith("container.state", "Complete")
1753         c.Assert(final, NotNil)
1754         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1755         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1756
1757         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1758 }
1759
1760 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1761         s.api.callraw = true
1762         err := s.runner.fetchContainerRecord()
1763         c.Assert(err, IsNil)
1764         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1765         c.Logf("%#v", s.runner.Container)
1766         c.Check(err, IsNil)
1767         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1768 }
1769
1770 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1771         nextState := ""
1772         for _, setup := range []func(){
1773                 func() {
1774                         c.Log("// waitErr = ocl runtime error")
1775                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1776                         nextState = "Cancelled"
1777                 },
1778                 func() {
1779                         c.Log("// loadErr = cannot connect")
1780                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1781                         *brokenNodeHook = c.MkDir() + "/broken-node-hook"
1782                         err := ioutil.WriteFile(*brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1783                         c.Assert(err, IsNil)
1784                         nextState = "Queued"
1785                 },
1786         } {
1787                 s.SetUpTest(c)
1788                 setup()
1789                 s.fullRunHelper(c, `{
1790     "command": ["echo", "hello world"],
1791     "container_image": "`+arvadostest.DockerImage112PDH+`",
1792     "cwd": ".",
1793     "environment": {},
1794     "mounts": {"/tmp": {"kind": "tmp"} },
1795     "output_path": "/tmp",
1796     "priority": 1,
1797     "runtime_constraints": {},
1798     "state": "Locked"
1799 }`, nil, 0, func() {})
1800                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1801                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1802                 if *brokenNodeHook != "" {
1803                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1804                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1805                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1806                 } else {
1807                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1808                 }
1809         }
1810 }
1811
1812 func (s *TestSuite) TestBadCommand(c *C) {
1813         for _, startError := range []string{
1814                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1815                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1816                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1817         } {
1818                 s.SetUpTest(c)
1819                 s.executor.startErr = errors.New(startError)
1820                 s.fullRunHelper(c, `{
1821     "command": ["echo", "hello world"],
1822     "container_image": "`+arvadostest.DockerImage112PDH+`",
1823     "cwd": ".",
1824     "environment": {},
1825     "mounts": {"/tmp": {"kind": "tmp"} },
1826     "output_path": "/tmp",
1827     "priority": 1,
1828     "runtime_constraints": {},
1829     "state": "Locked"
1830 }`, nil, 0, func() {})
1831                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1832                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1833         }
1834 }
1835
1836 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1837         helperRecord := `{
1838                 "command": ["true"],
1839                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1840                 "cwd": "/bin",
1841                 "mounts": {
1842                     "/tmp": {"kind": "tmp"},
1843                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1844                 },
1845                 "secret_mounts": {
1846                 },
1847                 "output_path": "/tmp",
1848                 "priority": 1,
1849                 "runtime_constraints": {},
1850                 "state": "Locked"
1851         }`
1852
1853         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1854                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1855                 c.Check(err, IsNil)
1856                 c.Check(string(content), Equals, "mypassword")
1857         })
1858
1859         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1860         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1861         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
1862         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
1863
1864         // under secret mounts, not captured in output
1865         helperRecord = `{
1866                 "command": ["true"],
1867                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1868                 "cwd": "/bin",
1869                 "mounts": {
1870                     "/tmp": {"kind": "tmp"}
1871                 },
1872                 "secret_mounts": {
1873                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1874                 },
1875                 "output_path": "/tmp",
1876                 "priority": 1,
1877                 "runtime_constraints": {},
1878                 "state": "Locked"
1879         }`
1880
1881         s.SetUpTest(c)
1882         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1883                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1884                 c.Check(err, IsNil)
1885                 c.Check(string(content), Equals, "mypassword")
1886         })
1887
1888         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1889         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1890         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
1891         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
1892 }
1893
1894 type FakeProcess struct {
1895         cmdLine []string
1896 }
1897
1898 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
1899         return fp.cmdLine, nil
1900 }