lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/manifest"
  29         "golang.org/x/net/context"
  30
  31         . "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func TestCrunchExec(t *testing.T) {
  36         TestingT(t)
  37 }
  38
  39 var _ = Suite(&TestSuite{})
  40
  41 type TestSuite struct {
  42         client    *arvados.Client
  43         api       *ArvTestClient
  44         runner    *ContainerRunner
  45         executor  *stubExecutor
  46         keepmount string
  47 }
  48
  49 func (s *TestSuite) SetUpTest(c *C) {
  50         *brokenNodeHook = ""
  51         s.client = arvados.NewClientFromEnv()
  52         s.executor = &stubExecutor{}
  53         var err error
  54         s.api = &ArvTestClient{}
  55         s.runner, err = NewContainerRunner(s.client, s.api, &KeepTestClient{}, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  56         c.Assert(err, IsNil)
  57         s.runner.executor = s.executor
  58         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  59                 return s.api, &KeepTestClient{}, s.client, nil
  60         }
  61         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  62                 s.runner.ArvMountPoint = s.keepmount
  63                 return nil, nil
  64         }
  65         s.keepmount = c.MkDir()
  66         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  67         c.Assert(err, IsNil)
  68         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  69         c.Assert(err, IsNil)
  70         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  71         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  72         c.Assert(err, IsNil)
  73         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  74         c.Assert(err, IsNil)
  75         s.runner.ArvMountPoint = s.keepmount
  76 }
  77
  78 type ArvTestClient struct {
  79         Total   int64
  80         Calls   int
  81         Content []arvadosclient.Dict
  82         arvados.Container
  83         secretMounts []byte
  84         Logs         map[string]*bytes.Buffer
  85         sync.Mutex
  86         WasSetRunning bool
  87         callraw       bool
  88 }
  89
  90 type KeepTestClient struct {
  91         Called  bool
  92         Content []byte
  93 }
  94
  95 type stubExecutor struct {
  96         imageLoaded bool
  97         loaded      string
  98         loadErr     error
  99         exitCode    int
 100         createErr   error
 101         created     containerSpec
 102         startErr    error
 103         waitSleep   time.Duration
 104         waitErr     error
 105         stopErr     error
 106         stopped     bool
 107         closed      bool
 108         runFunc     func()
 109         exit        chan int
 110 }
 111
 112 func (e *stubExecutor) ImageLoaded(imageID string) bool { return e.imageLoaded }
 113 func (e *stubExecutor) LoadImage(filename string) error { e.loaded = filename; return e.loadErr }
 114 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 115 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 116 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 117 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 118 func (e *stubExecutor) Close()                          { e.closed = true }
 119 func (e *stubExecutor) Wait(context.Context) (int, error) {
 120         defer e.created.Stdout.Close()
 121         defer e.created.Stderr.Close()
 122         return <-e.exit, e.waitErr
 123 }
 124
 125 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 126
 127 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 128 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 129 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 130
 131 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 132 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 133
 134 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 135 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 136 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 137 `
 138
 139 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 140
 141 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 142 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 143
 144 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 145 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 146
 147 func (client *ArvTestClient) Create(resourceType string,
 148         parameters arvadosclient.Dict,
 149         output interface{}) error {
 150
 151         client.Mutex.Lock()
 152         defer client.Mutex.Unlock()
 153
 154         client.Calls++
 155         client.Content = append(client.Content, parameters)
 156
 157         if resourceType == "logs" {
 158                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 159                 if client.Logs == nil {
 160                         client.Logs = make(map[string]*bytes.Buffer)
 161                 }
 162                 if client.Logs[et] == nil {
 163                         client.Logs[et] = &bytes.Buffer{}
 164                 }
 165                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 166         }
 167
 168         if resourceType == "collections" && output != nil {
 169                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 170                 outmap := output.(*arvados.Collection)
 171                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 172                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 173         }
 174
 175         return nil
 176 }
 177
 178 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 179         switch {
 180         case method == "GET" && resourceType == "containers" && action == "auth":
 181                 return json.Unmarshal([]byte(`{
 182                         "kind": "arvados#api_client_authorization",
 183                         "uuid": "`+fakeAuthUUID+`",
 184                         "api_token": "`+fakeAuthToken+`"
 185                         }`), output)
 186         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 187                 if client.secretMounts != nil {
 188                         return json.Unmarshal(client.secretMounts, output)
 189                 }
 190                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 191         default:
 192                 return fmt.Errorf("Not found")
 193         }
 194 }
 195
 196 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 197         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 198         var j []byte
 199         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 200                 j = []byte(`{
 201                         "kind": "arvados#nodeList",
 202                         "items": [{
 203                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 204                                 "hostname": "compute2",
 205                                 "properties": {"total_cpu_cores": 16}
 206                         }]}`)
 207         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 208                 if uuid == "" {
 209                         j, err = json.Marshal(map[string]interface{}{
 210                                 "items": []interface{}{client.Container},
 211                                 "kind":  "arvados#nodeList",
 212                         })
 213                 } else {
 214                         j, err = json.Marshal(client.Container)
 215                 }
 216         } else {
 217                 j = []byte(`{
 218                         "command": ["sleep", "1"],
 219                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 220                         "cwd": ".",
 221                         "environment": {},
 222                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 223                         "output_path": "/tmp",
 224                         "priority": 1,
 225                         "runtime_constraints": {}
 226                 }`)
 227         }
 228         return ioutil.NopCloser(bytes.NewReader(j)), err
 229 }
 230
 231 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 232         if resourceType == "collections" {
 233                 if uuid == hwPDH {
 234                         output.(*arvados.Collection).ManifestText = hwManifest
 235                 } else if uuid == otherPDH {
 236                         output.(*arvados.Collection).ManifestText = otherManifest
 237                 } else if uuid == normalizedWithSubdirsPDH {
 238                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 239                 } else if uuid == denormalizedWithSubdirsPDH {
 240                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 241                 }
 242         }
 243         if resourceType == "containers" {
 244                 (*output.(*arvados.Container)) = client.Container
 245         }
 246         return nil
 247 }
 248
 249 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 250         client.Mutex.Lock()
 251         defer client.Mutex.Unlock()
 252         client.Calls++
 253         client.Content = append(client.Content, parameters)
 254         if resourceType == "containers" {
 255                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 256                         client.WasSetRunning = true
 257                 }
 258         } else if resourceType == "collections" {
 259                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 260                 output.(*arvados.Collection).UUID = uuid
 261                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 262         }
 263         return nil
 264 }
 265
 266 var discoveryMap = map[string]interface{}{
 267         "defaultTrashLifetime":               float64(1209600),
 268         "crunchLimitLogBytesPerJob":          float64(67108864),
 269         "crunchLogThrottleBytes":             float64(65536),
 270         "crunchLogThrottlePeriod":            float64(60),
 271         "crunchLogThrottleLines":             float64(1024),
 272         "crunchLogPartialLineThrottlePeriod": float64(5),
 273         "crunchLogBytesPerEvent":             float64(4096),
 274         "crunchLogSecondsBetweenEvents":      float64(1),
 275 }
 276
 277 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 278         return discoveryMap[key], nil
 279 }
 280
 281 // CalledWith returns the parameters from the first API call whose
 282 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 283 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 284 // no call matches, it returns nil.
 285 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 286 call:
 287         for _, content := range client.Content {
 288                 var v interface{} = content
 289                 for _, k := range strings.Split(jpath, ".") {
 290                         if dict, ok := v.(arvadosclient.Dict); !ok {
 291                                 continue call
 292                         } else {
 293                                 v = dict[k]
 294                         }
 295                 }
 296                 if v == expect {
 297                         return content
 298                 }
 299         }
 300         return nil
 301 }
 302
 303 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 304         return locator, nil
 305 }
 306
 307 func (client *KeepTestClient) PutB(buf []byte) (string, int, error) {
 308         client.Content = buf
 309         return fmt.Sprintf("%x+%d", md5.Sum(buf), len(buf)), len(buf), nil
 310 }
 311
 312 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 313         return 0, errors.New("not implemented")
 314 }
 315
 316 func (client *KeepTestClient) ClearBlockCache() {
 317 }
 318
 319 func (client *KeepTestClient) Close() {
 320         client.Content = nil
 321 }
 322
 323 type FileWrapper struct {
 324         io.ReadCloser
 325         len int64
 326 }
 327
 328 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 329         return nil, errors.New("not implemented")
 330 }
 331
 332 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 333         return 0, errors.New("not implemented")
 334 }
 335
 336 func (fw FileWrapper) Size() int64 {
 337         return fw.len
 338 }
 339
 340 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 341         return nil, errors.New("not implemented")
 342 }
 343
 344 func (fw FileWrapper) Truncate(int64) error {
 345         return errors.New("not implemented")
 346 }
 347
 348 func (fw FileWrapper) Write([]byte) (int, error) {
 349         return 0, errors.New("not implemented")
 350 }
 351
 352 func (fw FileWrapper) Sync() error {
 353         return errors.New("not implemented")
 354 }
 355
 356 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 357         if filename == hwImageID+".tar" {
 358                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 359                 client.Called = true
 360                 return FileWrapper{rdr, 1321984}, nil
 361         } else if filename == "/file1_in_main.txt" {
 362                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 363                 client.Called = true
 364                 return FileWrapper{rdr, 3}, nil
 365         }
 366         return nil, nil
 367 }
 368
 369 func (s *TestSuite) TestLoadImage(c *C) {
 370         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 371         s.runner.Container.Mounts = map[string]arvados.Mount{
 372                 "/out": {Kind: "tmp", Writable: true},
 373         }
 374         s.runner.Container.OutputPath = "/out"
 375
 376         _, err := s.runner.SetupMounts()
 377         c.Assert(err, IsNil)
 378
 379         imageID, err := s.runner.LoadImage()
 380         c.Check(err, IsNil)
 381         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 382         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 383
 384         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 385         s.executor.imageLoaded = false
 386         s.executor.loaded = ""
 387         s.executor.loadErr = errors.New("bork")
 388         imageID, err = s.runner.LoadImage()
 389         c.Check(err, ErrorMatches, ".*bork")
 390         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 391
 392         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 393         s.executor.imageLoaded = false
 394         s.executor.loaded = ""
 395         s.executor.loadErr = nil
 396         imageID, err = s.runner.LoadImage()
 397         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 398         c.Check(s.executor.loaded, Equals, "")
 399
 400         // if executor reports image is already loaded, LoadImage should not be called
 401         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 402         s.executor.imageLoaded = true
 403         s.executor.loaded = ""
 404         s.executor.loadErr = nil
 405         imageID, err = s.runner.LoadImage()
 406         c.Check(err, IsNil)
 407         c.Check(s.executor.loaded, Equals, "")
 408         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 409 }
 410
 411 type ArvErrorTestClient struct{}
 412
 413 func (ArvErrorTestClient) Create(resourceType string,
 414         parameters arvadosclient.Dict,
 415         output interface{}) error {
 416         return nil
 417 }
 418
 419 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 420         if method == "GET" && resourceType == "containers" && action == "auth" {
 421                 return nil
 422         }
 423         return errors.New("ArvError")
 424 }
 425
 426 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 427         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 428         return nil, errors.New("ArvError")
 429 }
 430
 431 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 432         return errors.New("ArvError")
 433 }
 434
 435 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 436         return nil
 437 }
 438
 439 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 440         return discoveryMap[key], nil
 441 }
 442
 443 type KeepErrorTestClient struct {
 444         KeepTestClient
 445 }
 446
 447 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 448         return nil, errors.New("KeepError")
 449 }
 450
 451 func (*KeepErrorTestClient) PutB(buf []byte) (string, int, error) {
 452         return "", 0, errors.New("KeepError")
 453 }
 454
 455 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 456         return "", errors.New("KeepError")
 457 }
 458
 459 type KeepReadErrorTestClient struct {
 460         KeepTestClient
 461 }
 462
 463 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 464         return 0, errors.New("KeepError")
 465 }
 466
 467 type ErrorReader struct {
 468         FileWrapper
 469 }
 470
 471 func (ErrorReader) Read(p []byte) (n int, err error) {
 472         return 0, errors.New("ErrorReader")
 473 }
 474
 475 func (ErrorReader) Seek(int64, int) (int64, error) {
 476         return 0, errors.New("ErrorReader")
 477 }
 478
 479 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 480         return ErrorReader{}, nil
 481 }
 482
 483 type ClosableBuffer struct {
 484         bytes.Buffer
 485 }
 486
 487 func (*ClosableBuffer) Close() error {
 488         return nil
 489 }
 490
 491 type TestLogs struct {
 492         Stdout ClosableBuffer
 493         Stderr ClosableBuffer
 494 }
 495
 496 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 497         if logstr == "stdout" {
 498                 return &tl.Stdout, nil
 499         }
 500         if logstr == "stderr" {
 501                 return &tl.Stderr, nil
 502         }
 503         return nil, errors.New("???")
 504 }
 505
 506 func dockerLog(fd byte, msg string) []byte {
 507         by := []byte(msg)
 508         header := make([]byte, 8+len(by))
 509         header[0] = fd
 510         header[7] = byte(len(by))
 511         copy(header[8:], by)
 512         return header
 513 }
 514
 515 func (s *TestSuite) TestRunContainer(c *C) {
 516         s.executor.runFunc = func() {
 517                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 518                 s.executor.created.Stdout.Close()
 519                 s.executor.created.Stderr.Close()
 520                 s.executor.exit <- 0
 521         }
 522
 523         var logs TestLogs
 524         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 525         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 526         s.runner.Container.Command = []string{"./hw"}
 527
 528         imageID, err := s.runner.LoadImage()
 529         c.Assert(err, IsNil)
 530
 531         err = s.runner.CreateContainer(imageID, nil)
 532         c.Assert(err, IsNil)
 533
 534         err = s.runner.StartContainer()
 535         c.Assert(err, IsNil)
 536
 537         err = s.runner.WaitFinish()
 538         c.Assert(err, IsNil)
 539
 540         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 541         c.Check(logs.Stderr.String(), Equals, "")
 542 }
 543
 544 func (s *TestSuite) TestCommitLogs(c *C) {
 545         api := &ArvTestClient{}
 546         kc := &KeepTestClient{}
 547         defer kc.Close()
 548         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 549         c.Assert(err, IsNil)
 550         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 551
 552         cr.CrunchLog.Print("Hello world!")
 553         cr.CrunchLog.Print("Goodbye")
 554         cr.finalState = "Complete"
 555
 556         err = cr.CommitLogs()
 557         c.Check(err, IsNil)
 558
 559         c.Check(api.Calls, Equals, 2)
 560         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 561         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 562         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 563         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 564 }
 565
 566 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 567         api := &ArvTestClient{}
 568         kc := &KeepTestClient{}
 569         defer kc.Close()
 570         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 571         c.Assert(err, IsNil)
 572
 573         err = cr.UpdateContainerRunning()
 574         c.Check(err, IsNil)
 575
 576         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 577 }
 578
 579 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 580         api := &ArvTestClient{}
 581         kc := &KeepTestClient{}
 582         defer kc.Close()
 583         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 584         c.Assert(err, IsNil)
 585
 586         cr.LogsPDH = new(string)
 587         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 588
 589         cr.ExitCode = new(int)
 590         *cr.ExitCode = 42
 591         cr.finalState = "Complete"
 592
 593         err = cr.UpdateContainerFinal()
 594         c.Check(err, IsNil)
 595
 596         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 597         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 598         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 599 }
 600
 601 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 602         api := &ArvTestClient{}
 603         kc := &KeepTestClient{}
 604         defer kc.Close()
 605         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 606         c.Assert(err, IsNil)
 607         cr.cCancelled = true
 608         cr.finalState = "Cancelled"
 609
 610         err = cr.UpdateContainerFinal()
 611         c.Check(err, IsNil)
 612
 613         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 614         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 615         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 616 }
 617
 618 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 619 // dress rehearsal of the Run() function, starting from a JSON container record.
 620 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 621         err := json.Unmarshal([]byte(record), &s.api.Container)
 622         c.Assert(err, IsNil)
 623         initialState := s.api.Container.State
 624
 625         var sm struct {
 626                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 627         }
 628         err = json.Unmarshal([]byte(record), &sm)
 629         c.Check(err, IsNil)
 630         secretMounts, err := json.Marshal(sm)
 631         c.Assert(err, IsNil)
 632         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 633
 634         s.executor.runFunc = func() {
 635                 fn()
 636                 s.executor.exit <- exitCode
 637         }
 638
 639         s.runner.statInterval = 100 * time.Millisecond
 640         s.runner.containerWatchdogInterval = time.Second
 641         am := &ArvMountCmdLine{}
 642         s.runner.RunArvMount = am.ArvMountTest
 643
 644         realTemp := c.MkDir()
 645         tempcount := 0
 646         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 647                 tempcount++
 648                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 649                 err := os.Mkdir(d, os.ModePerm)
 650                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 651                         // Test case must have pre-populated the tempdir
 652                         err = nil
 653                 }
 654                 return d, err
 655         }
 656         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 657                 return &ArvTestClient{secretMounts: secretMounts}, &KeepTestClient{}, nil, nil
 658         }
 659
 660         if extraMounts != nil && len(extraMounts) > 0 {
 661                 err := s.runner.SetupArvMountPoint("keep")
 662                 c.Check(err, IsNil)
 663
 664                 for _, m := range extraMounts {
 665                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 666                 }
 667         }
 668
 669         err = s.runner.Run()
 670         if s.api.CalledWith("container.state", "Complete") != nil {
 671                 c.Check(err, IsNil)
 672         }
 673         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 674                 c.Check(s.api.WasSetRunning, Equals, true)
 675                 var lastupdate arvadosclient.Dict
 676                 for _, content := range s.api.Content {
 677                         if content["container"] != nil {
 678                                 lastupdate = content["container"].(arvadosclient.Dict)
 679                         }
 680                 }
 681                 if lastupdate["log"] == nil {
 682                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 683                 }
 684         }
 685
 686         if err != nil {
 687                 for k, v := range s.api.Logs {
 688                         c.Log(k)
 689                         c.Log(v.String())
 690                 }
 691         }
 692
 693         return s.api, s.runner, realTemp
 694 }
 695
 696 func (s *TestSuite) TestFullRunHello(c *C) {
 697         s.runner.networkMode = "default"
 698         s.fullRunHelper(c, `{
 699     "command": ["echo", "hello world"],
 700     "container_image": "`+arvadostest.DockerImage112PDH+`",
 701     "cwd": ".",
 702     "environment": {"foo":"bar","baz":"waz"},
 703     "mounts": {"/tmp": {"kind": "tmp"} },
 704     "output_path": "/tmp",
 705     "priority": 1,
 706     "runtime_constraints": {"vcpus":1,"ram":1000000},
 707     "state": "Locked"
 708 }`, nil, 0, func() {
 709                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 710                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 711                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 712                 c.Check(s.executor.created.VCPUs, Equals, 1)
 713                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 714                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 715                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 716                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 717         })
 718
 719         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 720         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 721         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 722
 723 }
 724
 725 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 726         var ran bool
 727         s.fullRunHelper(c, `{
 728     "command": ["sleep", "3"],
 729     "container_image": "`+arvadostest.DockerImage112PDH+`",
 730     "cwd": ".",
 731     "environment": {},
 732     "mounts": {"/tmp": {"kind": "tmp"} },
 733     "output_path": "/tmp",
 734     "priority": 1,
 735     "runtime_constraints": {},
 736     "scheduling_parameters":{"max_run_time": 1},
 737     "state": "Running"
 738 }`, nil, 2, func() {
 739                 ran = true
 740         })
 741         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 742         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 743         c.Check(ran, Equals, false)
 744 }
 745
 746 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 747         s.fullRunHelper(c, `{
 748     "command": ["sleep", "3"],
 749     "container_image": "`+arvadostest.DockerImage112PDH+`",
 750     "cwd": ".",
 751     "environment": {},
 752     "mounts": {"/tmp": {"kind": "tmp"} },
 753     "output_path": "/tmp",
 754     "priority": 1,
 755     "runtime_constraints": {},
 756     "scheduling_parameters":{"max_run_time": 1},
 757     "state": "Locked"
 758 }`, nil, 0, func() {
 759                 time.Sleep(3 * time.Second)
 760         })
 761
 762         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 763         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 764 }
 765
 766 func (s *TestSuite) TestContainerWaitFails(c *C) {
 767         s.fullRunHelper(c, `{
 768     "command": ["sleep", "3"],
 769     "container_image": "`+arvadostest.DockerImage112PDH+`",
 770     "cwd": ".",
 771     "mounts": {"/tmp": {"kind": "tmp"} },
 772     "output_path": "/tmp",
 773     "priority": 1,
 774     "state": "Locked"
 775 }`, nil, 0, func() {
 776                 s.executor.waitErr = errors.New("Container is not running")
 777         })
 778
 779         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 780         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 781 }
 782
 783 func (s *TestSuite) TestCrunchstat(c *C) {
 784         s.fullRunHelper(c, `{
 785                 "command": ["sleep", "1"],
 786                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 787                 "cwd": ".",
 788                 "environment": {},
 789                 "mounts": {"/tmp": {"kind": "tmp"} },
 790                 "output_path": "/tmp",
 791                 "priority": 1,
 792                 "runtime_constraints": {},
 793                 "state": "Locked"
 794         }`, nil, 0, func() {
 795                 time.Sleep(time.Second)
 796         })
 797
 798         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 799         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 800
 801         // We didn't actually start a container, so crunchstat didn't
 802         // find accounting files and therefore didn't log any stats.
 803         // It should have logged a "can't find accounting files"
 804         // message after one poll interval, though, so we can confirm
 805         // it's alive:
 806         c.Assert(s.api.Logs["crunchstat"], NotNil)
 807         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 808
 809         // The "files never appeared" log assures us that we called
 810         // (*crunchstat.Reporter)Stop(), and that we set it up with
 811         // the correct container ID "abcde":
 812         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 813 }
 814
 815 func (s *TestSuite) TestNodeInfoLog(c *C) {
 816         os.Setenv("SLURMD_NODENAME", "compute2")
 817         s.fullRunHelper(c, `{
 818                 "command": ["sleep", "1"],
 819                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 820                 "cwd": ".",
 821                 "environment": {},
 822                 "mounts": {"/tmp": {"kind": "tmp"} },
 823                 "output_path": "/tmp",
 824                 "priority": 1,
 825                 "runtime_constraints": {},
 826                 "state": "Locked"
 827         }`, nil, 0,
 828                 func() {
 829                         time.Sleep(time.Second)
 830                 })
 831
 832         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 833         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 834
 835         c.Assert(s.api.Logs["node"], NotNil)
 836         json := s.api.Logs["node"].String()
 837         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 838         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 839         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 840
 841         c.Assert(s.api.Logs["node-info"], NotNil)
 842         json = s.api.Logs["node-info"].String()
 843         c.Check(json, Matches, `(?ms).*Host Information.*`)
 844         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 845         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 846         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 847         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 848 }
 849
 850 func (s *TestSuite) TestContainerRecordLog(c *C) {
 851         s.fullRunHelper(c, `{
 852                 "command": ["sleep", "1"],
 853                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 854                 "cwd": ".",
 855                 "environment": {},
 856                 "mounts": {"/tmp": {"kind": "tmp"} },
 857                 "output_path": "/tmp",
 858                 "priority": 1,
 859                 "runtime_constraints": {},
 860                 "state": "Locked"
 861         }`, nil, 0,
 862                 func() {
 863                         time.Sleep(time.Second)
 864                 })
 865
 866         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 867         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 868
 869         c.Assert(s.api.Logs["container"], NotNil)
 870         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 871 }
 872
 873 func (s *TestSuite) TestFullRunStderr(c *C) {
 874         s.fullRunHelper(c, `{
 875     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 876     "container_image": "`+arvadostest.DockerImage112PDH+`",
 877     "cwd": ".",
 878     "environment": {},
 879     "mounts": {"/tmp": {"kind": "tmp"} },
 880     "output_path": "/tmp",
 881     "priority": 1,
 882     "runtime_constraints": {},
 883     "state": "Locked"
 884 }`, nil, 1, func() {
 885                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 886                 fmt.Fprintln(s.executor.created.Stderr, "world")
 887         })
 888
 889         final := s.api.CalledWith("container.state", "Complete")
 890         c.Assert(final, NotNil)
 891         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 892         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 893
 894         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 895         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 896 }
 897
 898 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 899         s.fullRunHelper(c, `{
 900     "command": ["pwd"],
 901     "container_image": "`+arvadostest.DockerImage112PDH+`",
 902     "cwd": ".",
 903     "environment": {},
 904     "mounts": {"/tmp": {"kind": "tmp"} },
 905     "output_path": "/tmp",
 906     "priority": 1,
 907     "runtime_constraints": {},
 908     "state": "Locked"
 909 }`, nil, 0, func() {
 910                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 911         })
 912
 913         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 914         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 915         c.Log(s.api.Logs["stdout"])
 916         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 917 }
 918
 919 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 920         s.fullRunHelper(c, `{
 921     "command": ["pwd"],
 922     "container_image": "`+arvadostest.DockerImage112PDH+`",
 923     "cwd": "/bin",
 924     "environment": {},
 925     "mounts": {"/tmp": {"kind": "tmp"} },
 926     "output_path": "/tmp",
 927     "priority": 1,
 928     "runtime_constraints": {},
 929     "state": "Locked"
 930 }`, nil, 0, func() {
 931                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 932         })
 933
 934         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 935         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 936         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 937 }
 938
 939 func (s *TestSuite) TestStopOnSignal(c *C) {
 940         s.executor.runFunc = func() {
 941                 s.executor.created.Stdout.Write([]byte("foo\n"))
 942                 s.runner.SigChan <- syscall.SIGINT
 943         }
 944         s.testStopContainer(c)
 945 }
 946
 947 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
 948         s.executor.runFunc = func() {
 949                 s.executor.created.Stdout.Write([]byte("foo\n"))
 950                 s.runner.ArvMountExit <- nil
 951                 close(s.runner.ArvMountExit)
 952         }
 953         s.runner.ArvMountExit = make(chan error)
 954         s.testStopContainer(c)
 955 }
 956
 957 func (s *TestSuite) testStopContainer(c *C) {
 958         record := `{
 959     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
 960     "container_image": "` + arvadostest.DockerImage112PDH + `",
 961     "cwd": ".",
 962     "environment": {},
 963     "mounts": {"/tmp": {"kind": "tmp"} },
 964     "output_path": "/tmp",
 965     "priority": 1,
 966     "runtime_constraints": {},
 967     "state": "Locked"
 968 }`
 969
 970         err := json.Unmarshal([]byte(record), &s.api.Container)
 971         c.Assert(err, IsNil)
 972
 973         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
 974         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 975                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
 976         }
 977
 978         done := make(chan error)
 979         go func() {
 980                 done <- s.runner.Run()
 981         }()
 982         select {
 983         case <-time.After(20 * time.Second):
 984                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
 985                 c.Fatal("timed out")
 986         case err = <-done:
 987                 c.Check(err, IsNil)
 988         }
 989         for k, v := range s.api.Logs {
 990                 c.Log(k)
 991                 c.Log(v.String(), "\n")
 992         }
 993
 994         c.Check(s.api.CalledWith("container.log", nil), NotNil)
 995         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 996         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
 997 }
 998
 999 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1000         s.fullRunHelper(c, `{
1001     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1002     "container_image": "`+arvadostest.DockerImage112PDH+`",
1003     "cwd": "/bin",
1004     "environment": {"FROBIZ": "bilbo"},
1005     "mounts": {"/tmp": {"kind": "tmp"} },
1006     "output_path": "/tmp",
1007     "priority": 1,
1008     "runtime_constraints": {},
1009     "state": "Locked"
1010 }`, nil, 0, func() {
1011                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1012         })
1013
1014         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1015         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1016         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1017 }
1018
1019 type ArvMountCmdLine struct {
1020         Cmd   []string
1021         token string
1022 }
1023
1024 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1025         am.Cmd = c
1026         am.token = token
1027         return nil, nil
1028 }
1029
1030 func stubCert(temp string) string {
1031         path := temp + "/ca-certificates.crt"
1032         crt, _ := os.Create(path)
1033         crt.Close()
1034         arvadosclient.CertFiles = []string{path}
1035         return path
1036 }
1037
1038 func (s *TestSuite) TestSetupMounts(c *C) {
1039         cr := s.runner
1040         am := &ArvMountCmdLine{}
1041         cr.RunArvMount = am.ArvMountTest
1042         cr.ContainerArvClient = &ArvTestClient{}
1043         cr.ContainerKeepClient = &KeepTestClient{}
1044
1045         realTemp := c.MkDir()
1046         certTemp := c.MkDir()
1047         stubCertPath := stubCert(certTemp)
1048         cr.parentTemp = realTemp
1049
1050         i := 0
1051         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1052                 i++
1053                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1054                 err := os.Mkdir(d, os.ModePerm)
1055                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1056                         // Test case must have pre-populated the tempdir
1057                         err = nil
1058                 }
1059                 return d, err
1060         }
1061
1062         checkEmpty := func() {
1063                 // Should be deleted.
1064                 _, err := os.Stat(realTemp)
1065                 c.Assert(os.IsNotExist(err), Equals, true)
1066
1067                 // Now recreate it for the next test.
1068                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1069         }
1070
1071         {
1072                 i = 0
1073                 cr.ArvMountPoint = ""
1074                 cr.Container.Mounts = make(map[string]arvados.Mount)
1075                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1076                 cr.Container.OutputPath = "/tmp"
1077                 cr.statInterval = 5 * time.Second
1078                 bindmounts, err := cr.SetupMounts()
1079                 c.Check(err, IsNil)
1080                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1081                         "--read-write", "--crunchstat-interval=5",
1082                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1083                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1084                 os.RemoveAll(cr.ArvMountPoint)
1085                 cr.CleanupDirs()
1086                 checkEmpty()
1087         }
1088
1089         {
1090                 i = 0
1091                 cr.ArvMountPoint = ""
1092                 cr.Container.Mounts = make(map[string]arvados.Mount)
1093                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1094                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1095                 cr.Container.OutputPath = "/out"
1096
1097                 bindmounts, err := cr.SetupMounts()
1098                 c.Check(err, IsNil)
1099                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1100                         "--read-write", "--crunchstat-interval=5",
1101                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1102                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1103                 os.RemoveAll(cr.ArvMountPoint)
1104                 cr.CleanupDirs()
1105                 checkEmpty()
1106         }
1107
1108         {
1109                 i = 0
1110                 cr.ArvMountPoint = ""
1111                 cr.Container.Mounts = make(map[string]arvados.Mount)
1112                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1113                 cr.Container.OutputPath = "/tmp"
1114                 cr.Container.RuntimeConstraints.API = true
1115
1116                 bindmounts, err := cr.SetupMounts()
1117                 c.Check(err, IsNil)
1118                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1119                         "--read-write", "--crunchstat-interval=5",
1120                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1121                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1122                 os.RemoveAll(cr.ArvMountPoint)
1123                 cr.CleanupDirs()
1124                 checkEmpty()
1125
1126                 cr.Container.RuntimeConstraints.API = false
1127         }
1128
1129         {
1130                 i = 0
1131                 cr.ArvMountPoint = ""
1132                 cr.Container.Mounts = map[string]arvados.Mount{
1133                         "/keeptmp": {Kind: "collection", Writable: true},
1134                 }
1135                 cr.Container.OutputPath = "/keeptmp"
1136
1137                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1138
1139                 bindmounts, err := cr.SetupMounts()
1140                 c.Check(err, IsNil)
1141                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1142                         "--read-write", "--crunchstat-interval=5",
1143                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1144                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1145                 os.RemoveAll(cr.ArvMountPoint)
1146                 cr.CleanupDirs()
1147                 checkEmpty()
1148         }
1149
1150         {
1151                 i = 0
1152                 cr.ArvMountPoint = ""
1153                 cr.Container.Mounts = map[string]arvados.Mount{
1154                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1155                         "/keepout": {Kind: "collection", Writable: true},
1156                 }
1157                 cr.Container.OutputPath = "/keepout"
1158
1159                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1160                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1161
1162                 bindmounts, err := cr.SetupMounts()
1163                 c.Check(err, IsNil)
1164                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1165                         "--read-write", "--crunchstat-interval=5",
1166                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1167                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1168                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1169                         "/keepout": {realTemp + "/keep1/tmp0", false},
1170                 })
1171                 os.RemoveAll(cr.ArvMountPoint)
1172                 cr.CleanupDirs()
1173                 checkEmpty()
1174         }
1175
1176         {
1177                 i = 0
1178                 cr.ArvMountPoint = ""
1179                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1180                 cr.Container.Mounts = map[string]arvados.Mount{
1181                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1182                         "/keepout": {Kind: "collection", Writable: true},
1183                 }
1184                 cr.Container.OutputPath = "/keepout"
1185
1186                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1187                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1188
1189                 bindmounts, err := cr.SetupMounts()
1190                 c.Check(err, IsNil)
1191                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1192                         "--read-write", "--crunchstat-interval=5",
1193                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1194                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1195                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1196                         "/keepout": {realTemp + "/keep1/tmp0", false},
1197                 })
1198                 os.RemoveAll(cr.ArvMountPoint)
1199                 cr.CleanupDirs()
1200                 checkEmpty()
1201         }
1202
1203         for _, test := range []struct {
1204                 in  interface{}
1205                 out string
1206         }{
1207                 {in: "foo", out: `"foo"`},
1208                 {in: nil, out: `null`},
1209                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1210         } {
1211                 i = 0
1212                 cr.ArvMountPoint = ""
1213                 cr.Container.Mounts = map[string]arvados.Mount{
1214                         "/mnt/test.json": {Kind: "json", Content: test.in},
1215                 }
1216                 bindmounts, err := cr.SetupMounts()
1217                 c.Check(err, IsNil)
1218                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1219                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1220                 })
1221                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1222                 c.Check(err, IsNil)
1223                 c.Check(content, DeepEquals, []byte(test.out))
1224                 os.RemoveAll(cr.ArvMountPoint)
1225                 cr.CleanupDirs()
1226                 checkEmpty()
1227         }
1228
1229         for _, test := range []struct {
1230                 in  interface{}
1231                 out string
1232         }{
1233                 {in: "foo", out: `foo`},
1234                 {in: nil, out: "error"},
1235                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1236         } {
1237                 i = 0
1238                 cr.ArvMountPoint = ""
1239                 cr.Container.Mounts = map[string]arvados.Mount{
1240                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1241                 }
1242                 bindmounts, err := cr.SetupMounts()
1243                 if test.out == "error" {
1244                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1245                 } else {
1246                         c.Check(err, IsNil)
1247                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1248                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1249                         })
1250                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1251                         c.Check(err, IsNil)
1252                         c.Check(content, DeepEquals, []byte(test.out))
1253                 }
1254                 os.RemoveAll(cr.ArvMountPoint)
1255                 cr.CleanupDirs()
1256                 checkEmpty()
1257         }
1258
1259         // Read-only mount points are allowed underneath output_dir mount point
1260         {
1261                 i = 0
1262                 cr.ArvMountPoint = ""
1263                 cr.Container.Mounts = make(map[string]arvados.Mount)
1264                 cr.Container.Mounts = map[string]arvados.Mount{
1265                         "/tmp":     {Kind: "tmp"},
1266                         "/tmp/foo": {Kind: "collection"},
1267                 }
1268                 cr.Container.OutputPath = "/tmp"
1269
1270                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1271
1272                 bindmounts, err := cr.SetupMounts()
1273                 c.Check(err, IsNil)
1274                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1275                         "--read-write", "--crunchstat-interval=5",
1276                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1277                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1278                         "/tmp":     {realTemp + "/tmp2", false},
1279                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1280                 })
1281                 os.RemoveAll(cr.ArvMountPoint)
1282                 cr.CleanupDirs()
1283                 checkEmpty()
1284         }
1285
1286         // Writable mount points copied to output_dir mount point
1287         {
1288                 i = 0
1289                 cr.ArvMountPoint = ""
1290                 cr.Container.Mounts = make(map[string]arvados.Mount)
1291                 cr.Container.Mounts = map[string]arvados.Mount{
1292                         "/tmp": {Kind: "tmp"},
1293                         "/tmp/foo": {Kind: "collection",
1294                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1295                                 Writable:         true},
1296                         "/tmp/bar": {Kind: "collection",
1297                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1298                                 Path:             "baz",
1299                                 Writable:         true},
1300                 }
1301                 cr.Container.OutputPath = "/tmp"
1302
1303                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1304                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1305
1306                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1307                 rf.Write([]byte("bar"))
1308                 rf.Close()
1309
1310                 _, err := cr.SetupMounts()
1311                 c.Check(err, IsNil)
1312                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1313                 c.Check(err, IsNil)
1314                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1315                 c.Check(err, IsNil)
1316                 os.RemoveAll(cr.ArvMountPoint)
1317                 cr.CleanupDirs()
1318                 checkEmpty()
1319         }
1320
1321         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1322         {
1323                 i = 0
1324                 cr.ArvMountPoint = ""
1325                 cr.Container.Mounts = make(map[string]arvados.Mount)
1326                 cr.Container.Mounts = map[string]arvados.Mount{
1327                         "/tmp":     {Kind: "tmp"},
1328                         "/tmp/foo": {Kind: "tmp"},
1329                 }
1330                 cr.Container.OutputPath = "/tmp"
1331
1332                 _, err := cr.SetupMounts()
1333                 c.Check(err, NotNil)
1334                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1335                 os.RemoveAll(cr.ArvMountPoint)
1336                 cr.CleanupDirs()
1337                 checkEmpty()
1338         }
1339
1340         // Only mount point of kind 'collection' is allowed for stdin
1341         {
1342                 i = 0
1343                 cr.ArvMountPoint = ""
1344                 cr.Container.Mounts = make(map[string]arvados.Mount)
1345                 cr.Container.Mounts = map[string]arvados.Mount{
1346                         "stdin": {Kind: "tmp"},
1347                 }
1348
1349                 _, err := cr.SetupMounts()
1350                 c.Check(err, NotNil)
1351                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1352                 os.RemoveAll(cr.ArvMountPoint)
1353                 cr.CleanupDirs()
1354                 checkEmpty()
1355         }
1356
1357         // git_tree mounts
1358         {
1359                 i = 0
1360                 cr.ArvMountPoint = ""
1361                 (*GitMountSuite)(nil).useTestGitServer(c)
1362                 cr.token = arvadostest.ActiveToken
1363                 cr.Container.Mounts = make(map[string]arvados.Mount)
1364                 cr.Container.Mounts = map[string]arvados.Mount{
1365                         "/tip": {
1366                                 Kind:   "git_tree",
1367                                 UUID:   arvadostest.Repository2UUID,
1368                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1369                                 Path:   "/",
1370                         },
1371                         "/non-tip": {
1372                                 Kind:   "git_tree",
1373                                 UUID:   arvadostest.Repository2UUID,
1374                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1375                                 Path:   "/",
1376                         },
1377                 }
1378                 cr.Container.OutputPath = "/tmp"
1379
1380                 bindmounts, err := cr.SetupMounts()
1381                 c.Check(err, IsNil)
1382
1383                 for path, mount := range bindmounts {
1384                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1385                 }
1386
1387                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1388                 c.Check(err, IsNil)
1389                 c.Check(string(data), Equals, "\000\001\002\003")
1390                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1391                 c.Check(err, FitsTypeOf, &os.PathError{})
1392                 c.Check(os.IsNotExist(err), Equals, true)
1393
1394                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1395                 c.Check(err, IsNil)
1396                 c.Check(string(data), Equals, "\000\001\002\003")
1397                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1398                 c.Check(err, IsNil)
1399                 c.Check(string(data), Equals, "testfile\n")
1400
1401                 cr.CleanupDirs()
1402                 checkEmpty()
1403         }
1404 }
1405
1406 func (s *TestSuite) TestStdout(c *C) {
1407         helperRecord := `{
1408                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1409                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1410                 "cwd": "/bin",
1411                 "environment": {"FROBIZ": "bilbo"},
1412                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1413                 "output_path": "/tmp",
1414                 "priority": 1,
1415                 "runtime_constraints": {},
1416                 "state": "Locked"
1417         }`
1418
1419         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1420                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1421         })
1422
1423         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1424         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1425         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1426 }
1427
1428 // Used by the TestStdoutWithWrongPath*()
1429 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1430         err := json.Unmarshal([]byte(record), &s.api.Container)
1431         c.Assert(err, IsNil)
1432         s.executor.runFunc = fn
1433         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1434         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1435                 return s.api, &KeepTestClient{}, nil, nil
1436         }
1437         return s.api, s.runner, s.runner.Run()
1438 }
1439
1440 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1441         _, _, err := s.stdoutErrorRunHelper(c, `{
1442     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1443     "output_path": "/tmp",
1444     "state": "Locked"
1445 }`, func() {})
1446         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1447 }
1448
1449 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1450         _, _, err := s.stdoutErrorRunHelper(c, `{
1451     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1452     "output_path": "/tmp",
1453     "state": "Locked"
1454 }`, func() {})
1455         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1456 }
1457
1458 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1459         _, _, err := s.stdoutErrorRunHelper(c, `{
1460     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1461     "output_path": "/tmp",
1462     "state": "Locked"
1463 }`, func() {})
1464         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1465 }
1466
1467 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1468         s.fullRunHelper(c, `{
1469     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1470     "container_image": "`+arvadostest.DockerImage112PDH+`",
1471     "cwd": "/bin",
1472     "environment": {},
1473     "mounts": {"/tmp": {"kind": "tmp"} },
1474     "output_path": "/tmp",
1475     "priority": 1,
1476     "runtime_constraints": {"API": true},
1477     "state": "Locked"
1478 }`, nil, 0, func() {
1479                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1480                 s.executor.exit <- 3
1481         })
1482         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1483         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1484 }
1485
1486 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1487         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1488         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1489         s.fullRunHelper(c, `{
1490     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1491     "container_image": "`+arvadostest.DockerImage112PDH+`",
1492     "cwd": "/bin",
1493     "environment": {},
1494     "mounts": {"/tmp": {"kind": "tmp"} },
1495     "output_path": "/tmp",
1496     "priority": 1,
1497     "runtime_constraints": {"API": true},
1498     "state": "Locked"
1499 }`, nil, 0, func() {
1500                 s.api.Container.Output = arvadostest.DockerImage112PDH
1501         })
1502
1503         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1504         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1505         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1506 }
1507
1508 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1509         helperRecord := `{
1510                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1511                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1512                 "cwd": "/bin",
1513                 "environment": {"FROBIZ": "bilbo"},
1514                 "mounts": {
1515         "/tmp": {"kind": "tmp"},
1516         "/tmp/foo": {"kind": "collection",
1517                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1518                      "exclude_from_output": true
1519         },
1520         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1521     },
1522                 "output_path": "/tmp",
1523                 "priority": 1,
1524                 "runtime_constraints": {},
1525                 "state": "Locked"
1526         }`
1527
1528         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1529
1530         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1531                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1532         })
1533
1534         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1535         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1536         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1537 }
1538
1539 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1540         helperRecord := `{
1541                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1542                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1543                 "cwd": "/bin",
1544                 "environment": {"FROBIZ": "bilbo"},
1545                 "mounts": {
1546         "/tmp": {"kind": "tmp"},
1547         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1548         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1549         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1550         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1551         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1552     },
1553                 "output_path": "/tmp",
1554                 "priority": 1,
1555                 "runtime_constraints": {},
1556                 "state": "Locked"
1557         }`
1558
1559         extraMounts := []string{
1560                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1561                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1562                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1563         }
1564
1565         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1566                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1567         })
1568
1569         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1570                 "/tmp":                   {realtemp + "/tmp1", false},
1571                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1572                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1573                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1574                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1575         })
1576
1577         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1578         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1579         for _, v := range api.Content {
1580                 if v["collection"] != nil {
1581                         c.Check(v["ensure_unique_name"], Equals, true)
1582                         collection := v["collection"].(arvadosclient.Dict)
1583                         if strings.Index(collection["name"].(string), "output") == 0 {
1584                                 manifest := collection["manifest_text"].(string)
1585
1586                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1587 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1588 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1589 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1590 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1591 `)
1592                         }
1593                 }
1594         }
1595 }
1596
1597 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1598         helperRecord := `{
1599                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1600                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1601                 "cwd": "/bin",
1602                 "environment": {"FROBIZ": "bilbo"},
1603                 "mounts": {
1604         "/tmp": {"kind": "tmp"},
1605         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1606         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1607     },
1608                 "output_path": "/tmp",
1609                 "priority": 1,
1610                 "runtime_constraints": {},
1611                 "state": "Locked"
1612         }`
1613
1614         extraMounts := []string{
1615                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1616         }
1617
1618         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1619                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1620         })
1621
1622         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1623         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1624         for _, v := range s.api.Content {
1625                 if v["collection"] != nil {
1626                         collection := v["collection"].(arvadosclient.Dict)
1627                         if strings.Index(collection["name"].(string), "output") == 0 {
1628                                 manifest := collection["manifest_text"].(string)
1629
1630                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1631 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1632 `)
1633                         }
1634                 }
1635         }
1636 }
1637
1638 func (s *TestSuite) TestOutputError(c *C) {
1639         helperRecord := `{
1640                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1641                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1642                 "cwd": "/bin",
1643                 "environment": {"FROBIZ": "bilbo"},
1644                 "mounts": {
1645                         "/tmp": {"kind": "tmp"}
1646                 },
1647                 "output_path": "/tmp",
1648                 "priority": 1,
1649                 "runtime_constraints": {},
1650                 "state": "Locked"
1651         }`
1652         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1653                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1654         })
1655
1656         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1657 }
1658
1659 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1660         helperRecord := `{
1661                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1662                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1663                 "cwd": "/bin",
1664                 "environment": {"FROBIZ": "bilbo"},
1665                 "mounts": {
1666         "/tmp": {"kind": "tmp"},
1667         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1668         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1669     },
1670                 "output_path": "/tmp",
1671                 "priority": 1,
1672                 "runtime_constraints": {},
1673                 "state": "Locked"
1674         }`
1675
1676         extraMounts := []string{
1677                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1678         }
1679
1680         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1681                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1682         })
1683
1684         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1685         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1686         for _, v := range api.Content {
1687                 if v["collection"] != nil {
1688                         collection := v["collection"].(arvadosclient.Dict)
1689                         if strings.Index(collection["name"].(string), "output") == 0 {
1690                                 manifest := collection["manifest_text"].(string)
1691                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1692 `)
1693                         }
1694                 }
1695         }
1696 }
1697
1698 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1699         helperRecord := `{
1700                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1701                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1702                 "cwd": "/bin",
1703                 "environment": {"FROBIZ": "bilbo"},
1704                 "mounts": {
1705         "/tmp": {"kind": "tmp"},
1706         "stdin": {"kind": "json", "content": "foo"},
1707         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1708     },
1709                 "output_path": "/tmp",
1710                 "priority": 1,
1711                 "runtime_constraints": {},
1712                 "state": "Locked"
1713         }`
1714
1715         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1716                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1717         })
1718
1719         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1720         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1721         for _, v := range api.Content {
1722                 if v["collection"] != nil {
1723                         collection := v["collection"].(arvadosclient.Dict)
1724                         if strings.Index(collection["name"].(string), "output") == 0 {
1725                                 manifest := collection["manifest_text"].(string)
1726                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1727 `)
1728                         }
1729                 }
1730         }
1731 }
1732
1733 func (s *TestSuite) TestStderrMount(c *C) {
1734         api, cr, _ := s.fullRunHelper(c, `{
1735     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1736     "container_image": "`+arvadostest.DockerImage112PDH+`",
1737     "cwd": ".",
1738     "environment": {},
1739     "mounts": {"/tmp": {"kind": "tmp"},
1740                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1741                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1742     "output_path": "/tmp",
1743     "priority": 1,
1744     "runtime_constraints": {},
1745     "state": "Locked"
1746 }`, nil, 1, func() {
1747                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1748                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1749         })
1750
1751         final := api.CalledWith("container.state", "Complete")
1752         c.Assert(final, NotNil)
1753         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1754         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1755
1756         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1757 }
1758
1759 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1760         s.api.callraw = true
1761         err := s.runner.fetchContainerRecord()
1762         c.Assert(err, IsNil)
1763         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1764         c.Logf("%#v", s.runner.Container)
1765         c.Check(err, IsNil)
1766         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1767 }
1768
1769 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1770         nextState := ""
1771         for _, setup := range []func(){
1772                 func() {
1773                         c.Log("// waitErr = ocl runtime error")
1774                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1775                         nextState = "Cancelled"
1776                 },
1777                 func() {
1778                         c.Log("// loadErr = cannot connect")
1779                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1780                         *brokenNodeHook = c.MkDir() + "/broken-node-hook"
1781                         err := ioutil.WriteFile(*brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1782                         c.Assert(err, IsNil)
1783                         nextState = "Queued"
1784                 },
1785         } {
1786                 s.SetUpTest(c)
1787                 setup()
1788                 s.fullRunHelper(c, `{
1789     "command": ["echo", "hello world"],
1790     "container_image": "`+arvadostest.DockerImage112PDH+`",
1791     "cwd": ".",
1792     "environment": {},
1793     "mounts": {"/tmp": {"kind": "tmp"} },
1794     "output_path": "/tmp",
1795     "priority": 1,
1796     "runtime_constraints": {},
1797     "state": "Locked"
1798 }`, nil, 0, func() {})
1799                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1800                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1801                 if *brokenNodeHook != "" {
1802                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1803                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1804                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1805                 } else {
1806                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1807                 }
1808         }
1809 }
1810
1811 func (s *TestSuite) TestBadCommand(c *C) {
1812         for _, startError := range []string{
1813                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1814                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1815                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1816         } {
1817                 s.SetUpTest(c)
1818                 s.executor.startErr = errors.New(startError)
1819                 s.fullRunHelper(c, `{
1820     "command": ["echo", "hello world"],
1821     "container_image": "`+arvadostest.DockerImage112PDH+`",
1822     "cwd": ".",
1823     "environment": {},
1824     "mounts": {"/tmp": {"kind": "tmp"} },
1825     "output_path": "/tmp",
1826     "priority": 1,
1827     "runtime_constraints": {},
1828     "state": "Locked"
1829 }`, nil, 0, func() {})
1830                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1831                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1832         }
1833 }
1834
1835 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1836         helperRecord := `{
1837                 "command": ["true"],
1838                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1839                 "cwd": "/bin",
1840                 "mounts": {
1841                     "/tmp": {"kind": "tmp"},
1842                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1843                 },
1844                 "secret_mounts": {
1845                 },
1846                 "output_path": "/tmp",
1847                 "priority": 1,
1848                 "runtime_constraints": {},
1849                 "state": "Locked"
1850         }`
1851
1852         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1853                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1854                 c.Check(err, IsNil)
1855                 c.Check(string(content), Equals, "mypassword")
1856         })
1857
1858         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1859         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1860         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
1861         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
1862
1863         // under secret mounts, not captured in output
1864         helperRecord = `{
1865                 "command": ["true"],
1866                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1867                 "cwd": "/bin",
1868                 "mounts": {
1869                     "/tmp": {"kind": "tmp"}
1870                 },
1871                 "secret_mounts": {
1872                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1873                 },
1874                 "output_path": "/tmp",
1875                 "priority": 1,
1876                 "runtime_constraints": {},
1877                 "state": "Locked"
1878         }`
1879
1880         s.SetUpTest(c)
1881         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1882                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1883                 c.Check(err, IsNil)
1884                 c.Check(string(content), Equals, "mypassword")
1885         })
1886
1887         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1888         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1889         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
1890         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
1891 }
1892
1893 type FakeProcess struct {
1894         cmdLine []string
1895 }
1896
1897 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
1898         return fp.cmdLine, nil
1899 }