lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/manifest"
  29         "golang.org/x/net/context"
  30
  31         . "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func TestCrunchExec(t *testing.T) {
  36         TestingT(t)
  37 }
  38
  39 var _ = Suite(&TestSuite{})
  40
  41 type TestSuite struct {
  42         client                   *arvados.Client
  43         api                      *ArvTestClient
  44         runner                   *ContainerRunner
  45         executor                 *stubExecutor
  46         keepmount                string
  47         testDispatcherKeepClient KeepTestClient
  48         testContainerKeepClient  KeepTestClient
  49 }
  50
  51 func (s *TestSuite) SetUpTest(c *C) {
  52         *brokenNodeHook = ""
  53         s.client = arvados.NewClientFromEnv()
  54         s.executor = &stubExecutor{}
  55         var err error
  56         s.api = &ArvTestClient{}
  57         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  58         c.Assert(err, IsNil)
  59         s.runner.executor = s.executor
  60         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  61                 return s.api, &s.testContainerKeepClient, s.client, nil
  62         }
  63         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  64                 s.runner.ArvMountPoint = s.keepmount
  65                 return nil, nil
  66         }
  67         s.keepmount = c.MkDir()
  68         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  69         c.Assert(err, IsNil)
  70         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  71         c.Assert(err, IsNil)
  72         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  73         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  74         c.Assert(err, IsNil)
  75         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  76         c.Assert(err, IsNil)
  77         s.runner.ArvMountPoint = s.keepmount
  78 }
  79
  80 type ArvTestClient struct {
  81         Total   int64
  82         Calls   int
  83         Content []arvadosclient.Dict
  84         arvados.Container
  85         secretMounts []byte
  86         Logs         map[string]*bytes.Buffer
  87         sync.Mutex
  88         WasSetRunning bool
  89         callraw       bool
  90 }
  91
  92 type KeepTestClient struct {
  93         Called         bool
  94         Content        []byte
  95         StorageClasses []string
  96 }
  97
  98 type stubExecutor struct {
  99         imageLoaded bool
 100         loaded      string
 101         loadErr     error
 102         exitCode    int
 103         createErr   error
 104         created     containerSpec
 105         startErr    error
 106         waitSleep   time.Duration
 107         waitErr     error
 108         stopErr     error
 109         stopped     bool
 110         closed      bool
 111         runFunc     func()
 112         exit        chan int
 113 }
 114
 115 func (e *stubExecutor) ImageLoaded(imageID string) bool { return e.imageLoaded }
 116 func (e *stubExecutor) LoadImage(filename string) error { e.loaded = filename; return e.loadErr }
 117 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 118 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 119 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 120 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 121 func (e *stubExecutor) Close()                          { e.closed = true }
 122 func (e *stubExecutor) Wait(context.Context) (int, error) {
 123         return <-e.exit, e.waitErr
 124 }
 125
 126 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 127
 128 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 129 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 130 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 131
 132 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 133 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 134
 135 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 136 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 137 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 138 `
 139
 140 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 141
 142 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 143 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 144
 145 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 146 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 147
 148 func (client *ArvTestClient) Create(resourceType string,
 149         parameters arvadosclient.Dict,
 150         output interface{}) error {
 151
 152         client.Mutex.Lock()
 153         defer client.Mutex.Unlock()
 154
 155         client.Calls++
 156         client.Content = append(client.Content, parameters)
 157
 158         if resourceType == "logs" {
 159                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 160                 if client.Logs == nil {
 161                         client.Logs = make(map[string]*bytes.Buffer)
 162                 }
 163                 if client.Logs[et] == nil {
 164                         client.Logs[et] = &bytes.Buffer{}
 165                 }
 166                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 167         }
 168
 169         if resourceType == "collections" && output != nil {
 170                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 171                 outmap := output.(*arvados.Collection)
 172                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 173                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 174         }
 175
 176         return nil
 177 }
 178
 179 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 180         switch {
 181         case method == "GET" && resourceType == "containers" && action == "auth":
 182                 return json.Unmarshal([]byte(`{
 183                         "kind": "arvados#api_client_authorization",
 184                         "uuid": "`+fakeAuthUUID+`",
 185                         "api_token": "`+fakeAuthToken+`"
 186                         }`), output)
 187         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 188                 if client.secretMounts != nil {
 189                         return json.Unmarshal(client.secretMounts, output)
 190                 }
 191                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 192         default:
 193                 return fmt.Errorf("Not found")
 194         }
 195 }
 196
 197 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 198         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 199         var j []byte
 200         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 201                 j = []byte(`{
 202                         "kind": "arvados#nodeList",
 203                         "items": [{
 204                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 205                                 "hostname": "compute2",
 206                                 "properties": {"total_cpu_cores": 16}
 207                         }]}`)
 208         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 209                 if uuid == "" {
 210                         j, err = json.Marshal(map[string]interface{}{
 211                                 "items": []interface{}{client.Container},
 212                                 "kind":  "arvados#nodeList",
 213                         })
 214                 } else {
 215                         j, err = json.Marshal(client.Container)
 216                 }
 217         } else {
 218                 j = []byte(`{
 219                         "command": ["sleep", "1"],
 220                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 221                         "cwd": ".",
 222                         "environment": {},
 223                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 224                         "output_path": "/tmp",
 225                         "priority": 1,
 226                         "runtime_constraints": {}
 227                 }`)
 228         }
 229         return ioutil.NopCloser(bytes.NewReader(j)), err
 230 }
 231
 232 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 233         if resourceType == "collections" {
 234                 if uuid == hwPDH {
 235                         output.(*arvados.Collection).ManifestText = hwManifest
 236                 } else if uuid == otherPDH {
 237                         output.(*arvados.Collection).ManifestText = otherManifest
 238                 } else if uuid == normalizedWithSubdirsPDH {
 239                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 240                 } else if uuid == denormalizedWithSubdirsPDH {
 241                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 242                 }
 243         }
 244         if resourceType == "containers" {
 245                 (*output.(*arvados.Container)) = client.Container
 246         }
 247         return nil
 248 }
 249
 250 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 251         client.Mutex.Lock()
 252         defer client.Mutex.Unlock()
 253         client.Calls++
 254         client.Content = append(client.Content, parameters)
 255         if resourceType == "containers" {
 256                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 257                         client.WasSetRunning = true
 258                 }
 259         } else if resourceType == "collections" {
 260                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 261                 output.(*arvados.Collection).UUID = uuid
 262                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 263         }
 264         return nil
 265 }
 266
 267 var discoveryMap = map[string]interface{}{
 268         "defaultTrashLifetime":               float64(1209600),
 269         "crunchLimitLogBytesPerJob":          float64(67108864),
 270         "crunchLogThrottleBytes":             float64(65536),
 271         "crunchLogThrottlePeriod":            float64(60),
 272         "crunchLogThrottleLines":             float64(1024),
 273         "crunchLogPartialLineThrottlePeriod": float64(5),
 274         "crunchLogBytesPerEvent":             float64(4096),
 275         "crunchLogSecondsBetweenEvents":      float64(1),
 276 }
 277
 278 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 279         return discoveryMap[key], nil
 280 }
 281
 282 // CalledWith returns the parameters from the first API call whose
 283 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 284 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 285 // no call matches, it returns nil.
 286 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 287 call:
 288         for _, content := range client.Content {
 289                 var v interface{} = content
 290                 for _, k := range strings.Split(jpath, ".") {
 291                         if dict, ok := v.(arvadosclient.Dict); !ok {
 292                                 continue call
 293                         } else {
 294                                 v = dict[k]
 295                         }
 296                 }
 297                 if v == expect {
 298                         return content
 299                 }
 300         }
 301         return nil
 302 }
 303
 304 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 305         return locator, nil
 306 }
 307
 308 func (client *KeepTestClient) PutB(buf []byte) (string, int, error) {
 309         client.Content = buf
 310         return fmt.Sprintf("%x+%d", md5.Sum(buf), len(buf)), len(buf), nil
 311 }
 312
 313 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 314         return 0, errors.New("not implemented")
 315 }
 316
 317 func (client *KeepTestClient) ClearBlockCache() {
 318 }
 319
 320 func (client *KeepTestClient) Close() {
 321         client.Content = nil
 322 }
 323
 324 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 325         client.StorageClasses = sc
 326 }
 327
 328 type FileWrapper struct {
 329         io.ReadCloser
 330         len int64
 331 }
 332
 333 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 334         return nil, errors.New("not implemented")
 335 }
 336
 337 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 338         return 0, errors.New("not implemented")
 339 }
 340
 341 func (fw FileWrapper) Size() int64 {
 342         return fw.len
 343 }
 344
 345 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 346         return nil, errors.New("not implemented")
 347 }
 348
 349 func (fw FileWrapper) Truncate(int64) error {
 350         return errors.New("not implemented")
 351 }
 352
 353 func (fw FileWrapper) Write([]byte) (int, error) {
 354         return 0, errors.New("not implemented")
 355 }
 356
 357 func (fw FileWrapper) Sync() error {
 358         return errors.New("not implemented")
 359 }
 360
 361 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 362         if filename == hwImageID+".tar" {
 363                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 364                 client.Called = true
 365                 return FileWrapper{rdr, 1321984}, nil
 366         } else if filename == "/file1_in_main.txt" {
 367                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 368                 client.Called = true
 369                 return FileWrapper{rdr, 3}, nil
 370         }
 371         return nil, nil
 372 }
 373
 374 func (s *TestSuite) TestLoadImage(c *C) {
 375         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 376         s.runner.Container.Mounts = map[string]arvados.Mount{
 377                 "/out": {Kind: "tmp", Writable: true},
 378         }
 379         s.runner.Container.OutputPath = "/out"
 380
 381         _, err := s.runner.SetupMounts()
 382         c.Assert(err, IsNil)
 383
 384         imageID, err := s.runner.LoadImage()
 385         c.Check(err, IsNil)
 386         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 387         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 388
 389         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 390         s.executor.imageLoaded = false
 391         s.executor.loaded = ""
 392         s.executor.loadErr = errors.New("bork")
 393         imageID, err = s.runner.LoadImage()
 394         c.Check(err, ErrorMatches, ".*bork")
 395         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 396
 397         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 398         s.executor.imageLoaded = false
 399         s.executor.loaded = ""
 400         s.executor.loadErr = nil
 401         imageID, err = s.runner.LoadImage()
 402         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 403         c.Check(s.executor.loaded, Equals, "")
 404
 405         // if executor reports image is already loaded, LoadImage should not be called
 406         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 407         s.executor.imageLoaded = true
 408         s.executor.loaded = ""
 409         s.executor.loadErr = nil
 410         imageID, err = s.runner.LoadImage()
 411         c.Check(err, IsNil)
 412         c.Check(s.executor.loaded, Equals, "")
 413         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 414 }
 415
 416 type ArvErrorTestClient struct{}
 417
 418 func (ArvErrorTestClient) Create(resourceType string,
 419         parameters arvadosclient.Dict,
 420         output interface{}) error {
 421         return nil
 422 }
 423
 424 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 425         if method == "GET" && resourceType == "containers" && action == "auth" {
 426                 return nil
 427         }
 428         return errors.New("ArvError")
 429 }
 430
 431 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 432         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 433         return nil, errors.New("ArvError")
 434 }
 435
 436 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 437         return errors.New("ArvError")
 438 }
 439
 440 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 441         return nil
 442 }
 443
 444 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 445         return discoveryMap[key], nil
 446 }
 447
 448 type KeepErrorTestClient struct {
 449         KeepTestClient
 450 }
 451
 452 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 453         return nil, errors.New("KeepError")
 454 }
 455
 456 func (*KeepErrorTestClient) PutB(buf []byte) (string, int, error) {
 457         return "", 0, errors.New("KeepError")
 458 }
 459
 460 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 461         return "", errors.New("KeepError")
 462 }
 463
 464 type KeepReadErrorTestClient struct {
 465         KeepTestClient
 466 }
 467
 468 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 469         return 0, errors.New("KeepError")
 470 }
 471
 472 type ErrorReader struct {
 473         FileWrapper
 474 }
 475
 476 func (ErrorReader) Read(p []byte) (n int, err error) {
 477         return 0, errors.New("ErrorReader")
 478 }
 479
 480 func (ErrorReader) Seek(int64, int) (int64, error) {
 481         return 0, errors.New("ErrorReader")
 482 }
 483
 484 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 485         return ErrorReader{}, nil
 486 }
 487
 488 type ClosableBuffer struct {
 489         bytes.Buffer
 490 }
 491
 492 func (*ClosableBuffer) Close() error {
 493         return nil
 494 }
 495
 496 type TestLogs struct {
 497         Stdout ClosableBuffer
 498         Stderr ClosableBuffer
 499 }
 500
 501 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 502         if logstr == "stdout" {
 503                 return &tl.Stdout, nil
 504         }
 505         if logstr == "stderr" {
 506                 return &tl.Stderr, nil
 507         }
 508         return nil, errors.New("???")
 509 }
 510
 511 func dockerLog(fd byte, msg string) []byte {
 512         by := []byte(msg)
 513         header := make([]byte, 8+len(by))
 514         header[0] = fd
 515         header[7] = byte(len(by))
 516         copy(header[8:], by)
 517         return header
 518 }
 519
 520 func (s *TestSuite) TestRunContainer(c *C) {
 521         s.executor.runFunc = func() {
 522                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 523                 s.executor.exit <- 0
 524         }
 525
 526         var logs TestLogs
 527         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 528         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 529         s.runner.Container.Command = []string{"./hw"}
 530         s.runner.Container.OutputStorageClasses = []string{"default"}
 531
 532         imageID, err := s.runner.LoadImage()
 533         c.Assert(err, IsNil)
 534
 535         err = s.runner.CreateContainer(imageID, nil)
 536         c.Assert(err, IsNil)
 537
 538         err = s.runner.StartContainer()
 539         c.Assert(err, IsNil)
 540
 541         err = s.runner.WaitFinish()
 542         c.Assert(err, IsNil)
 543
 544         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 545         c.Check(logs.Stderr.String(), Equals, "")
 546 }
 547
 548 func (s *TestSuite) TestCommitLogs(c *C) {
 549         api := &ArvTestClient{}
 550         kc := &KeepTestClient{}
 551         defer kc.Close()
 552         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 553         c.Assert(err, IsNil)
 554         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 555
 556         cr.CrunchLog.Print("Hello world!")
 557         cr.CrunchLog.Print("Goodbye")
 558         cr.finalState = "Complete"
 559
 560         err = cr.CommitLogs()
 561         c.Check(err, IsNil)
 562
 563         c.Check(api.Calls, Equals, 2)
 564         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 565         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 566         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 567         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 568 }
 569
 570 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 571         api := &ArvTestClient{}
 572         kc := &KeepTestClient{}
 573         defer kc.Close()
 574         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 575         c.Assert(err, IsNil)
 576
 577         err = cr.UpdateContainerRunning()
 578         c.Check(err, IsNil)
 579
 580         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 581 }
 582
 583 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 584         api := &ArvTestClient{}
 585         kc := &KeepTestClient{}
 586         defer kc.Close()
 587         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 588         c.Assert(err, IsNil)
 589
 590         cr.LogsPDH = new(string)
 591         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 592
 593         cr.ExitCode = new(int)
 594         *cr.ExitCode = 42
 595         cr.finalState = "Complete"
 596
 597         err = cr.UpdateContainerFinal()
 598         c.Check(err, IsNil)
 599
 600         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 601         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 602         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 603 }
 604
 605 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 606         api := &ArvTestClient{}
 607         kc := &KeepTestClient{}
 608         defer kc.Close()
 609         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 610         c.Assert(err, IsNil)
 611         cr.cCancelled = true
 612         cr.finalState = "Cancelled"
 613
 614         err = cr.UpdateContainerFinal()
 615         c.Check(err, IsNil)
 616
 617         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 618         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 619         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 620 }
 621
 622 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 623 // dress rehearsal of the Run() function, starting from a JSON container record.
 624 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 625         err := json.Unmarshal([]byte(record), &s.api.Container)
 626         c.Assert(err, IsNil)
 627         initialState := s.api.Container.State
 628
 629         var sm struct {
 630                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 631         }
 632         err = json.Unmarshal([]byte(record), &sm)
 633         c.Check(err, IsNil)
 634         secretMounts, err := json.Marshal(sm)
 635         c.Assert(err, IsNil)
 636         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 637
 638         s.executor.runFunc = func() {
 639                 fn()
 640                 s.executor.exit <- exitCode
 641         }
 642
 643         s.runner.statInterval = 100 * time.Millisecond
 644         s.runner.containerWatchdogInterval = time.Second
 645         am := &ArvMountCmdLine{}
 646         s.runner.RunArvMount = am.ArvMountTest
 647
 648         realTemp := c.MkDir()
 649         tempcount := 0
 650         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 651                 tempcount++
 652                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 653                 err := os.Mkdir(d, os.ModePerm)
 654                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 655                         // Test case must have pre-populated the tempdir
 656                         err = nil
 657                 }
 658                 return d, err
 659         }
 660         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 661                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 662         }
 663
 664         if extraMounts != nil && len(extraMounts) > 0 {
 665                 err := s.runner.SetupArvMountPoint("keep")
 666                 c.Check(err, IsNil)
 667
 668                 for _, m := range extraMounts {
 669                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 670                 }
 671         }
 672
 673         err = s.runner.Run()
 674         if s.api.CalledWith("container.state", "Complete") != nil {
 675                 c.Check(err, IsNil)
 676         }
 677         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 678                 c.Check(s.api.WasSetRunning, Equals, true)
 679                 var lastupdate arvadosclient.Dict
 680                 for _, content := range s.api.Content {
 681                         if content["container"] != nil {
 682                                 lastupdate = content["container"].(arvadosclient.Dict)
 683                         }
 684                 }
 685                 if lastupdate["log"] == nil {
 686                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 687                 }
 688         }
 689
 690         if err != nil {
 691                 for k, v := range s.api.Logs {
 692                         c.Log(k)
 693                         c.Log(v.String())
 694                 }
 695         }
 696
 697         return s.api, s.runner, realTemp
 698 }
 699
 700 func (s *TestSuite) TestFullRunHello(c *C) {
 701         s.runner.enableMemoryLimit = true
 702         s.runner.networkMode = "default"
 703         s.fullRunHelper(c, `{
 704     "command": ["echo", "hello world"],
 705     "container_image": "`+arvadostest.DockerImage112PDH+`",
 706     "cwd": ".",
 707     "environment": {"foo":"bar","baz":"waz"},
 708     "mounts": {"/tmp": {"kind": "tmp"} },
 709     "output_path": "/tmp",
 710     "priority": 1,
 711     "runtime_constraints": {"vcpus":1,"ram":1000000},
 712     "state": "Locked",
 713     "output_storage_classes": ["default"]
 714 }`, nil, 0, func() {
 715                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 716                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 717                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 718                 c.Check(s.executor.created.VCPUs, Equals, 1)
 719                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 720                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 721                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 722                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 723         })
 724
 725         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 726         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 727         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 728         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 729         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 730 }
 731
 732 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 733         var ran bool
 734         s.fullRunHelper(c, `{
 735     "command": ["sleep", "3"],
 736     "container_image": "`+arvadostest.DockerImage112PDH+`",
 737     "cwd": ".",
 738     "environment": {},
 739     "mounts": {"/tmp": {"kind": "tmp"} },
 740     "output_path": "/tmp",
 741     "priority": 1,
 742     "runtime_constraints": {},
 743     "scheduling_parameters":{"max_run_time": 1},
 744     "state": "Running"
 745 }`, nil, 2, func() {
 746                 ran = true
 747         })
 748         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 749         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 750         c.Check(ran, Equals, false)
 751 }
 752
 753 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 754         s.fullRunHelper(c, `{
 755     "command": ["sleep", "3"],
 756     "container_image": "`+arvadostest.DockerImage112PDH+`",
 757     "cwd": ".",
 758     "environment": {},
 759     "mounts": {"/tmp": {"kind": "tmp"} },
 760     "output_path": "/tmp",
 761     "priority": 1,
 762     "runtime_constraints": {},
 763     "scheduling_parameters":{"max_run_time": 1},
 764     "state": "Locked"
 765 }`, nil, 0, func() {
 766                 time.Sleep(3 * time.Second)
 767         })
 768
 769         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 770         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 771 }
 772
 773 func (s *TestSuite) TestContainerWaitFails(c *C) {
 774         s.fullRunHelper(c, `{
 775     "command": ["sleep", "3"],
 776     "container_image": "`+arvadostest.DockerImage112PDH+`",
 777     "cwd": ".",
 778     "mounts": {"/tmp": {"kind": "tmp"} },
 779     "output_path": "/tmp",
 780     "priority": 1,
 781     "state": "Locked"
 782 }`, nil, 0, func() {
 783                 s.executor.waitErr = errors.New("Container is not running")
 784         })
 785
 786         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 787         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 788 }
 789
 790 func (s *TestSuite) TestCrunchstat(c *C) {
 791         s.fullRunHelper(c, `{
 792                 "command": ["sleep", "1"],
 793                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 794                 "cwd": ".",
 795                 "environment": {},
 796                 "mounts": {"/tmp": {"kind": "tmp"} },
 797                 "output_path": "/tmp",
 798                 "priority": 1,
 799                 "runtime_constraints": {},
 800                 "state": "Locked"
 801         }`, nil, 0, func() {
 802                 time.Sleep(time.Second)
 803         })
 804
 805         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 806         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 807
 808         // We didn't actually start a container, so crunchstat didn't
 809         // find accounting files and therefore didn't log any stats.
 810         // It should have logged a "can't find accounting files"
 811         // message after one poll interval, though, so we can confirm
 812         // it's alive:
 813         c.Assert(s.api.Logs["crunchstat"], NotNil)
 814         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 815
 816         // The "files never appeared" log assures us that we called
 817         // (*crunchstat.Reporter)Stop(), and that we set it up with
 818         // the correct container ID "abcde":
 819         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 820 }
 821
 822 func (s *TestSuite) TestNodeInfoLog(c *C) {
 823         os.Setenv("SLURMD_NODENAME", "compute2")
 824         s.fullRunHelper(c, `{
 825                 "command": ["sleep", "1"],
 826                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 827                 "cwd": ".",
 828                 "environment": {},
 829                 "mounts": {"/tmp": {"kind": "tmp"} },
 830                 "output_path": "/tmp",
 831                 "priority": 1,
 832                 "runtime_constraints": {},
 833                 "state": "Locked"
 834         }`, nil, 0,
 835                 func() {
 836                         time.Sleep(time.Second)
 837                 })
 838
 839         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 840         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 841
 842         c.Assert(s.api.Logs["node"], NotNil)
 843         json := s.api.Logs["node"].String()
 844         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 845         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 846         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 847
 848         c.Assert(s.api.Logs["node-info"], NotNil)
 849         json = s.api.Logs["node-info"].String()
 850         c.Check(json, Matches, `(?ms).*Host Information.*`)
 851         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 852         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 853         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 854         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 855 }
 856
 857 func (s *TestSuite) TestContainerRecordLog(c *C) {
 858         s.fullRunHelper(c, `{
 859                 "command": ["sleep", "1"],
 860                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 861                 "cwd": ".",
 862                 "environment": {},
 863                 "mounts": {"/tmp": {"kind": "tmp"} },
 864                 "output_path": "/tmp",
 865                 "priority": 1,
 866                 "runtime_constraints": {},
 867                 "state": "Locked"
 868         }`, nil, 0,
 869                 func() {
 870                         time.Sleep(time.Second)
 871                 })
 872
 873         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 874         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 875
 876         c.Assert(s.api.Logs["container"], NotNil)
 877         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 878 }
 879
 880 func (s *TestSuite) TestFullRunStderr(c *C) {
 881         s.fullRunHelper(c, `{
 882     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 883     "container_image": "`+arvadostest.DockerImage112PDH+`",
 884     "cwd": ".",
 885     "environment": {},
 886     "mounts": {"/tmp": {"kind": "tmp"} },
 887     "output_path": "/tmp",
 888     "priority": 1,
 889     "runtime_constraints": {},
 890     "state": "Locked"
 891 }`, nil, 1, func() {
 892                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 893                 fmt.Fprintln(s.executor.created.Stderr, "world")
 894         })
 895
 896         final := s.api.CalledWith("container.state", "Complete")
 897         c.Assert(final, NotNil)
 898         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 899         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 900
 901         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 902         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 903 }
 904
 905 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 906         s.fullRunHelper(c, `{
 907     "command": ["pwd"],
 908     "container_image": "`+arvadostest.DockerImage112PDH+`",
 909     "cwd": ".",
 910     "environment": {},
 911     "mounts": {"/tmp": {"kind": "tmp"} },
 912     "output_path": "/tmp",
 913     "priority": 1,
 914     "runtime_constraints": {},
 915     "state": "Locked"
 916 }`, nil, 0, func() {
 917                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 918         })
 919
 920         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 921         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 922         c.Log(s.api.Logs["stdout"])
 923         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 924 }
 925
 926 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 927         s.fullRunHelper(c, `{
 928     "command": ["pwd"],
 929     "container_image": "`+arvadostest.DockerImage112PDH+`",
 930     "cwd": "/bin",
 931     "environment": {},
 932     "mounts": {"/tmp": {"kind": "tmp"} },
 933     "output_path": "/tmp",
 934     "priority": 1,
 935     "runtime_constraints": {},
 936     "state": "Locked"
 937 }`, nil, 0, func() {
 938                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 939         })
 940
 941         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 942         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 943         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 944 }
 945
 946 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
 947         s.fullRunHelper(c, `{
 948     "command": ["pwd"],
 949     "container_image": "`+arvadostest.DockerImage112PDH+`",
 950     "cwd": "/bin",
 951     "environment": {},
 952     "mounts": {"/tmp": {"kind": "tmp"} },
 953     "output_path": "/tmp",
 954     "priority": 1,
 955     "runtime_constraints": {},
 956     "state": "Locked",
 957     "output_storage_classes": ["foo", "bar"]
 958 }`, nil, 0, func() {
 959                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 960         })
 961
 962         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 963         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 964         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 965         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 966         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 967 }
 968
 969 func (s *TestSuite) TestStopOnSignal(c *C) {
 970         s.executor.runFunc = func() {
 971                 s.executor.created.Stdout.Write([]byte("foo\n"))
 972                 s.runner.SigChan <- syscall.SIGINT
 973         }
 974         s.testStopContainer(c)
 975 }
 976
 977 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
 978         s.executor.runFunc = func() {
 979                 s.executor.created.Stdout.Write([]byte("foo\n"))
 980                 s.runner.ArvMountExit <- nil
 981                 close(s.runner.ArvMountExit)
 982         }
 983         s.runner.ArvMountExit = make(chan error)
 984         s.testStopContainer(c)
 985 }
 986
 987 func (s *TestSuite) testStopContainer(c *C) {
 988         record := `{
 989     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
 990     "container_image": "` + arvadostest.DockerImage112PDH + `",
 991     "cwd": ".",
 992     "environment": {},
 993     "mounts": {"/tmp": {"kind": "tmp"} },
 994     "output_path": "/tmp",
 995     "priority": 1,
 996     "runtime_constraints": {},
 997     "state": "Locked"
 998 }`
 999
1000         err := json.Unmarshal([]byte(record), &s.api.Container)
1001         c.Assert(err, IsNil)
1002
1003         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1004         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1005                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1006         }
1007
1008         done := make(chan error)
1009         go func() {
1010                 done <- s.runner.Run()
1011         }()
1012         select {
1013         case <-time.After(20 * time.Second):
1014                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1015                 c.Fatal("timed out")
1016         case err = <-done:
1017                 c.Check(err, IsNil)
1018         }
1019         for k, v := range s.api.Logs {
1020                 c.Log(k)
1021                 c.Log(v.String(), "\n")
1022         }
1023
1024         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1025         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1026         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1027 }
1028
1029 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1030         s.fullRunHelper(c, `{
1031     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1032     "container_image": "`+arvadostest.DockerImage112PDH+`",
1033     "cwd": "/bin",
1034     "environment": {"FROBIZ": "bilbo"},
1035     "mounts": {"/tmp": {"kind": "tmp"} },
1036     "output_path": "/tmp",
1037     "priority": 1,
1038     "runtime_constraints": {},
1039     "state": "Locked"
1040 }`, nil, 0, func() {
1041                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1042         })
1043
1044         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1045         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1046         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1047 }
1048
1049 type ArvMountCmdLine struct {
1050         Cmd   []string
1051         token string
1052 }
1053
1054 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1055         am.Cmd = c
1056         am.token = token
1057         return nil, nil
1058 }
1059
1060 func stubCert(temp string) string {
1061         path := temp + "/ca-certificates.crt"
1062         crt, _ := os.Create(path)
1063         crt.Close()
1064         arvadosclient.CertFiles = []string{path}
1065         return path
1066 }
1067
1068 func (s *TestSuite) TestSetupMounts(c *C) {
1069         cr := s.runner
1070         am := &ArvMountCmdLine{}
1071         cr.RunArvMount = am.ArvMountTest
1072         cr.ContainerArvClient = &ArvTestClient{}
1073         cr.ContainerKeepClient = &KeepTestClient{}
1074         cr.Container.OutputStorageClasses = []string{"default"}
1075
1076         realTemp := c.MkDir()
1077         certTemp := c.MkDir()
1078         stubCertPath := stubCert(certTemp)
1079         cr.parentTemp = realTemp
1080
1081         i := 0
1082         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1083                 i++
1084                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1085                 err := os.Mkdir(d, os.ModePerm)
1086                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1087                         // Test case must have pre-populated the tempdir
1088                         err = nil
1089                 }
1090                 return d, err
1091         }
1092
1093         checkEmpty := func() {
1094                 // Should be deleted.
1095                 _, err := os.Stat(realTemp)
1096                 c.Assert(os.IsNotExist(err), Equals, true)
1097
1098                 // Now recreate it for the next test.
1099                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1100         }
1101
1102         {
1103                 i = 0
1104                 cr.ArvMountPoint = ""
1105                 cr.Container.Mounts = make(map[string]arvados.Mount)
1106                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1107                 cr.Container.OutputPath = "/tmp"
1108                 cr.statInterval = 5 * time.Second
1109                 bindmounts, err := cr.SetupMounts()
1110                 c.Check(err, IsNil)
1111                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1112                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1113                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1114                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1115                 os.RemoveAll(cr.ArvMountPoint)
1116                 cr.CleanupDirs()
1117                 checkEmpty()
1118         }
1119
1120         {
1121                 i = 0
1122                 cr.ArvMountPoint = ""
1123                 cr.Container.Mounts = make(map[string]arvados.Mount)
1124                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1125                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1126                 cr.Container.OutputPath = "/out"
1127                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1128
1129                 bindmounts, err := cr.SetupMounts()
1130                 c.Check(err, IsNil)
1131                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1132                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1133                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1134                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1135                 os.RemoveAll(cr.ArvMountPoint)
1136                 cr.CleanupDirs()
1137                 checkEmpty()
1138         }
1139
1140         {
1141                 i = 0
1142                 cr.ArvMountPoint = ""
1143                 cr.Container.Mounts = make(map[string]arvados.Mount)
1144                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1145                 cr.Container.OutputPath = "/tmp"
1146                 cr.Container.RuntimeConstraints.API = true
1147                 cr.Container.OutputStorageClasses = []string{"default"}
1148
1149                 bindmounts, err := cr.SetupMounts()
1150                 c.Check(err, IsNil)
1151                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1152                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1153                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1154                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1155                 os.RemoveAll(cr.ArvMountPoint)
1156                 cr.CleanupDirs()
1157                 checkEmpty()
1158
1159                 cr.Container.RuntimeConstraints.API = false
1160         }
1161
1162         {
1163                 i = 0
1164                 cr.ArvMountPoint = ""
1165                 cr.Container.Mounts = map[string]arvados.Mount{
1166                         "/keeptmp": {Kind: "collection", Writable: true},
1167                 }
1168                 cr.Container.OutputPath = "/keeptmp"
1169
1170                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1171
1172                 bindmounts, err := cr.SetupMounts()
1173                 c.Check(err, IsNil)
1174                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1175                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1176                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1177                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1178                 os.RemoveAll(cr.ArvMountPoint)
1179                 cr.CleanupDirs()
1180                 checkEmpty()
1181         }
1182
1183         {
1184                 i = 0
1185                 cr.ArvMountPoint = ""
1186                 cr.Container.Mounts = map[string]arvados.Mount{
1187                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1188                         "/keepout": {Kind: "collection", Writable: true},
1189                 }
1190                 cr.Container.OutputPath = "/keepout"
1191
1192                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1193                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1194
1195                 bindmounts, err := cr.SetupMounts()
1196                 c.Check(err, IsNil)
1197                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1198                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1199                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1200                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1201                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1202                         "/keepout": {realTemp + "/keep1/tmp0", false},
1203                 })
1204                 os.RemoveAll(cr.ArvMountPoint)
1205                 cr.CleanupDirs()
1206                 checkEmpty()
1207         }
1208
1209         {
1210                 i = 0
1211                 cr.ArvMountPoint = ""
1212                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1213                 cr.Container.Mounts = map[string]arvados.Mount{
1214                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1215                         "/keepout": {Kind: "collection", Writable: true},
1216                 }
1217                 cr.Container.OutputPath = "/keepout"
1218
1219                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1220                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1221
1222                 bindmounts, err := cr.SetupMounts()
1223                 c.Check(err, IsNil)
1224                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1225                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1226                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1227                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1228                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1229                         "/keepout": {realTemp + "/keep1/tmp0", false},
1230                 })
1231                 os.RemoveAll(cr.ArvMountPoint)
1232                 cr.CleanupDirs()
1233                 checkEmpty()
1234         }
1235
1236         for _, test := range []struct {
1237                 in  interface{}
1238                 out string
1239         }{
1240                 {in: "foo", out: `"foo"`},
1241                 {in: nil, out: `null`},
1242                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1243         } {
1244                 i = 0
1245                 cr.ArvMountPoint = ""
1246                 cr.Container.Mounts = map[string]arvados.Mount{
1247                         "/mnt/test.json": {Kind: "json", Content: test.in},
1248                 }
1249                 bindmounts, err := cr.SetupMounts()
1250                 c.Check(err, IsNil)
1251                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1252                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1253                 })
1254                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1255                 c.Check(err, IsNil)
1256                 c.Check(content, DeepEquals, []byte(test.out))
1257                 os.RemoveAll(cr.ArvMountPoint)
1258                 cr.CleanupDirs()
1259                 checkEmpty()
1260         }
1261
1262         for _, test := range []struct {
1263                 in  interface{}
1264                 out string
1265         }{
1266                 {in: "foo", out: `foo`},
1267                 {in: nil, out: "error"},
1268                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1269         } {
1270                 i = 0
1271                 cr.ArvMountPoint = ""
1272                 cr.Container.Mounts = map[string]arvados.Mount{
1273                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1274                 }
1275                 bindmounts, err := cr.SetupMounts()
1276                 if test.out == "error" {
1277                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1278                 } else {
1279                         c.Check(err, IsNil)
1280                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1281                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1282                         })
1283                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1284                         c.Check(err, IsNil)
1285                         c.Check(content, DeepEquals, []byte(test.out))
1286                 }
1287                 os.RemoveAll(cr.ArvMountPoint)
1288                 cr.CleanupDirs()
1289                 checkEmpty()
1290         }
1291
1292         // Read-only mount points are allowed underneath output_dir mount point
1293         {
1294                 i = 0
1295                 cr.ArvMountPoint = ""
1296                 cr.Container.Mounts = make(map[string]arvados.Mount)
1297                 cr.Container.Mounts = map[string]arvados.Mount{
1298                         "/tmp":     {Kind: "tmp"},
1299                         "/tmp/foo": {Kind: "collection"},
1300                 }
1301                 cr.Container.OutputPath = "/tmp"
1302
1303                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1304
1305                 bindmounts, err := cr.SetupMounts()
1306                 c.Check(err, IsNil)
1307                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1308                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1309                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1310                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1311                         "/tmp":     {realTemp + "/tmp2", false},
1312                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1313                 })
1314                 os.RemoveAll(cr.ArvMountPoint)
1315                 cr.CleanupDirs()
1316                 checkEmpty()
1317         }
1318
1319         // Writable mount points copied to output_dir mount point
1320         {
1321                 i = 0
1322                 cr.ArvMountPoint = ""
1323                 cr.Container.Mounts = make(map[string]arvados.Mount)
1324                 cr.Container.Mounts = map[string]arvados.Mount{
1325                         "/tmp": {Kind: "tmp"},
1326                         "/tmp/foo": {Kind: "collection",
1327                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1328                                 Writable:         true},
1329                         "/tmp/bar": {Kind: "collection",
1330                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1331                                 Path:             "baz",
1332                                 Writable:         true},
1333                 }
1334                 cr.Container.OutputPath = "/tmp"
1335
1336                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1337                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1338
1339                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1340                 rf.Write([]byte("bar"))
1341                 rf.Close()
1342
1343                 _, err := cr.SetupMounts()
1344                 c.Check(err, IsNil)
1345                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1346                 c.Check(err, IsNil)
1347                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1348                 c.Check(err, IsNil)
1349                 os.RemoveAll(cr.ArvMountPoint)
1350                 cr.CleanupDirs()
1351                 checkEmpty()
1352         }
1353
1354         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1355         {
1356                 i = 0
1357                 cr.ArvMountPoint = ""
1358                 cr.Container.Mounts = make(map[string]arvados.Mount)
1359                 cr.Container.Mounts = map[string]arvados.Mount{
1360                         "/tmp":     {Kind: "tmp"},
1361                         "/tmp/foo": {Kind: "tmp"},
1362                 }
1363                 cr.Container.OutputPath = "/tmp"
1364
1365                 _, err := cr.SetupMounts()
1366                 c.Check(err, NotNil)
1367                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1368                 os.RemoveAll(cr.ArvMountPoint)
1369                 cr.CleanupDirs()
1370                 checkEmpty()
1371         }
1372
1373         // Only mount point of kind 'collection' is allowed for stdin
1374         {
1375                 i = 0
1376                 cr.ArvMountPoint = ""
1377                 cr.Container.Mounts = make(map[string]arvados.Mount)
1378                 cr.Container.Mounts = map[string]arvados.Mount{
1379                         "stdin": {Kind: "tmp"},
1380                 }
1381
1382                 _, err := cr.SetupMounts()
1383                 c.Check(err, NotNil)
1384                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1385                 os.RemoveAll(cr.ArvMountPoint)
1386                 cr.CleanupDirs()
1387                 checkEmpty()
1388         }
1389
1390         // git_tree mounts
1391         {
1392                 i = 0
1393                 cr.ArvMountPoint = ""
1394                 (*GitMountSuite)(nil).useTestGitServer(c)
1395                 cr.token = arvadostest.ActiveToken
1396                 cr.Container.Mounts = make(map[string]arvados.Mount)
1397                 cr.Container.Mounts = map[string]arvados.Mount{
1398                         "/tip": {
1399                                 Kind:   "git_tree",
1400                                 UUID:   arvadostest.Repository2UUID,
1401                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1402                                 Path:   "/",
1403                         },
1404                         "/non-tip": {
1405                                 Kind:   "git_tree",
1406                                 UUID:   arvadostest.Repository2UUID,
1407                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1408                                 Path:   "/",
1409                         },
1410                 }
1411                 cr.Container.OutputPath = "/tmp"
1412
1413                 bindmounts, err := cr.SetupMounts()
1414                 c.Check(err, IsNil)
1415
1416                 for path, mount := range bindmounts {
1417                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1418                 }
1419
1420                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1421                 c.Check(err, IsNil)
1422                 c.Check(string(data), Equals, "\000\001\002\003")
1423                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1424                 c.Check(err, FitsTypeOf, &os.PathError{})
1425                 c.Check(os.IsNotExist(err), Equals, true)
1426
1427                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1428                 c.Check(err, IsNil)
1429                 c.Check(string(data), Equals, "\000\001\002\003")
1430                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1431                 c.Check(err, IsNil)
1432                 c.Check(string(data), Equals, "testfile\n")
1433
1434                 cr.CleanupDirs()
1435                 checkEmpty()
1436         }
1437 }
1438
1439 func (s *TestSuite) TestStdout(c *C) {
1440         helperRecord := `{
1441                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1442                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1443                 "cwd": "/bin",
1444                 "environment": {"FROBIZ": "bilbo"},
1445                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1446                 "output_path": "/tmp",
1447                 "priority": 1,
1448                 "runtime_constraints": {},
1449                 "state": "Locked"
1450         }`
1451
1452         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1453                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1454         })
1455
1456         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1457         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1458         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1459 }
1460
1461 // Used by the TestStdoutWithWrongPath*()
1462 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1463         err := json.Unmarshal([]byte(record), &s.api.Container)
1464         c.Assert(err, IsNil)
1465         s.executor.runFunc = fn
1466         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1467         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1468                 return s.api, &KeepTestClient{}, nil, nil
1469         }
1470         return s.api, s.runner, s.runner.Run()
1471 }
1472
1473 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1474         _, _, err := s.stdoutErrorRunHelper(c, `{
1475     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1476     "output_path": "/tmp",
1477     "state": "Locked"
1478 }`, func() {})
1479         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1480 }
1481
1482 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1483         _, _, err := s.stdoutErrorRunHelper(c, `{
1484     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1485     "output_path": "/tmp",
1486     "state": "Locked"
1487 }`, func() {})
1488         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1489 }
1490
1491 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1492         _, _, err := s.stdoutErrorRunHelper(c, `{
1493     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1494     "output_path": "/tmp",
1495     "state": "Locked"
1496 }`, func() {})
1497         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1498 }
1499
1500 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1501         s.fullRunHelper(c, `{
1502     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1503     "container_image": "`+arvadostest.DockerImage112PDH+`",
1504     "cwd": "/bin",
1505     "environment": {},
1506     "mounts": {"/tmp": {"kind": "tmp"} },
1507     "output_path": "/tmp",
1508     "priority": 1,
1509     "runtime_constraints": {"API": true},
1510     "state": "Locked"
1511 }`, nil, 0, func() {
1512                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1513                 s.executor.exit <- 3
1514         })
1515         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1516         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1517 }
1518
1519 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1520         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1521         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1522         s.fullRunHelper(c, `{
1523     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1524     "container_image": "`+arvadostest.DockerImage112PDH+`",
1525     "cwd": "/bin",
1526     "environment": {},
1527     "mounts": {"/tmp": {"kind": "tmp"} },
1528     "output_path": "/tmp",
1529     "priority": 1,
1530     "runtime_constraints": {"API": true},
1531     "state": "Locked"
1532 }`, nil, 0, func() {
1533                 s.api.Container.Output = arvadostest.DockerImage112PDH
1534         })
1535
1536         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1537         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1538         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1539 }
1540
1541 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1542         helperRecord := `{
1543                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1544                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1545                 "cwd": "/bin",
1546                 "environment": {"FROBIZ": "bilbo"},
1547                 "mounts": {
1548         "/tmp": {"kind": "tmp"},
1549         "/tmp/foo": {"kind": "collection",
1550                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1551                      "exclude_from_output": true
1552         },
1553         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1554     },
1555                 "output_path": "/tmp",
1556                 "priority": 1,
1557                 "runtime_constraints": {},
1558                 "state": "Locked"
1559         }`
1560
1561         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1562
1563         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1564                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1565         })
1566
1567         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1568         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1569         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1570 }
1571
1572 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1573         helperRecord := `{
1574                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1575                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1576                 "cwd": "/bin",
1577                 "environment": {"FROBIZ": "bilbo"},
1578                 "mounts": {
1579         "/tmp": {"kind": "tmp"},
1580         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1581         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1582         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1583         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1584         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1585     },
1586                 "output_path": "/tmp",
1587                 "priority": 1,
1588                 "runtime_constraints": {},
1589                 "state": "Locked"
1590         }`
1591
1592         extraMounts := []string{
1593                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1594                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1595                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1596         }
1597
1598         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1599                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1600         })
1601
1602         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1603                 "/tmp":                   {realtemp + "/tmp1", false},
1604                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1605                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1606                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1607                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1608         })
1609
1610         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1611         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1612         for _, v := range api.Content {
1613                 if v["collection"] != nil {
1614                         c.Check(v["ensure_unique_name"], Equals, true)
1615                         collection := v["collection"].(arvadosclient.Dict)
1616                         if strings.Index(collection["name"].(string), "output") == 0 {
1617                                 manifest := collection["manifest_text"].(string)
1618
1619                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1620 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1621 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1622 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1623 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1624 `)
1625                         }
1626                 }
1627         }
1628 }
1629
1630 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1631         helperRecord := `{
1632                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1633                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1634                 "cwd": "/bin",
1635                 "environment": {"FROBIZ": "bilbo"},
1636                 "mounts": {
1637         "/tmp": {"kind": "tmp"},
1638         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1639         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1640     },
1641                 "output_path": "/tmp",
1642                 "priority": 1,
1643                 "runtime_constraints": {},
1644                 "state": "Locked"
1645         }`
1646
1647         extraMounts := []string{
1648                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1649         }
1650
1651         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1652                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1653         })
1654
1655         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1656         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1657         for _, v := range s.api.Content {
1658                 if v["collection"] != nil {
1659                         collection := v["collection"].(arvadosclient.Dict)
1660                         if strings.Index(collection["name"].(string), "output") == 0 {
1661                                 manifest := collection["manifest_text"].(string)
1662
1663                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1664 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1665 `)
1666                         }
1667                 }
1668         }
1669 }
1670
1671 func (s *TestSuite) TestOutputError(c *C) {
1672         helperRecord := `{
1673                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1674                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1675                 "cwd": "/bin",
1676                 "environment": {"FROBIZ": "bilbo"},
1677                 "mounts": {
1678                         "/tmp": {"kind": "tmp"}
1679                 },
1680                 "output_path": "/tmp",
1681                 "priority": 1,
1682                 "runtime_constraints": {},
1683                 "state": "Locked"
1684         }`
1685         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1686                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1687         })
1688
1689         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1690 }
1691
1692 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1693         helperRecord := `{
1694                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1695                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1696                 "cwd": "/bin",
1697                 "environment": {"FROBIZ": "bilbo"},
1698                 "mounts": {
1699         "/tmp": {"kind": "tmp"},
1700         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1701         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1702     },
1703                 "output_path": "/tmp",
1704                 "priority": 1,
1705                 "runtime_constraints": {},
1706                 "state": "Locked"
1707         }`
1708
1709         extraMounts := []string{
1710                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1711         }
1712
1713         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1714                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1715         })
1716
1717         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1718         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1719         for _, v := range api.Content {
1720                 if v["collection"] != nil {
1721                         collection := v["collection"].(arvadosclient.Dict)
1722                         if strings.Index(collection["name"].(string), "output") == 0 {
1723                                 manifest := collection["manifest_text"].(string)
1724                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1725 `)
1726                         }
1727                 }
1728         }
1729 }
1730
1731 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1732         helperRecord := `{
1733                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1734                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1735                 "cwd": "/bin",
1736                 "environment": {"FROBIZ": "bilbo"},
1737                 "mounts": {
1738         "/tmp": {"kind": "tmp"},
1739         "stdin": {"kind": "json", "content": "foo"},
1740         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1741     },
1742                 "output_path": "/tmp",
1743                 "priority": 1,
1744                 "runtime_constraints": {},
1745                 "state": "Locked"
1746         }`
1747
1748         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1749                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1750         })
1751
1752         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1753         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1754         for _, v := range api.Content {
1755                 if v["collection"] != nil {
1756                         collection := v["collection"].(arvadosclient.Dict)
1757                         if strings.Index(collection["name"].(string), "output") == 0 {
1758                                 manifest := collection["manifest_text"].(string)
1759                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1760 `)
1761                         }
1762                 }
1763         }
1764 }
1765
1766 func (s *TestSuite) TestStderrMount(c *C) {
1767         api, cr, _ := s.fullRunHelper(c, `{
1768     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1769     "container_image": "`+arvadostest.DockerImage112PDH+`",
1770     "cwd": ".",
1771     "environment": {},
1772     "mounts": {"/tmp": {"kind": "tmp"},
1773                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1774                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1775     "output_path": "/tmp",
1776     "priority": 1,
1777     "runtime_constraints": {},
1778     "state": "Locked"
1779 }`, nil, 1, func() {
1780                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1781                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1782         })
1783
1784         final := api.CalledWith("container.state", "Complete")
1785         c.Assert(final, NotNil)
1786         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1787         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1788
1789         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1790 }
1791
1792 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1793         s.api.callraw = true
1794         err := s.runner.fetchContainerRecord()
1795         c.Assert(err, IsNil)
1796         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1797         c.Logf("%#v", s.runner.Container)
1798         c.Check(err, IsNil)
1799         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1800 }
1801
1802 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1803         nextState := ""
1804         for _, setup := range []func(){
1805                 func() {
1806                         c.Log("// waitErr = ocl runtime error")
1807                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1808                         nextState = "Cancelled"
1809                 },
1810                 func() {
1811                         c.Log("// loadErr = cannot connect")
1812                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1813                         *brokenNodeHook = c.MkDir() + "/broken-node-hook"
1814                         err := ioutil.WriteFile(*brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1815                         c.Assert(err, IsNil)
1816                         nextState = "Queued"
1817                 },
1818         } {
1819                 s.SetUpTest(c)
1820                 setup()
1821                 s.fullRunHelper(c, `{
1822     "command": ["echo", "hello world"],
1823     "container_image": "`+arvadostest.DockerImage112PDH+`",
1824     "cwd": ".",
1825     "environment": {},
1826     "mounts": {"/tmp": {"kind": "tmp"} },
1827     "output_path": "/tmp",
1828     "priority": 1,
1829     "runtime_constraints": {},
1830     "state": "Locked"
1831 }`, nil, 0, func() {})
1832                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1833                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1834                 if *brokenNodeHook != "" {
1835                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1836                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1837                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1838                 } else {
1839                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1840                 }
1841         }
1842 }
1843
1844 func (s *TestSuite) TestBadCommand(c *C) {
1845         for _, startError := range []string{
1846                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1847                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1848                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1849         } {
1850                 s.SetUpTest(c)
1851                 s.executor.startErr = errors.New(startError)
1852                 s.fullRunHelper(c, `{
1853     "command": ["echo", "hello world"],
1854     "container_image": "`+arvadostest.DockerImage112PDH+`",
1855     "cwd": ".",
1856     "environment": {},
1857     "mounts": {"/tmp": {"kind": "tmp"} },
1858     "output_path": "/tmp",
1859     "priority": 1,
1860     "runtime_constraints": {},
1861     "state": "Locked"
1862 }`, nil, 0, func() {})
1863                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1864                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1865         }
1866 }
1867
1868 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1869         helperRecord := `{
1870                 "command": ["true"],
1871                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1872                 "cwd": "/bin",
1873                 "mounts": {
1874                     "/tmp": {"kind": "tmp"},
1875                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1876                 },
1877                 "secret_mounts": {
1878                 },
1879                 "output_path": "/tmp",
1880                 "priority": 1,
1881                 "runtime_constraints": {},
1882                 "state": "Locked"
1883         }`
1884
1885         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1886                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1887                 c.Check(err, IsNil)
1888                 c.Check(string(content), Equals, "mypassword")
1889         })
1890
1891         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1892         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1893         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
1894         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
1895
1896         // under secret mounts, not captured in output
1897         helperRecord = `{
1898                 "command": ["true"],
1899                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1900                 "cwd": "/bin",
1901                 "mounts": {
1902                     "/tmp": {"kind": "tmp"}
1903                 },
1904                 "secret_mounts": {
1905                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1906                 },
1907                 "output_path": "/tmp",
1908                 "priority": 1,
1909                 "runtime_constraints": {},
1910                 "state": "Locked"
1911         }`
1912
1913         s.SetUpTest(c)
1914         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1915                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1916                 c.Check(err, IsNil)
1917                 c.Check(string(content), Equals, "mypassword")
1918         })
1919
1920         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1921         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1922         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
1923         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
1924 }
1925
1926 type FakeProcess struct {
1927         cmdLine []string
1928 }
1929
1930 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
1931         return fp.cmdLine, nil
1932 }