lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/manifest"
  29         "golang.org/x/net/context"
  30
  31         . "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func TestCrunchExec(t *testing.T) {
  36         TestingT(t)
  37 }
  38
  39 var _ = Suite(&TestSuite{})
  40
  41 type TestSuite struct {
  42         client                   *arvados.Client
  43         api                      *ArvTestClient
  44         runner                   *ContainerRunner
  45         executor                 *stubExecutor
  46         keepmount                string
  47         testDispatcherKeepClient KeepTestClient
  48         testContainerKeepClient  KeepTestClient
  49 }
  50
  51 func (s *TestSuite) SetUpTest(c *C) {
  52         *brokenNodeHook = ""
  53         s.client = arvados.NewClientFromEnv()
  54         s.executor = &stubExecutor{}
  55         var err error
  56         s.api = &ArvTestClient{}
  57         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  58         c.Assert(err, IsNil)
  59         s.runner.executor = s.executor
  60         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  61                 return s.api, &s.testContainerKeepClient, s.client, nil
  62         }
  63         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  64                 s.runner.ArvMountPoint = s.keepmount
  65                 return nil, nil
  66         }
  67         s.keepmount = c.MkDir()
  68         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  69         c.Assert(err, IsNil)
  70         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  71         c.Assert(err, IsNil)
  72         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  73         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  74         c.Assert(err, IsNil)
  75         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  76         c.Assert(err, IsNil)
  77         s.runner.ArvMountPoint = s.keepmount
  78 }
  79
  80 type ArvTestClient struct {
  81         Total   int64
  82         Calls   int
  83         Content []arvadosclient.Dict
  84         arvados.Container
  85         secretMounts []byte
  86         Logs         map[string]*bytes.Buffer
  87         sync.Mutex
  88         WasSetRunning bool
  89         callraw       bool
  90 }
  91
  92 type KeepTestClient struct {
  93         Called         bool
  94         Content        []byte
  95         StorageClasses []string
  96 }
  97
  98 type stubExecutor struct {
  99         imageLoaded bool
 100         loaded      string
 101         loadErr     error
 102         exitCode    int
 103         createErr   error
 104         created     containerSpec
 105         startErr    error
 106         waitSleep   time.Duration
 107         waitErr     error
 108         stopErr     error
 109         stopped     bool
 110         closed      bool
 111         runFunc     func()
 112         exit        chan int
 113 }
 114
 115 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 116         containerClient *arvados.Client) error {
 117         e.loaded = tarball
 118         return e.loadErr
 119 }
 120 func (e *stubExecutor) Runtime() string                 { return "stub" }
 121 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 122 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 123 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 124 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 125 func (e *stubExecutor) Close()                          { e.closed = true }
 126 func (e *stubExecutor) Wait(context.Context) (int, error) {
 127         return <-e.exit, e.waitErr
 128 }
 129
 130 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 131
 132 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 133 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 134 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 135
 136 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 137 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 138
 139 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 140 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 141 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 142 `
 143
 144 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 145
 146 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 147 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 148
 149 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 150 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 151
 152 func (client *ArvTestClient) Create(resourceType string,
 153         parameters arvadosclient.Dict,
 154         output interface{}) error {
 155
 156         client.Mutex.Lock()
 157         defer client.Mutex.Unlock()
 158
 159         client.Calls++
 160         client.Content = append(client.Content, parameters)
 161
 162         if resourceType == "logs" {
 163                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 164                 if client.Logs == nil {
 165                         client.Logs = make(map[string]*bytes.Buffer)
 166                 }
 167                 if client.Logs[et] == nil {
 168                         client.Logs[et] = &bytes.Buffer{}
 169                 }
 170                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 171         }
 172
 173         if resourceType == "collections" && output != nil {
 174                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 175                 outmap := output.(*arvados.Collection)
 176                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 177                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 178         }
 179
 180         return nil
 181 }
 182
 183 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 184         switch {
 185         case method == "GET" && resourceType == "containers" && action == "auth":
 186                 return json.Unmarshal([]byte(`{
 187                         "kind": "arvados#api_client_authorization",
 188                         "uuid": "`+fakeAuthUUID+`",
 189                         "api_token": "`+fakeAuthToken+`"
 190                         }`), output)
 191         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 192                 if client.secretMounts != nil {
 193                         return json.Unmarshal(client.secretMounts, output)
 194                 }
 195                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 196         default:
 197                 return fmt.Errorf("Not found")
 198         }
 199 }
 200
 201 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 202         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 203         var j []byte
 204         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 205                 j = []byte(`{
 206                         "kind": "arvados#nodeList",
 207                         "items": [{
 208                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 209                                 "hostname": "compute2",
 210                                 "properties": {"total_cpu_cores": 16}
 211                         }]}`)
 212         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 213                 if uuid == "" {
 214                         j, err = json.Marshal(map[string]interface{}{
 215                                 "items": []interface{}{client.Container},
 216                                 "kind":  "arvados#nodeList",
 217                         })
 218                 } else {
 219                         j, err = json.Marshal(client.Container)
 220                 }
 221         } else {
 222                 j = []byte(`{
 223                         "command": ["sleep", "1"],
 224                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 225                         "cwd": ".",
 226                         "environment": {},
 227                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 228                         "output_path": "/tmp",
 229                         "priority": 1,
 230                         "runtime_constraints": {}
 231                 }`)
 232         }
 233         return ioutil.NopCloser(bytes.NewReader(j)), err
 234 }
 235
 236 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 237         if resourceType == "collections" {
 238                 if uuid == hwPDH {
 239                         output.(*arvados.Collection).ManifestText = hwManifest
 240                 } else if uuid == otherPDH {
 241                         output.(*arvados.Collection).ManifestText = otherManifest
 242                 } else if uuid == normalizedWithSubdirsPDH {
 243                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 244                 } else if uuid == denormalizedWithSubdirsPDH {
 245                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 246                 }
 247         }
 248         if resourceType == "containers" {
 249                 (*output.(*arvados.Container)) = client.Container
 250         }
 251         return nil
 252 }
 253
 254 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 255         client.Mutex.Lock()
 256         defer client.Mutex.Unlock()
 257         client.Calls++
 258         client.Content = append(client.Content, parameters)
 259         if resourceType == "containers" {
 260                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 261                         client.WasSetRunning = true
 262                 }
 263         } else if resourceType == "collections" {
 264                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 265                 output.(*arvados.Collection).UUID = uuid
 266                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 267         }
 268         return nil
 269 }
 270
 271 var discoveryMap = map[string]interface{}{
 272         "defaultTrashLifetime":               float64(1209600),
 273         "crunchLimitLogBytesPerJob":          float64(67108864),
 274         "crunchLogThrottleBytes":             float64(65536),
 275         "crunchLogThrottlePeriod":            float64(60),
 276         "crunchLogThrottleLines":             float64(1024),
 277         "crunchLogPartialLineThrottlePeriod": float64(5),
 278         "crunchLogBytesPerEvent":             float64(4096),
 279         "crunchLogSecondsBetweenEvents":      float64(1),
 280 }
 281
 282 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 283         return discoveryMap[key], nil
 284 }
 285
 286 // CalledWith returns the parameters from the first API call whose
 287 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 288 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 289 // no call matches, it returns nil.
 290 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 291 call:
 292         for _, content := range client.Content {
 293                 var v interface{} = content
 294                 for _, k := range strings.Split(jpath, ".") {
 295                         if dict, ok := v.(arvadosclient.Dict); !ok {
 296                                 continue call
 297                         } else {
 298                                 v = dict[k]
 299                         }
 300                 }
 301                 if v == expect {
 302                         return content
 303                 }
 304         }
 305         return nil
 306 }
 307
 308 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 309         return locator, nil
 310 }
 311
 312 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 313         client.Content = opts.Data
 314         return arvados.BlockWriteResponse{
 315                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 316         }, nil
 317 }
 318
 319 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 320         return 0, errors.New("not implemented")
 321 }
 322
 323 func (client *KeepTestClient) ClearBlockCache() {
 324 }
 325
 326 func (client *KeepTestClient) Close() {
 327         client.Content = nil
 328 }
 329
 330 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 331         client.StorageClasses = sc
 332 }
 333
 334 type FileWrapper struct {
 335         io.ReadCloser
 336         len int64
 337 }
 338
 339 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 340         return nil, errors.New("not implemented")
 341 }
 342
 343 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 344         return 0, errors.New("not implemented")
 345 }
 346
 347 func (fw FileWrapper) Size() int64 {
 348         return fw.len
 349 }
 350
 351 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 352         return nil, errors.New("not implemented")
 353 }
 354
 355 func (fw FileWrapper) Truncate(int64) error {
 356         return errors.New("not implemented")
 357 }
 358
 359 func (fw FileWrapper) Write([]byte) (int, error) {
 360         return 0, errors.New("not implemented")
 361 }
 362
 363 func (fw FileWrapper) Sync() error {
 364         return errors.New("not implemented")
 365 }
 366
 367 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 368         if filename == hwImageID+".tar" {
 369                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 370                 client.Called = true
 371                 return FileWrapper{rdr, 1321984}, nil
 372         } else if filename == "/file1_in_main.txt" {
 373                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 374                 client.Called = true
 375                 return FileWrapper{rdr, 3}, nil
 376         }
 377         return nil, nil
 378 }
 379
 380 func (s *TestSuite) TestLoadImage(c *C) {
 381         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 382         s.runner.Container.Mounts = map[string]arvados.Mount{
 383                 "/out": {Kind: "tmp", Writable: true},
 384         }
 385         s.runner.Container.OutputPath = "/out"
 386
 387         _, err := s.runner.SetupMounts()
 388         c.Assert(err, IsNil)
 389
 390         imageID, err := s.runner.LoadImage()
 391         c.Check(err, IsNil)
 392         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 393         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 394
 395         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 396         s.executor.imageLoaded = false
 397         s.executor.loaded = ""
 398         s.executor.loadErr = errors.New("bork")
 399         imageID, err = s.runner.LoadImage()
 400         c.Check(err, ErrorMatches, ".*bork")
 401         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 402
 403         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 404         s.executor.imageLoaded = false
 405         s.executor.loaded = ""
 406         s.executor.loadErr = nil
 407         imageID, err = s.runner.LoadImage()
 408         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 409         c.Check(s.executor.loaded, Equals, "")
 410 }
 411
 412 type ArvErrorTestClient struct{}
 413
 414 func (ArvErrorTestClient) Create(resourceType string,
 415         parameters arvadosclient.Dict,
 416         output interface{}) error {
 417         return nil
 418 }
 419
 420 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 421         if method == "GET" && resourceType == "containers" && action == "auth" {
 422                 return nil
 423         }
 424         return errors.New("ArvError")
 425 }
 426
 427 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 428         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 429         return nil, errors.New("ArvError")
 430 }
 431
 432 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 433         return errors.New("ArvError")
 434 }
 435
 436 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 437         return nil
 438 }
 439
 440 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 441         return discoveryMap[key], nil
 442 }
 443
 444 type KeepErrorTestClient struct {
 445         KeepTestClient
 446 }
 447
 448 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 449         return nil, errors.New("KeepError")
 450 }
 451
 452 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 453         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 454 }
 455
 456 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 457         return "", errors.New("KeepError")
 458 }
 459
 460 type KeepReadErrorTestClient struct {
 461         KeepTestClient
 462 }
 463
 464 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 465         return 0, errors.New("KeepError")
 466 }
 467
 468 type ErrorReader struct {
 469         FileWrapper
 470 }
 471
 472 func (ErrorReader) Read(p []byte) (n int, err error) {
 473         return 0, errors.New("ErrorReader")
 474 }
 475
 476 func (ErrorReader) Seek(int64, int) (int64, error) {
 477         return 0, errors.New("ErrorReader")
 478 }
 479
 480 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 481         return ErrorReader{}, nil
 482 }
 483
 484 type ClosableBuffer struct {
 485         bytes.Buffer
 486 }
 487
 488 func (*ClosableBuffer) Close() error {
 489         return nil
 490 }
 491
 492 type TestLogs struct {
 493         Stdout ClosableBuffer
 494         Stderr ClosableBuffer
 495 }
 496
 497 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 498         if logstr == "stdout" {
 499                 return &tl.Stdout, nil
 500         }
 501         if logstr == "stderr" {
 502                 return &tl.Stderr, nil
 503         }
 504         return nil, errors.New("???")
 505 }
 506
 507 func dockerLog(fd byte, msg string) []byte {
 508         by := []byte(msg)
 509         header := make([]byte, 8+len(by))
 510         header[0] = fd
 511         header[7] = byte(len(by))
 512         copy(header[8:], by)
 513         return header
 514 }
 515
 516 func (s *TestSuite) TestRunContainer(c *C) {
 517         s.executor.runFunc = func() {
 518                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 519                 s.executor.exit <- 0
 520         }
 521
 522         var logs TestLogs
 523         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 524         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 525         s.runner.Container.Command = []string{"./hw"}
 526         s.runner.Container.OutputStorageClasses = []string{"default"}
 527
 528         imageID, err := s.runner.LoadImage()
 529         c.Assert(err, IsNil)
 530
 531         err = s.runner.CreateContainer(imageID, nil)
 532         c.Assert(err, IsNil)
 533
 534         err = s.runner.StartContainer()
 535         c.Assert(err, IsNil)
 536
 537         err = s.runner.WaitFinish()
 538         c.Assert(err, IsNil)
 539
 540         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 541         c.Check(logs.Stderr.String(), Equals, "")
 542 }
 543
 544 func (s *TestSuite) TestCommitLogs(c *C) {
 545         api := &ArvTestClient{}
 546         kc := &KeepTestClient{}
 547         defer kc.Close()
 548         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 549         c.Assert(err, IsNil)
 550         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 551
 552         cr.CrunchLog.Print("Hello world!")
 553         cr.CrunchLog.Print("Goodbye")
 554         cr.finalState = "Complete"
 555
 556         err = cr.CommitLogs()
 557         c.Check(err, IsNil)
 558
 559         c.Check(api.Calls, Equals, 2)
 560         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 561         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 562         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 563         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 564 }
 565
 566 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 567         api := &ArvTestClient{}
 568         kc := &KeepTestClient{}
 569         defer kc.Close()
 570         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 571         c.Assert(err, IsNil)
 572
 573         err = cr.UpdateContainerRunning()
 574         c.Check(err, IsNil)
 575
 576         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 577 }
 578
 579 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 580         api := &ArvTestClient{}
 581         kc := &KeepTestClient{}
 582         defer kc.Close()
 583         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 584         c.Assert(err, IsNil)
 585
 586         cr.LogsPDH = new(string)
 587         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 588
 589         cr.ExitCode = new(int)
 590         *cr.ExitCode = 42
 591         cr.finalState = "Complete"
 592
 593         err = cr.UpdateContainerFinal()
 594         c.Check(err, IsNil)
 595
 596         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 597         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 598         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 599 }
 600
 601 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 602         api := &ArvTestClient{}
 603         kc := &KeepTestClient{}
 604         defer kc.Close()
 605         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 606         c.Assert(err, IsNil)
 607         cr.cCancelled = true
 608         cr.finalState = "Cancelled"
 609
 610         err = cr.UpdateContainerFinal()
 611         c.Check(err, IsNil)
 612
 613         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 614         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 615         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 616 }
 617
 618 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 619 // dress rehearsal of the Run() function, starting from a JSON container record.
 620 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 621         err := json.Unmarshal([]byte(record), &s.api.Container)
 622         c.Assert(err, IsNil)
 623         initialState := s.api.Container.State
 624
 625         var sm struct {
 626                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 627         }
 628         err = json.Unmarshal([]byte(record), &sm)
 629         c.Check(err, IsNil)
 630         secretMounts, err := json.Marshal(sm)
 631         c.Assert(err, IsNil)
 632         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 633
 634         s.executor.runFunc = func() {
 635                 fn()
 636                 s.executor.exit <- exitCode
 637         }
 638
 639         s.runner.statInterval = 100 * time.Millisecond
 640         s.runner.containerWatchdogInterval = time.Second
 641         am := &ArvMountCmdLine{}
 642         s.runner.RunArvMount = am.ArvMountTest
 643
 644         realTemp := c.MkDir()
 645         tempcount := 0
 646         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 647                 tempcount++
 648                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 649                 err := os.Mkdir(d, os.ModePerm)
 650                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 651                         // Test case must have pre-populated the tempdir
 652                         err = nil
 653                 }
 654                 return d, err
 655         }
 656         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 657                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 658         }
 659
 660         if extraMounts != nil && len(extraMounts) > 0 {
 661                 err := s.runner.SetupArvMountPoint("keep")
 662                 c.Check(err, IsNil)
 663
 664                 for _, m := range extraMounts {
 665                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 666                 }
 667         }
 668
 669         err = s.runner.Run()
 670         if s.api.CalledWith("container.state", "Complete") != nil {
 671                 c.Check(err, IsNil)
 672         }
 673         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 674                 c.Check(s.api.WasSetRunning, Equals, true)
 675                 var lastupdate arvadosclient.Dict
 676                 for _, content := range s.api.Content {
 677                         if content["container"] != nil {
 678                                 lastupdate = content["container"].(arvadosclient.Dict)
 679                         }
 680                 }
 681                 if lastupdate["log"] == nil {
 682                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 683                 }
 684         }
 685
 686         if err != nil {
 687                 for k, v := range s.api.Logs {
 688                         c.Log(k)
 689                         c.Log(v.String())
 690                 }
 691         }
 692
 693         return s.api, s.runner, realTemp
 694 }
 695
 696 func (s *TestSuite) TestFullRunHello(c *C) {
 697         s.runner.enableMemoryLimit = true
 698         s.runner.networkMode = "default"
 699         s.fullRunHelper(c, `{
 700     "command": ["echo", "hello world"],
 701     "container_image": "`+arvadostest.DockerImage112PDH+`",
 702     "cwd": ".",
 703     "environment": {"foo":"bar","baz":"waz"},
 704     "mounts": {"/tmp": {"kind": "tmp"} },
 705     "output_path": "/tmp",
 706     "priority": 1,
 707     "runtime_constraints": {"vcpus":1,"ram":1000000},
 708     "state": "Locked",
 709     "output_storage_classes": ["default"]
 710 }`, nil, 0, func() {
 711                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 712                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 713                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 714                 c.Check(s.executor.created.VCPUs, Equals, 1)
 715                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 716                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 717                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 718                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 719         })
 720
 721         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 722         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 723         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 724         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 725         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 726 }
 727
 728 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 729         var ran bool
 730         s.fullRunHelper(c, `{
 731     "command": ["sleep", "3"],
 732     "container_image": "`+arvadostest.DockerImage112PDH+`",
 733     "cwd": ".",
 734     "environment": {},
 735     "mounts": {"/tmp": {"kind": "tmp"} },
 736     "output_path": "/tmp",
 737     "priority": 1,
 738     "runtime_constraints": {},
 739     "scheduling_parameters":{"max_run_time": 1},
 740     "state": "Running"
 741 }`, nil, 2, func() {
 742                 ran = true
 743         })
 744         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 745         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 746         c.Check(ran, Equals, false)
 747 }
 748
 749 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 750         s.fullRunHelper(c, `{
 751     "command": ["sleep", "3"],
 752     "container_image": "`+arvadostest.DockerImage112PDH+`",
 753     "cwd": ".",
 754     "environment": {},
 755     "mounts": {"/tmp": {"kind": "tmp"} },
 756     "output_path": "/tmp",
 757     "priority": 1,
 758     "runtime_constraints": {},
 759     "scheduling_parameters":{"max_run_time": 1},
 760     "state": "Locked"
 761 }`, nil, 0, func() {
 762                 time.Sleep(3 * time.Second)
 763         })
 764
 765         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 766         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 767 }
 768
 769 func (s *TestSuite) TestContainerWaitFails(c *C) {
 770         s.fullRunHelper(c, `{
 771     "command": ["sleep", "3"],
 772     "container_image": "`+arvadostest.DockerImage112PDH+`",
 773     "cwd": ".",
 774     "mounts": {"/tmp": {"kind": "tmp"} },
 775     "output_path": "/tmp",
 776     "priority": 1,
 777     "state": "Locked"
 778 }`, nil, 0, func() {
 779                 s.executor.waitErr = errors.New("Container is not running")
 780         })
 781
 782         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 783         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 784 }
 785
 786 func (s *TestSuite) TestCrunchstat(c *C) {
 787         s.fullRunHelper(c, `{
 788                 "command": ["sleep", "1"],
 789                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 790                 "cwd": ".",
 791                 "environment": {},
 792                 "mounts": {"/tmp": {"kind": "tmp"} },
 793                 "output_path": "/tmp",
 794                 "priority": 1,
 795                 "runtime_constraints": {},
 796                 "state": "Locked"
 797         }`, nil, 0, func() {
 798                 time.Sleep(time.Second)
 799         })
 800
 801         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 802         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 803
 804         // We didn't actually start a container, so crunchstat didn't
 805         // find accounting files and therefore didn't log any stats.
 806         // It should have logged a "can't find accounting files"
 807         // message after one poll interval, though, so we can confirm
 808         // it's alive:
 809         c.Assert(s.api.Logs["crunchstat"], NotNil)
 810         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 811
 812         // The "files never appeared" log assures us that we called
 813         // (*crunchstat.Reporter)Stop(), and that we set it up with
 814         // the correct container ID "abcde":
 815         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 816 }
 817
 818 func (s *TestSuite) TestNodeInfoLog(c *C) {
 819         os.Setenv("SLURMD_NODENAME", "compute2")
 820         s.fullRunHelper(c, `{
 821                 "command": ["sleep", "1"],
 822                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 823                 "cwd": ".",
 824                 "environment": {},
 825                 "mounts": {"/tmp": {"kind": "tmp"} },
 826                 "output_path": "/tmp",
 827                 "priority": 1,
 828                 "runtime_constraints": {},
 829                 "state": "Locked"
 830         }`, nil, 0,
 831                 func() {
 832                         time.Sleep(time.Second)
 833                 })
 834
 835         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 836         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 837
 838         c.Assert(s.api.Logs["node"], NotNil)
 839         json := s.api.Logs["node"].String()
 840         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 841         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 842         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 843
 844         c.Assert(s.api.Logs["node-info"], NotNil)
 845         json = s.api.Logs["node-info"].String()
 846         c.Check(json, Matches, `(?ms).*Host Information.*`)
 847         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 848         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 849         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 850         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 851 }
 852
 853 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 854         s.fullRunHelper(c, `{
 855                 "command": ["sleep", "1"],
 856                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 857                 "cwd": ".",
 858                 "environment": {},
 859                 "mounts": {"/tmp": {"kind": "tmp"} },
 860                 "output_path": "/tmp",
 861                 "priority": 1,
 862                 "runtime_constraints": {},
 863                 "state": "Locked"
 864         }`, nil, 0,
 865                 func() {
 866                 })
 867
 868         c.Assert(s.api.Logs["crunch-run"], NotNil)
 869         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 870         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' using stub runtime.*`)
 871 }
 872
 873 func (s *TestSuite) TestContainerRecordLog(c *C) {
 874         s.fullRunHelper(c, `{
 875                 "command": ["sleep", "1"],
 876                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 877                 "cwd": ".",
 878                 "environment": {},
 879                 "mounts": {"/tmp": {"kind": "tmp"} },
 880                 "output_path": "/tmp",
 881                 "priority": 1,
 882                 "runtime_constraints": {},
 883                 "state": "Locked"
 884         }`, nil, 0,
 885                 func() {
 886                         time.Sleep(time.Second)
 887                 })
 888
 889         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 890         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 891
 892         c.Assert(s.api.Logs["container"], NotNil)
 893         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 894 }
 895
 896 func (s *TestSuite) TestFullRunStderr(c *C) {
 897         s.fullRunHelper(c, `{
 898     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 899     "container_image": "`+arvadostest.DockerImage112PDH+`",
 900     "cwd": ".",
 901     "environment": {},
 902     "mounts": {"/tmp": {"kind": "tmp"} },
 903     "output_path": "/tmp",
 904     "priority": 1,
 905     "runtime_constraints": {},
 906     "state": "Locked"
 907 }`, nil, 1, func() {
 908                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 909                 fmt.Fprintln(s.executor.created.Stderr, "world")
 910         })
 911
 912         final := s.api.CalledWith("container.state", "Complete")
 913         c.Assert(final, NotNil)
 914         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 915         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 916
 917         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 918         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 919 }
 920
 921 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 922         s.fullRunHelper(c, `{
 923     "command": ["pwd"],
 924     "container_image": "`+arvadostest.DockerImage112PDH+`",
 925     "cwd": ".",
 926     "environment": {},
 927     "mounts": {"/tmp": {"kind": "tmp"} },
 928     "output_path": "/tmp",
 929     "priority": 1,
 930     "runtime_constraints": {},
 931     "state": "Locked"
 932 }`, nil, 0, func() {
 933                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 934         })
 935
 936         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 937         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 938         c.Log(s.api.Logs["stdout"])
 939         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 940 }
 941
 942 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 943         s.fullRunHelper(c, `{
 944     "command": ["pwd"],
 945     "container_image": "`+arvadostest.DockerImage112PDH+`",
 946     "cwd": "/bin",
 947     "environment": {},
 948     "mounts": {"/tmp": {"kind": "tmp"} },
 949     "output_path": "/tmp",
 950     "priority": 1,
 951     "runtime_constraints": {},
 952     "state": "Locked"
 953 }`, nil, 0, func() {
 954                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 955         })
 956
 957         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 958         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 959         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 960 }
 961
 962 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
 963         s.fullRunHelper(c, `{
 964     "command": ["pwd"],
 965     "container_image": "`+arvadostest.DockerImage112PDH+`",
 966     "cwd": "/bin",
 967     "environment": {},
 968     "mounts": {"/tmp": {"kind": "tmp"} },
 969     "output_path": "/tmp",
 970     "priority": 1,
 971     "runtime_constraints": {},
 972     "state": "Locked",
 973     "output_storage_classes": ["foo", "bar"]
 974 }`, nil, 0, func() {
 975                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 976         })
 977
 978         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 979         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 980         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 981         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 982         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 983 }
 984
 985 func (s *TestSuite) TestStopOnSignal(c *C) {
 986         s.executor.runFunc = func() {
 987                 s.executor.created.Stdout.Write([]byte("foo\n"))
 988                 s.runner.SigChan <- syscall.SIGINT
 989         }
 990         s.testStopContainer(c)
 991 }
 992
 993 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
 994         s.executor.runFunc = func() {
 995                 s.executor.created.Stdout.Write([]byte("foo\n"))
 996                 s.runner.ArvMountExit <- nil
 997                 close(s.runner.ArvMountExit)
 998         }
 999         s.runner.ArvMountExit = make(chan error)
1000         s.testStopContainer(c)
1001 }
1002
1003 func (s *TestSuite) testStopContainer(c *C) {
1004         record := `{
1005     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1006     "container_image": "` + arvadostest.DockerImage112PDH + `",
1007     "cwd": ".",
1008     "environment": {},
1009     "mounts": {"/tmp": {"kind": "tmp"} },
1010     "output_path": "/tmp",
1011     "priority": 1,
1012     "runtime_constraints": {},
1013     "state": "Locked"
1014 }`
1015
1016         err := json.Unmarshal([]byte(record), &s.api.Container)
1017         c.Assert(err, IsNil)
1018
1019         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1020         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1021                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1022         }
1023
1024         done := make(chan error)
1025         go func() {
1026                 done <- s.runner.Run()
1027         }()
1028         select {
1029         case <-time.After(20 * time.Second):
1030                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1031                 c.Fatal("timed out")
1032         case err = <-done:
1033                 c.Check(err, IsNil)
1034         }
1035         for k, v := range s.api.Logs {
1036                 c.Log(k)
1037                 c.Log(v.String(), "\n")
1038         }
1039
1040         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1041         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1042         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1043 }
1044
1045 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1046         s.fullRunHelper(c, `{
1047     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1048     "container_image": "`+arvadostest.DockerImage112PDH+`",
1049     "cwd": "/bin",
1050     "environment": {"FROBIZ": "bilbo"},
1051     "mounts": {"/tmp": {"kind": "tmp"} },
1052     "output_path": "/tmp",
1053     "priority": 1,
1054     "runtime_constraints": {},
1055     "state": "Locked"
1056 }`, nil, 0, func() {
1057                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1058         })
1059
1060         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1061         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1062         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1063 }
1064
1065 type ArvMountCmdLine struct {
1066         Cmd   []string
1067         token string
1068 }
1069
1070 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1071         am.Cmd = c
1072         am.token = token
1073         return nil, nil
1074 }
1075
1076 func stubCert(temp string) string {
1077         path := temp + "/ca-certificates.crt"
1078         crt, _ := os.Create(path)
1079         crt.Close()
1080         arvadosclient.CertFiles = []string{path}
1081         return path
1082 }
1083
1084 func (s *TestSuite) TestSetupMounts(c *C) {
1085         cr := s.runner
1086         am := &ArvMountCmdLine{}
1087         cr.RunArvMount = am.ArvMountTest
1088         cr.ContainerArvClient = &ArvTestClient{}
1089         cr.ContainerKeepClient = &KeepTestClient{}
1090         cr.Container.OutputStorageClasses = []string{"default"}
1091
1092         realTemp := c.MkDir()
1093         certTemp := c.MkDir()
1094         stubCertPath := stubCert(certTemp)
1095         cr.parentTemp = realTemp
1096
1097         i := 0
1098         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1099                 i++
1100                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1101                 err := os.Mkdir(d, os.ModePerm)
1102                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1103                         // Test case must have pre-populated the tempdir
1104                         err = nil
1105                 }
1106                 return d, err
1107         }
1108
1109         checkEmpty := func() {
1110                 // Should be deleted.
1111                 _, err := os.Stat(realTemp)
1112                 c.Assert(os.IsNotExist(err), Equals, true)
1113
1114                 // Now recreate it for the next test.
1115                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1116         }
1117
1118         {
1119                 i = 0
1120                 cr.ArvMountPoint = ""
1121                 cr.Container.Mounts = make(map[string]arvados.Mount)
1122                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1123                 cr.Container.OutputPath = "/tmp"
1124                 cr.statInterval = 5 * time.Second
1125                 bindmounts, err := cr.SetupMounts()
1126                 c.Check(err, IsNil)
1127                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1128                         "--read-write", "--disable-event-listening", "--storage-classes", "default", "--crunchstat-interval=5",
1129                         "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1130                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1131                 os.RemoveAll(cr.ArvMountPoint)
1132                 cr.CleanupDirs()
1133                 checkEmpty()
1134         }
1135
1136         {
1137                 i = 0
1138                 cr.ArvMountPoint = ""
1139                 cr.Container.Mounts = make(map[string]arvados.Mount)
1140                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1141                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1142                 cr.Container.OutputPath = "/out"
1143                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1144
1145                 bindmounts, err := cr.SetupMounts()
1146                 c.Check(err, IsNil)
1147                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1148                         "--read-write", "--disable-event-listening", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1149                         "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1150                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1151                 os.RemoveAll(cr.ArvMountPoint)
1152                 cr.CleanupDirs()
1153                 checkEmpty()
1154         }
1155
1156         {
1157                 i = 0
1158                 cr.ArvMountPoint = ""
1159                 cr.Container.Mounts = make(map[string]arvados.Mount)
1160                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1161                 cr.Container.OutputPath = "/tmp"
1162                 cr.Container.RuntimeConstraints.API = true
1163                 cr.Container.OutputStorageClasses = []string{"default"}
1164
1165                 bindmounts, err := cr.SetupMounts()
1166                 c.Check(err, IsNil)
1167                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1168                         "--read-write", "--disable-event-listening", "--storage-classes", "default", "--crunchstat-interval=5",
1169                         "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1170                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1171                 os.RemoveAll(cr.ArvMountPoint)
1172                 cr.CleanupDirs()
1173                 checkEmpty()
1174
1175                 cr.Container.RuntimeConstraints.API = false
1176         }
1177
1178         {
1179                 i = 0
1180                 cr.ArvMountPoint = ""
1181                 cr.Container.Mounts = map[string]arvados.Mount{
1182                         "/keeptmp": {Kind: "collection", Writable: true},
1183                 }
1184                 cr.Container.OutputPath = "/keeptmp"
1185
1186                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1187
1188                 bindmounts, err := cr.SetupMounts()
1189                 c.Check(err, IsNil)
1190                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1191                         "--read-write", "--disable-event-listening", "--storage-classes", "default", "--crunchstat-interval=5",
1192                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1193                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1194                 os.RemoveAll(cr.ArvMountPoint)
1195                 cr.CleanupDirs()
1196                 checkEmpty()
1197         }
1198
1199         {
1200                 i = 0
1201                 cr.ArvMountPoint = ""
1202                 cr.Container.Mounts = map[string]arvados.Mount{
1203                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1204                         "/keepout": {Kind: "collection", Writable: true},
1205                 }
1206                 cr.Container.OutputPath = "/keepout"
1207
1208                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1209                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1210
1211                 bindmounts, err := cr.SetupMounts()
1212                 c.Check(err, IsNil)
1213                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1214                         "--read-write", "--disable-event-listening", "--storage-classes", "default", "--crunchstat-interval=5",
1215                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1216                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1217                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1218                         "/keepout": {realTemp + "/keep1/tmp0", false},
1219                 })
1220                 os.RemoveAll(cr.ArvMountPoint)
1221                 cr.CleanupDirs()
1222                 checkEmpty()
1223         }
1224
1225         {
1226                 i = 0
1227                 cr.ArvMountPoint = ""
1228                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1229                 cr.Container.Mounts = map[string]arvados.Mount{
1230                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1231                         "/keepout": {Kind: "collection", Writable: true},
1232                 }
1233                 cr.Container.OutputPath = "/keepout"
1234
1235                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1236                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1237
1238                 bindmounts, err := cr.SetupMounts()
1239                 c.Check(err, IsNil)
1240                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1241                         "--read-write", "--disable-event-listening", "--storage-classes", "default", "--crunchstat-interval=5",
1242                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1243                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1244                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1245                         "/keepout": {realTemp + "/keep1/tmp0", false},
1246                 })
1247                 os.RemoveAll(cr.ArvMountPoint)
1248                 cr.CleanupDirs()
1249                 checkEmpty()
1250         }
1251
1252         for _, test := range []struct {
1253                 in  interface{}
1254                 out string
1255         }{
1256                 {in: "foo", out: `"foo"`},
1257                 {in: nil, out: `null`},
1258                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1259         } {
1260                 i = 0
1261                 cr.ArvMountPoint = ""
1262                 cr.Container.Mounts = map[string]arvados.Mount{
1263                         "/mnt/test.json": {Kind: "json", Content: test.in},
1264                 }
1265                 bindmounts, err := cr.SetupMounts()
1266                 c.Check(err, IsNil)
1267                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1268                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1269                 })
1270                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1271                 c.Check(err, IsNil)
1272                 c.Check(content, DeepEquals, []byte(test.out))
1273                 os.RemoveAll(cr.ArvMountPoint)
1274                 cr.CleanupDirs()
1275                 checkEmpty()
1276         }
1277
1278         for _, test := range []struct {
1279                 in  interface{}
1280                 out string
1281         }{
1282                 {in: "foo", out: `foo`},
1283                 {in: nil, out: "error"},
1284                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1285         } {
1286                 i = 0
1287                 cr.ArvMountPoint = ""
1288                 cr.Container.Mounts = map[string]arvados.Mount{
1289                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1290                 }
1291                 bindmounts, err := cr.SetupMounts()
1292                 if test.out == "error" {
1293                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1294                 } else {
1295                         c.Check(err, IsNil)
1296                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1297                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1298                         })
1299                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1300                         c.Check(err, IsNil)
1301                         c.Check(content, DeepEquals, []byte(test.out))
1302                 }
1303                 os.RemoveAll(cr.ArvMountPoint)
1304                 cr.CleanupDirs()
1305                 checkEmpty()
1306         }
1307
1308         // Read-only mount points are allowed underneath output_dir mount point
1309         {
1310                 i = 0
1311                 cr.ArvMountPoint = ""
1312                 cr.Container.Mounts = make(map[string]arvados.Mount)
1313                 cr.Container.Mounts = map[string]arvados.Mount{
1314                         "/tmp":     {Kind: "tmp"},
1315                         "/tmp/foo": {Kind: "collection"},
1316                 }
1317                 cr.Container.OutputPath = "/tmp"
1318
1319                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1320
1321                 bindmounts, err := cr.SetupMounts()
1322                 c.Check(err, IsNil)
1323                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1324                         "--read-write", "--disable-event-listening", "--storage-classes", "default", "--crunchstat-interval=5",
1325                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1326                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1327                         "/tmp":     {realTemp + "/tmp2", false},
1328                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1329                 })
1330                 os.RemoveAll(cr.ArvMountPoint)
1331                 cr.CleanupDirs()
1332                 checkEmpty()
1333         }
1334
1335         // Writable mount points copied to output_dir mount point
1336         {
1337                 i = 0
1338                 cr.ArvMountPoint = ""
1339                 cr.Container.Mounts = make(map[string]arvados.Mount)
1340                 cr.Container.Mounts = map[string]arvados.Mount{
1341                         "/tmp": {Kind: "tmp"},
1342                         "/tmp/foo": {Kind: "collection",
1343                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1344                                 Writable:         true},
1345                         "/tmp/bar": {Kind: "collection",
1346                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1347                                 Path:             "baz",
1348                                 Writable:         true},
1349                 }
1350                 cr.Container.OutputPath = "/tmp"
1351
1352                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1353                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1354
1355                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1356                 rf.Write([]byte("bar"))
1357                 rf.Close()
1358
1359                 _, err := cr.SetupMounts()
1360                 c.Check(err, IsNil)
1361                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1362                 c.Check(err, IsNil)
1363                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1364                 c.Check(err, IsNil)
1365                 os.RemoveAll(cr.ArvMountPoint)
1366                 cr.CleanupDirs()
1367                 checkEmpty()
1368         }
1369
1370         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1371         {
1372                 i = 0
1373                 cr.ArvMountPoint = ""
1374                 cr.Container.Mounts = make(map[string]arvados.Mount)
1375                 cr.Container.Mounts = map[string]arvados.Mount{
1376                         "/tmp":     {Kind: "tmp"},
1377                         "/tmp/foo": {Kind: "tmp"},
1378                 }
1379                 cr.Container.OutputPath = "/tmp"
1380
1381                 _, err := cr.SetupMounts()
1382                 c.Check(err, NotNil)
1383                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1384                 os.RemoveAll(cr.ArvMountPoint)
1385                 cr.CleanupDirs()
1386                 checkEmpty()
1387         }
1388
1389         // Only mount point of kind 'collection' is allowed for stdin
1390         {
1391                 i = 0
1392                 cr.ArvMountPoint = ""
1393                 cr.Container.Mounts = make(map[string]arvados.Mount)
1394                 cr.Container.Mounts = map[string]arvados.Mount{
1395                         "stdin": {Kind: "tmp"},
1396                 }
1397
1398                 _, err := cr.SetupMounts()
1399                 c.Check(err, NotNil)
1400                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1401                 os.RemoveAll(cr.ArvMountPoint)
1402                 cr.CleanupDirs()
1403                 checkEmpty()
1404         }
1405
1406         // git_tree mounts
1407         {
1408                 i = 0
1409                 cr.ArvMountPoint = ""
1410                 (*GitMountSuite)(nil).useTestGitServer(c)
1411                 cr.token = arvadostest.ActiveToken
1412                 cr.Container.Mounts = make(map[string]arvados.Mount)
1413                 cr.Container.Mounts = map[string]arvados.Mount{
1414                         "/tip": {
1415                                 Kind:   "git_tree",
1416                                 UUID:   arvadostest.Repository2UUID,
1417                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1418                                 Path:   "/",
1419                         },
1420                         "/non-tip": {
1421                                 Kind:   "git_tree",
1422                                 UUID:   arvadostest.Repository2UUID,
1423                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1424                                 Path:   "/",
1425                         },
1426                 }
1427                 cr.Container.OutputPath = "/tmp"
1428
1429                 bindmounts, err := cr.SetupMounts()
1430                 c.Check(err, IsNil)
1431
1432                 for path, mount := range bindmounts {
1433                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1434                 }
1435
1436                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1437                 c.Check(err, IsNil)
1438                 c.Check(string(data), Equals, "\000\001\002\003")
1439                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1440                 c.Check(err, FitsTypeOf, &os.PathError{})
1441                 c.Check(os.IsNotExist(err), Equals, true)
1442
1443                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1444                 c.Check(err, IsNil)
1445                 c.Check(string(data), Equals, "\000\001\002\003")
1446                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1447                 c.Check(err, IsNil)
1448                 c.Check(string(data), Equals, "testfile\n")
1449
1450                 cr.CleanupDirs()
1451                 checkEmpty()
1452         }
1453 }
1454
1455 func (s *TestSuite) TestStdout(c *C) {
1456         helperRecord := `{
1457                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1458                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1459                 "cwd": "/bin",
1460                 "environment": {"FROBIZ": "bilbo"},
1461                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1462                 "output_path": "/tmp",
1463                 "priority": 1,
1464                 "runtime_constraints": {},
1465                 "state": "Locked"
1466         }`
1467
1468         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1469                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1470         })
1471
1472         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1473         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1474         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1475 }
1476
1477 // Used by the TestStdoutWithWrongPath*()
1478 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1479         err := json.Unmarshal([]byte(record), &s.api.Container)
1480         c.Assert(err, IsNil)
1481         s.executor.runFunc = fn
1482         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1483         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1484                 return s.api, &KeepTestClient{}, nil, nil
1485         }
1486         return s.api, s.runner, s.runner.Run()
1487 }
1488
1489 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1490         _, _, err := s.stdoutErrorRunHelper(c, `{
1491     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1492     "output_path": "/tmp",
1493     "state": "Locked"
1494 }`, func() {})
1495         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1496 }
1497
1498 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1499         _, _, err := s.stdoutErrorRunHelper(c, `{
1500     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1501     "output_path": "/tmp",
1502     "state": "Locked"
1503 }`, func() {})
1504         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1505 }
1506
1507 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1508         _, _, err := s.stdoutErrorRunHelper(c, `{
1509     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1510     "output_path": "/tmp",
1511     "state": "Locked"
1512 }`, func() {})
1513         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1514 }
1515
1516 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1517         s.fullRunHelper(c, `{
1518     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1519     "container_image": "`+arvadostest.DockerImage112PDH+`",
1520     "cwd": "/bin",
1521     "environment": {},
1522     "mounts": {"/tmp": {"kind": "tmp"} },
1523     "output_path": "/tmp",
1524     "priority": 1,
1525     "runtime_constraints": {"API": true},
1526     "state": "Locked"
1527 }`, nil, 0, func() {
1528                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1529                 s.executor.exit <- 3
1530         })
1531         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1532         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1533 }
1534
1535 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1536         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1537         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1538         s.fullRunHelper(c, `{
1539     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1540     "container_image": "`+arvadostest.DockerImage112PDH+`",
1541     "cwd": "/bin",
1542     "environment": {},
1543     "mounts": {"/tmp": {"kind": "tmp"} },
1544     "output_path": "/tmp",
1545     "priority": 1,
1546     "runtime_constraints": {"API": true},
1547     "state": "Locked"
1548 }`, nil, 0, func() {
1549                 s.api.Container.Output = arvadostest.DockerImage112PDH
1550         })
1551
1552         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1553         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1554         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1555 }
1556
1557 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1558         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1559                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1560                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1561                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1562         }
1563         s.executor.runFunc = func() {
1564                 time.Sleep(time.Second)
1565                 s.executor.exit <- 0
1566         }
1567         record := `{
1568     "command": ["sleep", "1"],
1569     "container_image": "` + arvadostest.DockerImage112PDH + `",
1570     "cwd": "/bin",
1571     "environment": {},
1572     "mounts": {"/tmp": {"kind": "tmp"} },
1573     "output_path": "/tmp",
1574     "priority": 1,
1575     "runtime_constraints": {"API": true},
1576     "state": "Locked"
1577 }`
1578         err := json.Unmarshal([]byte(record), &s.api.Container)
1579         c.Assert(err, IsNil)
1580         err = s.runner.Run()
1581         c.Assert(err, IsNil)
1582         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1583         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1584         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1585         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1586 }
1587
1588 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1589         helperRecord := `{
1590                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1591                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1592                 "cwd": "/bin",
1593                 "environment": {"FROBIZ": "bilbo"},
1594                 "mounts": {
1595         "/tmp": {"kind": "tmp"},
1596         "/tmp/foo": {"kind": "collection",
1597                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1598                      "exclude_from_output": true
1599         },
1600         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1601     },
1602                 "output_path": "/tmp",
1603                 "priority": 1,
1604                 "runtime_constraints": {},
1605                 "state": "Locked"
1606         }`
1607
1608         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1609
1610         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1611                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1612         })
1613
1614         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1615         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1616         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1617 }
1618
1619 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1620         helperRecord := `{
1621                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1622                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1623                 "cwd": "/bin",
1624                 "environment": {"FROBIZ": "bilbo"},
1625                 "mounts": {
1626         "/tmp": {"kind": "tmp"},
1627         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1628         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1629         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1630         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1631         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1632     },
1633                 "output_path": "/tmp",
1634                 "priority": 1,
1635                 "runtime_constraints": {},
1636                 "state": "Locked"
1637         }`
1638
1639         extraMounts := []string{
1640                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1641                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1642                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1643         }
1644
1645         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1646                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1647         })
1648
1649         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1650                 "/tmp":                   {realtemp + "/tmp1", false},
1651                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1652                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1653                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1654                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1655         })
1656
1657         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1658         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1659         for _, v := range api.Content {
1660                 if v["collection"] != nil {
1661                         c.Check(v["ensure_unique_name"], Equals, true)
1662                         collection := v["collection"].(arvadosclient.Dict)
1663                         if strings.Index(collection["name"].(string), "output") == 0 {
1664                                 manifest := collection["manifest_text"].(string)
1665
1666                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1667 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1668 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1669 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1670 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1671 `)
1672                         }
1673                 }
1674         }
1675 }
1676
1677 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1678         helperRecord := `{
1679                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1680                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1681                 "cwd": "/bin",
1682                 "environment": {"FROBIZ": "bilbo"},
1683                 "mounts": {
1684         "/tmp": {"kind": "tmp"},
1685         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1686         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1687     },
1688                 "output_path": "/tmp",
1689                 "priority": 1,
1690                 "runtime_constraints": {},
1691                 "state": "Locked"
1692         }`
1693
1694         extraMounts := []string{
1695                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1696         }
1697
1698         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1699                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1700         })
1701
1702         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1703         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1704         for _, v := range s.api.Content {
1705                 if v["collection"] != nil {
1706                         collection := v["collection"].(arvadosclient.Dict)
1707                         if strings.Index(collection["name"].(string), "output") == 0 {
1708                                 manifest := collection["manifest_text"].(string)
1709
1710                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1711 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1712 `)
1713                         }
1714                 }
1715         }
1716 }
1717
1718 func (s *TestSuite) TestOutputError(c *C) {
1719         helperRecord := `{
1720                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1721                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1722                 "cwd": "/bin",
1723                 "environment": {"FROBIZ": "bilbo"},
1724                 "mounts": {
1725                         "/tmp": {"kind": "tmp"}
1726                 },
1727                 "output_path": "/tmp",
1728                 "priority": 1,
1729                 "runtime_constraints": {},
1730                 "state": "Locked"
1731         }`
1732         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1733                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1734         })
1735
1736         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1737 }
1738
1739 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1740         helperRecord := `{
1741                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1742                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1743                 "cwd": "/bin",
1744                 "environment": {"FROBIZ": "bilbo"},
1745                 "mounts": {
1746         "/tmp": {"kind": "tmp"},
1747         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1748         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1749     },
1750                 "output_path": "/tmp",
1751                 "priority": 1,
1752                 "runtime_constraints": {},
1753                 "state": "Locked"
1754         }`
1755
1756         extraMounts := []string{
1757                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1758         }
1759
1760         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1761                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1762         })
1763
1764         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1765         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1766         for _, v := range api.Content {
1767                 if v["collection"] != nil {
1768                         collection := v["collection"].(arvadosclient.Dict)
1769                         if strings.Index(collection["name"].(string), "output") == 0 {
1770                                 manifest := collection["manifest_text"].(string)
1771                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1772 `)
1773                         }
1774                 }
1775         }
1776 }
1777
1778 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1779         helperRecord := `{
1780                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1781                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1782                 "cwd": "/bin",
1783                 "environment": {"FROBIZ": "bilbo"},
1784                 "mounts": {
1785         "/tmp": {"kind": "tmp"},
1786         "stdin": {"kind": "json", "content": "foo"},
1787         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1788     },
1789                 "output_path": "/tmp",
1790                 "priority": 1,
1791                 "runtime_constraints": {},
1792                 "state": "Locked"
1793         }`
1794
1795         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1796                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1797         })
1798
1799         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1800         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1801         for _, v := range api.Content {
1802                 if v["collection"] != nil {
1803                         collection := v["collection"].(arvadosclient.Dict)
1804                         if strings.Index(collection["name"].(string), "output") == 0 {
1805                                 manifest := collection["manifest_text"].(string)
1806                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1807 `)
1808                         }
1809                 }
1810         }
1811 }
1812
1813 func (s *TestSuite) TestStderrMount(c *C) {
1814         api, cr, _ := s.fullRunHelper(c, `{
1815     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1816     "container_image": "`+arvadostest.DockerImage112PDH+`",
1817     "cwd": ".",
1818     "environment": {},
1819     "mounts": {"/tmp": {"kind": "tmp"},
1820                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1821                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1822     "output_path": "/tmp",
1823     "priority": 1,
1824     "runtime_constraints": {},
1825     "state": "Locked"
1826 }`, nil, 1, func() {
1827                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1828                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1829         })
1830
1831         final := api.CalledWith("container.state", "Complete")
1832         c.Assert(final, NotNil)
1833         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1834         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1835
1836         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1837 }
1838
1839 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1840         s.api.callraw = true
1841         err := s.runner.fetchContainerRecord()
1842         c.Assert(err, IsNil)
1843         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1844         c.Logf("%#v", s.runner.Container)
1845         c.Check(err, IsNil)
1846         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1847 }
1848
1849 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1850         nextState := ""
1851         for _, setup := range []func(){
1852                 func() {
1853                         c.Log("// waitErr = ocl runtime error")
1854                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1855                         nextState = "Cancelled"
1856                 },
1857                 func() {
1858                         c.Log("// loadErr = cannot connect")
1859                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1860                         *brokenNodeHook = c.MkDir() + "/broken-node-hook"
1861                         err := ioutil.WriteFile(*brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1862                         c.Assert(err, IsNil)
1863                         nextState = "Queued"
1864                 },
1865         } {
1866                 s.SetUpTest(c)
1867                 setup()
1868                 s.fullRunHelper(c, `{
1869     "command": ["echo", "hello world"],
1870     "container_image": "`+arvadostest.DockerImage112PDH+`",
1871     "cwd": ".",
1872     "environment": {},
1873     "mounts": {"/tmp": {"kind": "tmp"} },
1874     "output_path": "/tmp",
1875     "priority": 1,
1876     "runtime_constraints": {},
1877     "state": "Locked"
1878 }`, nil, 0, func() {})
1879                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1880                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1881                 if *brokenNodeHook != "" {
1882                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1883                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1884                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1885                 } else {
1886                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1887                 }
1888         }
1889 }
1890
1891 func (s *TestSuite) TestBadCommand(c *C) {
1892         for _, startError := range []string{
1893                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1894                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1895                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1896         } {
1897                 s.SetUpTest(c)
1898                 s.executor.startErr = errors.New(startError)
1899                 s.fullRunHelper(c, `{
1900     "command": ["echo", "hello world"],
1901     "container_image": "`+arvadostest.DockerImage112PDH+`",
1902     "cwd": ".",
1903     "environment": {},
1904     "mounts": {"/tmp": {"kind": "tmp"} },
1905     "output_path": "/tmp",
1906     "priority": 1,
1907     "runtime_constraints": {},
1908     "state": "Locked"
1909 }`, nil, 0, func() {})
1910                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1911                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1912         }
1913 }
1914
1915 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1916         helperRecord := `{
1917                 "command": ["true"],
1918                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1919                 "cwd": "/bin",
1920                 "mounts": {
1921                     "/tmp": {"kind": "tmp"},
1922                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1923                 },
1924                 "secret_mounts": {
1925                 },
1926                 "output_path": "/tmp",
1927                 "priority": 1,
1928                 "runtime_constraints": {},
1929                 "state": "Locked"
1930         }`
1931
1932         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1933                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1934                 c.Check(err, IsNil)
1935                 c.Check(string(content), Equals, "mypassword")
1936         })
1937
1938         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1939         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1940         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
1941         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
1942
1943         // under secret mounts, not captured in output
1944         helperRecord = `{
1945                 "command": ["true"],
1946                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1947                 "cwd": "/bin",
1948                 "mounts": {
1949                     "/tmp": {"kind": "tmp"}
1950                 },
1951                 "secret_mounts": {
1952                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1953                 },
1954                 "output_path": "/tmp",
1955                 "priority": 1,
1956                 "runtime_constraints": {},
1957                 "state": "Locked"
1958         }`
1959
1960         s.SetUpTest(c)
1961         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1962                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1963                 c.Check(err, IsNil)
1964                 c.Check(string(content), Equals, "mypassword")
1965         })
1966
1967         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1968         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1969         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
1970         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
1971 }
1972
1973 type FakeProcess struct {
1974         cmdLine []string
1975 }
1976
1977 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
1978         return fp.cmdLine, nil
1979 }